@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/cli/worker-usage-report.ts

@@ -0,0 +1,63 @@
+import { fleet } from '../fleet-config';
+import { readParked } from '../state/workers';
+import type { TailnetComputer } from '../tailscale';
+import type { FleetNode, NodeReport, Runner } from '../types';
+import { fleetUsage, usageSummary, type WorkerUsage } from '../worker-activity';
+import { parkedCheck, parkedOnline } from '../worker-parking';
+
+/** The workers fleet-status contacts: every one that is not parked. */
+export function activeNodes(nodes: FleetNode[]): FleetNode[] {
+  const parked = readParked();
+  return nodes.filter((node) => !parked[node.id]);
+}
+
+function reportOf(reports: NodeReport[], id: string): NodeReport {
+  const worker = fleet.workers.find((entry) => entry.id === id);
+  const found = reports.find(
+    (entry) =>
+      entry.node === id ||
+      (worker?.network && entry.node === worker.network.tailscaleName)
+  );
+  if (found) return found;
+  const added: NodeReport = { node: id, checks: [] };
+  reports.push(added);
+  return added;
+}
+
+/** Adds to fleet-status what it says about each worker beyond its health:
+ * parked workers as parked (and a warning when one is online anyway), and
+ * for every active worker it reaches its uptime, agent turns, last agent
+ * activity and `worker:idle`. Returns the `workers` of the JSON output and
+ * its `usage` summary.
+ */
+export async function appendUsage(
+  reports: NodeReport[],
+  nodes: FleetNode[],
+  {
+    run,
+    computers = [],
+    now = new Date(),
+  }: { run: Runner; computers?: TailnetComputer[]; now?: Date }
+): Promise<{ workers: WorkerUsage[]; usage: ReturnType<typeof usageSummary> }> {
+  const parked = readParked();
+  for (const worker of fleet.workers) {
+    const at = parked[worker.id];
+    if (!at) continue;
+    const report = reportOf(reports, worker.id);
+    report.checks.push(parkedCheck(worker.id, at, now));
+    const online = parkedOnline(
+      worker.id,
+      worker.network?.tailscaleName,
+      computers
+    );
+    if (online) report.checks.push(online);
+  }
+  const { workers, checks } = await fleetUsage(fleet, {
+    nodes: nodes.filter((node) => !parked[node.id]),
+    parked,
+    run,
+    now,
+  });
+  for (const [id, check] of checks) reportOf(reports, id).checks.push(check);
+  return { workers, usage: usageSummary(workers) };
+}

package/src/clock.ts

@@ -53,6 +53,48 @@ export function clockCheck(
   };
 }
 
+/** The offset of a worker's clock from the manager's, measured by the
+ * manager around one SSH round trip: the worker read its clock between
+ * `sentAt` and `receivedAt` (milliseconds on the manager's clock), so the
+ * offset is only known to lie in that window. The check judges the smallest
+ * offset the window allows, so a slow link never reads as a skewed clock.
+ */
+export function clockOffsetCheck(
+  remoteSeconds: number,
+  sentAt: number,
+  receivedAt: number
+): Check {
+  const remote = remoteSeconds * 1000;
+  const least =
+    remote > receivedAt
+      ? (remote - receivedAt) / 1000
+      : remote < sentAt
+        ? (sentAt - remote) / 1000
+        : 0;
+  const direction = remote > receivedAt ? 'ahead of' : 'behind';
+  const measured = `${least.toFixed(1)} s ${direction} the manager`;
+  const remedy = clockRemedy(undefined);
+  if (least > failOffsetSeconds)
+    return {
+      id: 'worker:clock-offset',
+      status: 'fail',
+      summary: `CLOCK_SKEWED ${measured}`,
+      remedy,
+    };
+  if (least > warnOffsetSeconds)
+    return {
+      id: 'worker:clock-offset',
+      status: 'warn',
+      summary: `CLOCK_DRIFTING ${measured}`,
+      remedy,
+    };
+  return {
+    id: 'worker:clock-offset',
+    status: 'pass',
+    summary: `The clock is within ${warnOffsetSeconds} s of the manager's.`,
+  };
+}
+
 function clockRemedy(timeService: number | undefined): { action: string } {
   return {
     action: