@crvouga/dev-fleet 0.7.0 → 0.9.0

package/src/machine/support-matrix.ts

@@ -1,6 +1,6 @@
 import macos from '../../ansible/macos/policy.json';
 import platforms from '../../ansible/platforms.json';
-import { UBUNTU_RELEASES } from '../platform/ubuntu-release';
+import { WORKER_PLATFORMS } from '../platform/ubuntu-worker';
 
 export type SupportStatus = 'supported' | 'best effort' | 'refused';
 export interface SupportRow {
@@ -25,14 +25,16 @@ export function supportMatrix(): SupportRow[] {
       status: 'supported',
       note: 'ansible/macos/policy.json',
     },
-    ...UBUNTU_RELEASES.map((entry): SupportRow => ({
-      role: 'worker',
-      platform: 'Ubuntu',
-      versions: `${entry.release} (${entry.codename})`,
-      architectures: entry.architectures.join(', '),
-      status: 'supported',
-      note: 'ansible/ubuntu/policy.json; a machine of its own, physical or virtual',
-    })),
+    ...WORKER_PLATFORMS.flatMap((platform) =>
+      platform.releases.map((entry): SupportRow => ({
+        role: 'worker',
+        platform: platform.label,
+        versions: `${entry.release} (${entry.codename})`,
+        architectures: entry.architectures.join(', '),
+        status: 'supported',
+        note: `${platform.policy}; a machine of its own, physical or virtual`,
+      }))
+    ),
     ...platforms.rows.map((row): SupportRow => ({
       role: row.role,
       platform: row.platform,

package/src/machine-health.ts

@@ -1,5 +1,6 @@
 import policy from '../ansible/ubuntu/policy.json';
 import type { DoctorCheck } from './doctor-check';
+import { threshold } from './worker-sizing';
 
 /** Machine health sample parsed from `net/remote-probe.sh` output. */
 export type HealthSample = {
@@ -8,6 +9,12 @@ export type HealthSample = {
   readonly memFreePct?: number;
   readonly swapPct?: number;
   readonly diskMb?: number;
+  /** The size of the home filesystem, in MiB. */
+  readonly diskTotalMb?: number;
+  /** All memory, in MiB. */
+  readonly memTotalMb?: number;
+  /** Agent sessions running now: T3's agent scopes. */
+  readonly agentSessions?: number;
   /** Free inodes on the home filesystem, as a percentage of all of them. */
   readonly inodesFreePct?: number;
   /** 1 when the clock is synchronised to a time source, 0 when not. */
@@ -24,6 +31,9 @@ const PROBE_KEYS = {
   memfree: 'memFreePct',
   swap: 'swapPct',
   diskmb: 'diskMb',
+  disktotalmb: 'diskTotalMb',
+  memtotalmb: 'memTotalMb',
+  agentsessions: 'agentSessions',
   inodefree: 'inodesFreePct',
   ntpsync: 'clockSynced',
   ntp: 'timeService',
@@ -52,6 +62,8 @@ const LABELS = {
   disk: 'Disk space',
   inodes: 'Free inodes',
 };
+/** The capacity checks of a machine, each reported as `worker:capacity:<id>`. */
+export const HEALTH_CHECK_IDS = Object.keys(LABELS);
 
 function check(
   id: keyof typeof LABELS,
@@ -92,17 +104,28 @@ export function memoryCheck(s: HealthSample): DoctorCheck | null {
   return check('memory', 'pass', what);
 }
 
-/** Free disk: <2 GB fails (VS Code server, git, worktrees break), <10 GB warns. */
-export function diskCheck(diskMb: number | undefined): DoctorCheck | null {
+/** Free disk, against lines that scale with the disk
+ * (`sizing.thresholds.capacityFail` and `capacityWarn` in policy.json): under
+ * the first fails (VS Code server, git, worktrees break), under the second
+ * warns. A disk of unknown size gets the upper bounds.
+ */
+export function diskCheck(
+  diskMb: number | undefined,
+  diskTotalMb?: number
+): DoctorCheck | null {
   if (diskMb === undefined) return null;
-  if (diskMb < 2048) {
+  const total = diskTotalMb ?? Number.MAX_SAFE_INTEGER;
+  const fail = threshold('capacityFail', total);
+  const warn = threshold('capacityWarn', total);
+  if (diskMb < fail) {
     return check(
       'disk',
       'fail',
       `almost full, ${diskMb} MB free. VS Code server, git and T3 worktrees will break.`
     );
   }
-  if (diskMb < 10_240) return check('disk', 'warn', `low, ${diskMb} MB free`);
+  if (diskMb < warn)
+    return check('disk', 'warn', `low, ${diskMb} MB free (under ${warn} MB)`);
   return check('disk', 'pass', `${Math.floor(diskMb / 1024)} GB free`);
 }
 
@@ -136,7 +159,7 @@ export function healthChecks(s: HealthSample, alias: string): DoctorCheck[] {
   return [
     loadCheck(s, alias),
     memoryCheck(s),
-    diskCheck(s.diskMb),
+    diskCheck(s.diskMb, s.diskTotalMb),
     inodesCheck(s.inodesFreePct),
   ].filter((c): c is DoctorCheck => c !== null);
 }

package/src/manager-health.ts

@@ -1,4 +1,5 @@
-import { homedir } from 'node:os';
+import { statSync } from 'node:fs';
+import { homedir, userInfo } from 'node:os';
 import { join } from 'node:path';
 import macos from '../ansible/macos/policy.json';
 import type { Check, Runner } from './types';
@@ -9,16 +10,161 @@ import type { Check, Runner } from './types';
  */
 export async function managerHealth(
   run: Runner,
-  home = homedir()
+  home = homedir(),
+  { uid, user }: { uid: number; user: string } = {
+    uid: userInfo().uid,
+    user: userInfo().username,
+  }
 ): Promise<Check[]> {
   return [
     await managerRelease(run),
     await managerUpdates(run),
     await managerDisk(run),
     await containerWatchdog(run, home),
+    await managerClock(run),
+    await managerWatch(run, uid),
+    await managerJobLogs(run, home, user),
   ];
 }
 
+/** The one grant every administrator step of the manager is given with. */
+const GRANT =
+  'ansible-playbook -e user=$(id -un) -K ansible/macos/manager-privilege.yml';
+
+/** The manager keeps network time: every worker's clock is measured against
+ * it. Reading the setting needs root, through the grant manager-privilege.yml
+ * installs; manager.yml asserts the same reading.
+ */
+export async function managerClock(run: Runner): Promise<Check> {
+  const result = await run(
+    ['sudo', '-n', '/usr/sbin/systemsetup', '-getusingnetworktime'],
+    { timeoutMs: 10_000 }
+  );
+  const on = /Network Time:\s*On\b/i.test(result.stdout);
+  const off = /Network Time:\s*Off\b/i.test(result.stdout);
+  if (!result.ok || (!on && !off))
+    return {
+      id: 'manager:clock',
+      status: 'warn',
+      summary: 'MANAGER_CLOCK_UNOBSERVED',
+      remedy: {
+        action: `Reading whether network time is on needs the manager's grant. Give it once, as an administrator, from the dev-fleet checkout fleet-sync runs: ${GRANT}`,
+      },
+    };
+  if (on)
+    return {
+      id: 'manager:clock',
+      status: 'pass',
+      summary: 'The manager keeps its clock on network time.',
+    };
+  return {
+    id: 'manager:clock',
+    status: 'fail',
+    summary: 'MANAGER_CLOCK_NOT_SYNCHRONISED',
+    remedy: {
+      action:
+        'Turn on Set time and date automatically in System Settings > General > Date & Time (sudo systemsetup -setusingnetworktime on). Every worker clock is measured against the manager.',
+    },
+  };
+}
+
+/** fleet-watch runs on a schedule as the launchd job manager.yml installs:
+ * without it no heartbeat is stored and no notification is sent.
+ */
+export async function managerWatch(run: Runner, uid: number): Promise<Check> {
+  const label = macos.fleetWatch.label;
+  const result = await run(['/bin/launchctl', 'print', `gui/${uid}/${label}`], {
+    timeoutMs: 10_000,
+  });
+  if (!result.ok)
+    return {
+      id: 'manager:watch',
+      status: 'fail',
+      summary: 'MANAGER_WATCH_NOT_LOADED',
+      remedy: {
+        action: `Run devfleet fleet-sync on the manager: it installs and loads ${label} with ansible/macos/manager.yml. Until then no heartbeat reaches the bucket, and the heartbeat workflow fails.`,
+      },
+    };
+  // launchd keeps how the last run ended; a job that never ran has none.
+  const exit = /last exit code = (\d+)/.exec(result.stdout)?.[1];
+  if (exit !== undefined && exit !== '0')
+    return {
+      id: 'manager:watch',
+      status: 'warn',
+      summary: `MANAGER_WATCH_FAILING exit ${exit}`,
+      remedy: {
+        action: `Its last run failed. Read ~/${macos.jobLogs.directory}/${label}.err.log, and run devfleet fleet-watch on the manager to see the same failure.`,
+      },
+    };
+  return {
+    id: 'manager:watch',
+    status: 'pass',
+    summary: `fleet-watch runs on a schedule (${label}).`,
+  };
+}
+
+/** The logs of the manager's scheduled jobs, the size newsyslog keeps them
+ * under, and the drop-in manager-privilege.yml installs for it.
+ */
+export function jobLogs(home: string): string[] {
+  return [macos.fleetSync.label, macos.fleetWatch.label].flatMap((label) =>
+    ['out', 'err'].map((stream) =>
+      join(home, macos.jobLogs.directory, `${label}.${stream}.log`)
+    )
+  );
+}
+
+export const newsyslogDropIn = (user: string): string =>
+  `/etc/newsyslog.d/dev-fleet-${user}.conf`;
+
+/** The logs of the scheduled jobs stay under a fixed size: the newsyslog
+ * drop-in that rotates them is in place, names each of them, and none has
+ * grown past twice the size it is rotated at (newsyslog runs hourly).
+ */
+export async function managerJobLogs(
+  run: Runner,
+  home: string,
+  user: string
+): Promise<Check> {
+  const logs = jobLogs(home);
+  const dropIn = await run(['cat', newsyslogDropIn(user)]);
+  const declared =
+    dropIn.ok &&
+    dropIn.stdout.includes('# Managed by dev-fleet: manager job logs') &&
+    logs.every((log) => dropIn.stdout.includes(`${log} `));
+  if (!declared)
+    return {
+      id: 'manager:job-logs',
+      status: 'warn',
+      summary: 'MANAGER_JOB_LOG_ROTATION_MISSING',
+      remedy: {
+        action: `newsyslog keeps the logs of the scheduled jobs under ${macos.jobLogs.maxKiB} KiB once the manager's grant installs its drop-in. Give it once, as an administrator, from the dev-fleet checkout fleet-sync runs: ${GRANT}`,
+      },
+    };
+  const limit = 2 * macos.jobLogs.maxKiB * 1024;
+  const large = logs.filter((log) => {
+    try {
+      return statSync(log).size > limit;
+    } catch {
+      return false;
+    }
+  });
+  if (large.length)
+    return {
+      id: 'manager:job-logs',
+      status: 'warn',
+      summary: `MANAGER_JOB_LOG_OVERSIZED ${large.length} logs`,
+      remedy: {
+        action: `newsyslog rotates these logs once they pass ${macos.jobLogs.maxKiB} KiB but has not: check that it runs (sudo newsyslog -nv -f ${newsyslogDropIn(user)}) and read /var/log/system.log for its errors.`,
+      },
+    };
+  return {
+    id: 'manager:job-logs',
+    status: 'pass',
+    summary: `newsyslog keeps the scheduled jobs' logs under ${macos.jobLogs.maxKiB} KiB.`,
+  };
+}
+
 /** The manager runs at least the macOS release the policy names. */
 export async function managerRelease(run: Runner): Promise<Check> {
   const id = 'manager:release';

package/src/manager-sync.ts

@@ -12,14 +12,19 @@ import {
   type TailnetComputer,
 } from './tailscale';
 import { matchComputers } from './tailnet-matching';
+import { rebootStopsRollout } from './worker-reboot';
 import { sshArgs, sshFailure } from './ssh';
 import type { FleetDefinition, MachineDefinition } from './definition';
 import { publicHubUrl } from './monitoring/beszel';
 import {
   evaluateEnvironment,
+  evaluateMachineClass,
   evaluatePlatform,
+  evaluateSize,
   PLATFORM_PROBE,
+  probedClock,
 } from './machine/platform';
+import { clockOffsetCheck } from './clock';
 
 export interface ManagerNode extends NodeReport {
   /** The singleton manager converges locally; workers over SSH. */
@@ -37,6 +42,12 @@ export interface ManagerReport {
   ok: boolean;
   applied: boolean;
   desiredRevision: string | null;
+  /** The worker that proved a revision new to the fleet first, when one did. */
+  canary?: string;
+  /** The workers this run enrolled (added to the roster, or bound to their
+   * tailnet device).
+   */
+  enrolled?: string[];
   checks: Check[];
   nodes: ManagerNode[];
   candidates: number;
@@ -47,17 +58,30 @@ export type DeployNode = (
   revision: string
 ) => Promise<NodeReport>;
 
-/** Minimal read-only POSIX inspection requires no workspace, Node, Bun or Git. */
+/** Minimal read-only POSIX inspection requires no workspace, Node, Bun or Git.
+ * Given the manager's clock, it also measures the worker's clock against it
+ * (`worker:clock-offset`).
+ */
 export async function inspectBootstrap(
   node: FleetNode,
-  run: Runner
+  run: Runner,
+  clock?: () => number
 ): Promise<Check[]> {
+  const sentAt = clock?.();
   const result = await run([...sshArgs(node), PLATFORM_PROBE], {
     timeoutMs: 20_000,
   });
+  const receivedAt = clock?.();
   if (!result.ok) return [sshFailure(result)];
   const baseline = evaluatePlatform(result.stdout);
   const environment = evaluateEnvironment(result.stdout);
+  const remote = probedClock(result.stdout);
+  const offset =
+    remote === undefined || sentAt === undefined || receivedAt === undefined
+      ? []
+      : [clockOffsetCheck(remote, sentAt, receivedAt)];
+  const machine = evaluateMachineClass(result.stdout);
+  const size = evaluateSize(result.stdout);
   return [
     {
       id: 'ssh:fresh-session',
@@ -83,6 +107,25 @@ export async function inspectBootstrap(
           'Run the worker on a physical or virtual machine of its own; docs/support-matrix.md lists what is supported.',
       },
     },
+    { id: machine.id, status: 'pass', summary: machine.summary },
+    ...(size
+      ? [
+          {
+            id: size.id,
+            status: size.supported ? ('pass' as const) : ('fail' as const),
+            summary: size.summary,
+            ...(size.supported
+              ? {}
+              : {
+                  remedy: {
+                    action:
+                      'Resize the machine to at least the minimum worker size docs/worker-contract.md states, then run fleet-sync. Nothing was installed.',
+                  },
+                }),
+          },
+        ]
+      : []),
+    ...offset,
   ];
 }
 
@@ -96,11 +139,17 @@ export async function synchronizeManager(
     revision: string;
     apply: boolean;
     canary?: string;
+    /** One worker at a time, canary first; a worker a reboot left unready
+     * stops the ones after it.
+     */
+    serial?: boolean;
     progress?: (node: string, phase: string) => void;
     /** Called once per node as soon as its result is final. */
     finished?: (node: ManagerNode) => void;
     /** Converges the manager locally; omitted in tests. */
     manager?: () => Promise<ManagerNode>;
+    /** The manager's clock, in milliseconds; workers are measured against it. */
+    clock?: () => number;
   },
   run: Runner,
   deploy: DeployNode
@@ -144,6 +193,7 @@ export async function synchronizeManager(
           revision: input.revision,
           computers: tailnet.computers,
           matches,
+          clock: input.clock ?? Date.now,
         },
         run
       );
@@ -190,18 +240,17 @@ async function applyNodes(
     nodes: FleetNode[];
     revision: string;
     canary?: string;
+    serial?: boolean;
     progress?: (node: string, phase: string) => void;
     finished?: (node: ManagerNode) => void;
   },
   deploy: DeployNode
 ): Promise<void> {
   const workers = report.nodes.filter((node) => node.role !== 'manager');
-  // A revision new to the fleet is proven on one worker first. When the run
-  // selects only later workers, the first selected one proves it instead:
-  // narrowing a run never skips the canary.
   const canary = input.canary
-    ? (workers.find((node) => node.node === input.canary) ?? workers[0])
+    ? chooseCanary(workers, input.canary, input.nodes)
     : undefined;
+  if (canary) report.canary = canary.node;
   let blocked = report.checks.some(blocking);
   const step = async (result: ManagerNode) => {
     const deployable = mayDeploy(result, input.nodes);
@@ -216,7 +265,36 @@ async function applyNodes(
     const deployable = await step(canary);
     if (!deployable || canary.checks.some(blocksPromotion)) blocked = true;
   }
-  await Promise.all(workers.filter((node) => node !== canary).map(step));
+  const rest = workers.filter((node) => node !== canary);
+  if (!input.serial) {
+    await Promise.all(rest.map(step));
+    return;
+  }
+  if (canary && rebootStopsRollout(canary)) blocked = true;
+  for (const node of rest) {
+    await step(node);
+    if (rebootStopsRollout(node)) blocked = true;
+  }
+}
+
+/** A revision new to the fleet is proven on one worker first: the preferred
+ * one, or when it cannot be reached (powered off, being rebuilt, offline)
+ * the next worker in declared order that can. An unreachable worker is
+ * reported as such and never blocks the others; only a canary that was
+ * deployed and failed stops promotion. When the run selects only later
+ * workers, the first selected one proves it instead: narrowing a run never
+ * skips the canary.
+ */
+function chooseCanary(
+  workers: ManagerNode[],
+  preferred: string,
+  nodes: FleetNode[]
+): ManagerNode | undefined {
+  const first = workers.find((node) => node.node === preferred);
+  const order = first
+    ? [first, ...workers.filter((node) => node !== first)]
+    : workers;
+  return order.find((node) => mayDeploy(node, nodes));
 }
 
 /** One node's rollout step; true when its remote operation ran. */
@@ -302,6 +380,7 @@ async function inspectManagerNode(
     revision: string;
     computers: TailnetComputer[];
     matches: Map<string, FleetNode[]>;
+    clock: () => number;
   },
   run: Runner
 ): Promise<ManagerNode> {
@@ -337,7 +416,9 @@ async function inspectManagerNode(
       },
     });
   else {
-    checks.push(...(await inspectBootstrap(node, run)));
+    // fleet-sync runs on the manager: the worker's clock is measured
+    // against the manager's own.
+    checks.push(...(await inspectBootstrap(node, run, context.clock)));
     checks.push(...identityChecks(node, peer));
     if (peer) checks.push(keyExpiryCheck(peer));
   }

package/src/monitoring/beszel.ts

@@ -27,7 +27,7 @@ function check(
   summary: string,
   action: string
 ): Check {
-  return { id: `beszel:${id}`, status, summary, remedy: { action } };
+  return { id, status, summary, remedy: { action } };
 }
 /** Public dashboard hostname served through the Cloudflare
  * tunnel `beszel`. Agents and the manager keep using the private hub URL.
@@ -68,7 +68,7 @@ export async function verifyBeszel(
     if (!object(info) || info['v'] !== lock.version) {
       result.checks.push(
         check(
-          'version',
+          'beszel:version',
           'fail',
           'BESZEL_VERSION_MISMATCH',
           'Apply the pinned hub release before using its versioned API contract.'
@@ -78,7 +78,7 @@ export async function verifyBeszel(
     }
     result.checks.push(
       check(
-        'version',
+        'beszel:version',
         'pass',
         `Authenticated hub reports ${lock.version}.`,
         'None.'
@@ -93,7 +93,7 @@ export async function verifyBeszel(
     if (records.length !== 1) {
       result.checks.push(
         check(
-          'registration',
+          'beszel:registration',
           'fail',
           records.length ? 'BESZEL_DUPLICATE_NODE' : 'BESZEL_NODE_MISSING',
           'Review the canonical node in the hub. Preserve its fingerprint and history; do not create a replacement identity.'
@@ -105,7 +105,7 @@ export async function verifyBeszel(
   } catch {
     result.checks.push(
       check(
-        'api',
+        'beszel:api',
         'fail',
         'BESZEL_API_UNVERIFIED',
         'Verify private reachability and a valid hub account with access to this node. Raw HTTP bodies are withheld.'
@@ -147,7 +147,7 @@ async function verifySample(
     throw new Error('BESZEL_PROTOCOL');
   result.checks.push(
     check(
-      'registration',
+      'beszel:registration',
       'pass',
       'Exactly one canonical node record is visible to this account.',
       'None.'
@@ -155,7 +155,7 @@ async function verifySample(
   );
   result.checks.push(
     check(
-      'connection',
+      'beszel:connection',
       record['status'] === 'up' ? 'pass' : 'fail',
       record['status'] === 'up'
         ? 'Hub reports node connected.'
@@ -187,7 +187,7 @@ async function checkFreshness(
     : null;
   result.checks.push(
     check(
-      'freshness',
+      'beszel:freshness',
       fresh ? 'pass' : 'fail',
       fresh
         ? `Hub received sample ${result.receivedAt}; within 180 seconds.`

package/src/monitoring/enroll.ts

@@ -3,7 +3,14 @@ import { record } from '../config';
 import { privateHub } from './beszel';
 import lock from '../../ansible/monitoring/lock.json';
 
-export type HubRequest = (path: string, body?: object) => Promise<unknown>;
+/** A call to the hub's API: GET without a body, POST with one, or the
+ * method named (PATCH a record, DELETE one).
+ */
+export type HubRequest = (
+  path: string,
+  body?: object,
+  method?: 'PATCH' | 'DELETE'
+) => Promise<unknown>;
 export interface AgentEnrollment {
   systemId: string;
   hubUrl: string;

package/src/monitoring/hub-access.ts

@@ -71,7 +71,7 @@ export async function hubAccess(
     url,
     '/api/collections/users/auth-with-password',
     '',
-    credentials
+    { body: credentials }
   );
   if (
     !record(login) ||
@@ -83,24 +83,30 @@ export async function hubAccess(
   const token = login['token'];
   return {
     userId: login['record']['id'],
-    request: (path, body) => call(url, path, token, body),
+    request: (path, body, method) =>
+      call(url, path, token, {
+        ...(body ? { body } : {}),
+        ...(method ? { method } : {}),
+      }),
   };
 }
 async function call(
   url: string,
   path: string,
   token: string,
-  body?: object
+  { body, method }: { body?: object; method?: 'PATCH' | 'DELETE' } = {}
 ): Promise<unknown> {
   if (!path.startsWith('/api/')) throw new Error('BESZEL_API_PATH');
   const response = await fetch(`${url}${path}`, {
-    method: body ? 'POST' : 'GET',
+    method: method ?? (body ? 'POST' : 'GET'),
     headers: { Authorization: token, 'Content-Type': 'application/json' },
     ...(body ? { body: JSON.stringify(body) } : {}),
     redirect: 'error',
     signal: AbortSignal.timeout(10_000),
   });
   if (!response.ok) throw new Error(failure(response.status, path));
+  // PocketBase answers a deletion with no content.
+  if (response.status === 204) return null;
   return response.json() as Promise<unknown>;
 }