@crvouga/dev-fleet 0.8.0 → 0.9.0

package/src/manager-health.ts

@@ -1,4 +1,5 @@
-import { homedir } from 'node:os';
+import { statSync } from 'node:fs';
+import { homedir, userInfo } from 'node:os';
 import { join } from 'node:path';
 import macos from '../ansible/macos/policy.json';
 import type { Check, Runner } from './types';
@@ -9,16 +10,161 @@ import type { Check, Runner } from './types';
  */
 export async function managerHealth(
   run: Runner,
-  home = homedir()
+  home = homedir(),
+  { uid, user }: { uid: number; user: string } = {
+    uid: userInfo().uid,
+    user: userInfo().username,
+  }
 ): Promise<Check[]> {
   return [
     await managerRelease(run),
     await managerUpdates(run),
     await managerDisk(run),
     await containerWatchdog(run, home),
+    await managerClock(run),
+    await managerWatch(run, uid),
+    await managerJobLogs(run, home, user),
   ];
 }
 
+/** The one grant every administrator step of the manager is given with. */
+const GRANT =
+  'ansible-playbook -e user=$(id -un) -K ansible/macos/manager-privilege.yml';
+
+/** The manager keeps network time: every worker's clock is measured against
+ * it. Reading the setting needs root, through the grant manager-privilege.yml
+ * installs; manager.yml asserts the same reading.
+ */
+export async function managerClock(run: Runner): Promise<Check> {
+  const result = await run(
+    ['sudo', '-n', '/usr/sbin/systemsetup', '-getusingnetworktime'],
+    { timeoutMs: 10_000 }
+  );
+  const on = /Network Time:\s*On\b/i.test(result.stdout);
+  const off = /Network Time:\s*Off\b/i.test(result.stdout);
+  if (!result.ok || (!on && !off))
+    return {
+      id: 'manager:clock',
+      status: 'warn',
+      summary: 'MANAGER_CLOCK_UNOBSERVED',
+      remedy: {
+        action: `Reading whether network time is on needs the manager's grant. Give it once, as an administrator, from the dev-fleet checkout fleet-sync runs: ${GRANT}`,
+      },
+    };
+  if (on)
+    return {
+      id: 'manager:clock',
+      status: 'pass',
+      summary: 'The manager keeps its clock on network time.',
+    };
+  return {
+    id: 'manager:clock',
+    status: 'fail',
+    summary: 'MANAGER_CLOCK_NOT_SYNCHRONISED',
+    remedy: {
+      action:
+        'Turn on Set time and date automatically in System Settings > General > Date & Time (sudo systemsetup -setusingnetworktime on). Every worker clock is measured against the manager.',
+    },
+  };
+}
+
+/** fleet-watch runs on a schedule as the launchd job manager.yml installs:
+ * without it no heartbeat is stored and no notification is sent.
+ */
+export async function managerWatch(run: Runner, uid: number): Promise<Check> {
+  const label = macos.fleetWatch.label;
+  const result = await run(['/bin/launchctl', 'print', `gui/${uid}/${label}`], {
+    timeoutMs: 10_000,
+  });
+  if (!result.ok)
+    return {
+      id: 'manager:watch',
+      status: 'fail',
+      summary: 'MANAGER_WATCH_NOT_LOADED',
+      remedy: {
+        action: `Run devfleet fleet-sync on the manager: it installs and loads ${label} with ansible/macos/manager.yml. Until then no heartbeat reaches the bucket, and the heartbeat workflow fails.`,
+      },
+    };
+  // launchd keeps how the last run ended; a job that never ran has none.
+  const exit = /last exit code = (\d+)/.exec(result.stdout)?.[1];
+  if (exit !== undefined && exit !== '0')
+    return {
+      id: 'manager:watch',
+      status: 'warn',
+      summary: `MANAGER_WATCH_FAILING exit ${exit}`,
+      remedy: {
+        action: `Its last run failed. Read ~/${macos.jobLogs.directory}/${label}.err.log, and run devfleet fleet-watch on the manager to see the same failure.`,
+      },
+    };
+  return {
+    id: 'manager:watch',
+    status: 'pass',
+    summary: `fleet-watch runs on a schedule (${label}).`,
+  };
+}
+
+/** The logs of the manager's scheduled jobs, the size newsyslog keeps them
+ * under, and the drop-in manager-privilege.yml installs for it.
+ */
+export function jobLogs(home: string): string[] {
+  return [macos.fleetSync.label, macos.fleetWatch.label].flatMap((label) =>
+    ['out', 'err'].map((stream) =>
+      join(home, macos.jobLogs.directory, `${label}.${stream}.log`)
+    )
+  );
+}
+
+export const newsyslogDropIn = (user: string): string =>
+  `/etc/newsyslog.d/dev-fleet-${user}.conf`;
+
+/** The logs of the scheduled jobs stay under a fixed size: the newsyslog
+ * drop-in that rotates them is in place, names each of them, and none has
+ * grown past twice the size it is rotated at (newsyslog runs hourly).
+ */
+export async function managerJobLogs(
+  run: Runner,
+  home: string,
+  user: string
+): Promise<Check> {
+  const logs = jobLogs(home);
+  const dropIn = await run(['cat', newsyslogDropIn(user)]);
+  const declared =
+    dropIn.ok &&
+    dropIn.stdout.includes('# Managed by dev-fleet: manager job logs') &&
+    logs.every((log) => dropIn.stdout.includes(`${log} `));
+  if (!declared)
+    return {
+      id: 'manager:job-logs',
+      status: 'warn',
+      summary: 'MANAGER_JOB_LOG_ROTATION_MISSING',
+      remedy: {
+        action: `newsyslog keeps the logs of the scheduled jobs under ${macos.jobLogs.maxKiB} KiB once the manager's grant installs its drop-in. Give it once, as an administrator, from the dev-fleet checkout fleet-sync runs: ${GRANT}`,
+      },
+    };
+  const limit = 2 * macos.jobLogs.maxKiB * 1024;
+  const large = logs.filter((log) => {
+    try {
+      return statSync(log).size > limit;
+    } catch {
+      return false;
+    }
+  });
+  if (large.length)
+    return {
+      id: 'manager:job-logs',
+      status: 'warn',
+      summary: `MANAGER_JOB_LOG_OVERSIZED ${large.length} logs`,
+      remedy: {
+        action: `newsyslog rotates these logs once they pass ${macos.jobLogs.maxKiB} KiB but has not: check that it runs (sudo newsyslog -nv -f ${newsyslogDropIn(user)}) and read /var/log/system.log for its errors.`,
+      },
+    };
+  return {
+    id: 'manager:job-logs',
+    status: 'pass',
+    summary: `newsyslog keeps the scheduled jobs' logs under ${macos.jobLogs.maxKiB} KiB.`,
+  };
+}
+
 /** The manager runs at least the macOS release the policy names. */
 export async function managerRelease(run: Runner): Promise<Check> {
   const id = 'manager:release';

package/src/manager-sync.ts

@@ -12,14 +12,19 @@ import {
   type TailnetComputer,
 } from './tailscale';
 import { matchComputers } from './tailnet-matching';
+import { rebootStopsRollout } from './worker-reboot';
 import { sshArgs, sshFailure } from './ssh';
 import type { FleetDefinition, MachineDefinition } from './definition';
 import { publicHubUrl } from './monitoring/beszel';
 import {
   evaluateEnvironment,
+  evaluateMachineClass,
   evaluatePlatform,
+  evaluateSize,
   PLATFORM_PROBE,
+  probedClock,
 } from './machine/platform';
+import { clockOffsetCheck } from './clock';
 
 export interface ManagerNode extends NodeReport {
   /** The singleton manager converges locally; workers over SSH. */
@@ -39,6 +44,10 @@ export interface ManagerReport {
   desiredRevision: string | null;
   /** The worker that proved a revision new to the fleet first, when one did. */
   canary?: string;
+  /** The workers this run enrolled (added to the roster, or bound to their
+   * tailnet device).
+   */
+  enrolled?: string[];
   checks: Check[];
   nodes: ManagerNode[];
   candidates: number;
@@ -49,17 +58,30 @@ export type DeployNode = (
   revision: string
 ) => Promise<NodeReport>;
 
-/** Minimal read-only POSIX inspection requires no workspace, Node, Bun or Git. */
+/** Minimal read-only POSIX inspection requires no workspace, Node, Bun or Git.
+ * Given the manager's clock, it also measures the worker's clock against it
+ * (`worker:clock-offset`).
+ */
 export async function inspectBootstrap(
   node: FleetNode,
-  run: Runner
+  run: Runner,
+  clock?: () => number
 ): Promise<Check[]> {
+  const sentAt = clock?.();
   const result = await run([...sshArgs(node), PLATFORM_PROBE], {
     timeoutMs: 20_000,
   });
+  const receivedAt = clock?.();
   if (!result.ok) return [sshFailure(result)];
   const baseline = evaluatePlatform(result.stdout);
   const environment = evaluateEnvironment(result.stdout);
+  const remote = probedClock(result.stdout);
+  const offset =
+    remote === undefined || sentAt === undefined || receivedAt === undefined
+      ? []
+      : [clockOffsetCheck(remote, sentAt, receivedAt)];
+  const machine = evaluateMachineClass(result.stdout);
+  const size = evaluateSize(result.stdout);
   return [
     {
       id: 'ssh:fresh-session',
@@ -85,6 +107,25 @@ export async function inspectBootstrap(
           'Run the worker on a physical or virtual machine of its own; docs/support-matrix.md lists what is supported.',
       },
     },
+    { id: machine.id, status: 'pass', summary: machine.summary },
+    ...(size
+      ? [
+          {
+            id: size.id,
+            status: size.supported ? ('pass' as const) : ('fail' as const),
+            summary: size.summary,
+            ...(size.supported
+              ? {}
+              : {
+                  remedy: {
+                    action:
+                      'Resize the machine to at least the minimum worker size docs/worker-contract.md states, then run fleet-sync. Nothing was installed.',
+                  },
+                }),
+          },
+        ]
+      : []),
+    ...offset,
   ];
 }
 
@@ -98,11 +139,17 @@ export async function synchronizeManager(
     revision: string;
     apply: boolean;
     canary?: string;
+    /** One worker at a time, canary first; a worker a reboot left unready
+     * stops the ones after it.
+     */
+    serial?: boolean;
     progress?: (node: string, phase: string) => void;
     /** Called once per node as soon as its result is final. */
     finished?: (node: ManagerNode) => void;
     /** Converges the manager locally; omitted in tests. */
     manager?: () => Promise<ManagerNode>;
+    /** The manager's clock, in milliseconds; workers are measured against it. */
+    clock?: () => number;
   },
   run: Runner,
   deploy: DeployNode
@@ -146,6 +193,7 @@ export async function synchronizeManager(
           revision: input.revision,
           computers: tailnet.computers,
           matches,
+          clock: input.clock ?? Date.now,
         },
         run
       );
@@ -192,6 +240,7 @@ async function applyNodes(
     nodes: FleetNode[];
     revision: string;
     canary?: string;
+    serial?: boolean;
     progress?: (node: string, phase: string) => void;
     finished?: (node: ManagerNode) => void;
   },
@@ -216,7 +265,16 @@ async function applyNodes(
     const deployable = await step(canary);
     if (!deployable || canary.checks.some(blocksPromotion)) blocked = true;
   }
-  await Promise.all(workers.filter((node) => node !== canary).map(step));
+  const rest = workers.filter((node) => node !== canary);
+  if (!input.serial) {
+    await Promise.all(rest.map(step));
+    return;
+  }
+  if (canary && rebootStopsRollout(canary)) blocked = true;
+  for (const node of rest) {
+    await step(node);
+    if (rebootStopsRollout(node)) blocked = true;
+  }
 }
 
 /** A revision new to the fleet is proven on one worker first: the preferred
@@ -322,6 +380,7 @@ async function inspectManagerNode(
     revision: string;
     computers: TailnetComputer[];
     matches: Map<string, FleetNode[]>;
+    clock: () => number;
   },
   run: Runner
 ): Promise<ManagerNode> {
@@ -357,7 +416,9 @@ async function inspectManagerNode(
       },
     });
   else {
-    checks.push(...(await inspectBootstrap(node, run)));
+    // fleet-sync runs on the manager: the worker's clock is measured
+    // against the manager's own.
+    checks.push(...(await inspectBootstrap(node, run, context.clock)));
     checks.push(...identityChecks(node, peer));
     if (peer) checks.push(keyExpiryCheck(peer));
   }

package/src/monitoring/beszel.ts

@@ -27,7 +27,7 @@ function check(
   summary: string,
   action: string
 ): Check {
-  return { id: `beszel:${id}`, status, summary, remedy: { action } };
+  return { id, status, summary, remedy: { action } };
 }
 /** Public dashboard hostname served through the Cloudflare
  * tunnel `beszel`. Agents and the manager keep using the private hub URL.
@@ -68,7 +68,7 @@ export async function verifyBeszel(
     if (!object(info) || info['v'] !== lock.version) {
       result.checks.push(
         check(
-          'version',
+          'beszel:version',
           'fail',
           'BESZEL_VERSION_MISMATCH',
           'Apply the pinned hub release before using its versioned API contract.'
@@ -78,7 +78,7 @@ export async function verifyBeszel(
     }
     result.checks.push(
       check(
-        'version',
+        'beszel:version',
         'pass',
         `Authenticated hub reports ${lock.version}.`,
         'None.'
@@ -93,7 +93,7 @@ export async function verifyBeszel(
     if (records.length !== 1) {
       result.checks.push(
         check(
-          'registration',
+          'beszel:registration',
           'fail',
           records.length ? 'BESZEL_DUPLICATE_NODE' : 'BESZEL_NODE_MISSING',
           'Review the canonical node in the hub. Preserve its fingerprint and history; do not create a replacement identity.'
@@ -105,7 +105,7 @@ export async function verifyBeszel(
   } catch {
     result.checks.push(
       check(
-        'api',
+        'beszel:api',
         'fail',
         'BESZEL_API_UNVERIFIED',
         'Verify private reachability and a valid hub account with access to this node. Raw HTTP bodies are withheld.'
@@ -147,7 +147,7 @@ async function verifySample(
     throw new Error('BESZEL_PROTOCOL');
   result.checks.push(
     check(
-      'registration',
+      'beszel:registration',
       'pass',
       'Exactly one canonical node record is visible to this account.',
       'None.'
@@ -155,7 +155,7 @@ async function verifySample(
   );
   result.checks.push(
     check(
-      'connection',
+      'beszel:connection',
       record['status'] === 'up' ? 'pass' : 'fail',
       record['status'] === 'up'
         ? 'Hub reports node connected.'
@@ -187,7 +187,7 @@ async function checkFreshness(
     : null;
   result.checks.push(
     check(
-      'freshness',
+      'beszel:freshness',
       fresh ? 'pass' : 'fail',
       fresh
         ? `Hub received sample ${result.receivedAt}; within 180 seconds.`

package/src/monitoring/enroll.ts

@@ -3,7 +3,14 @@ import { record } from '../config';
 import { privateHub } from './beszel';
 import lock from '../../ansible/monitoring/lock.json';
 
-export type HubRequest = (path: string, body?: object) => Promise<unknown>;
+/** A call to the hub's API: GET without a body, POST with one, or the
+ * method named (PATCH a record, DELETE one).
+ */
+export type HubRequest = (
+  path: string,
+  body?: object,
+  method?: 'PATCH' | 'DELETE'
+) => Promise<unknown>;
 export interface AgentEnrollment {
   systemId: string;
   hubUrl: string;

package/src/monitoring/hub-access.ts

@@ -71,7 +71,7 @@ export async function hubAccess(
     url,
     '/api/collections/users/auth-with-password',
     '',
-    credentials
+    { body: credentials }
   );
   if (
     !record(login) ||
@@ -83,24 +83,30 @@ export async function hubAccess(
   const token = login['token'];
   return {
     userId: login['record']['id'],
-    request: (path, body) => call(url, path, token, body),
+    request: (path, body, method) =>
+      call(url, path, token, {
+        ...(body ? { body } : {}),
+        ...(method ? { method } : {}),
+      }),
   };
 }
 async function call(
   url: string,
   path: string,
   token: string,
-  body?: object
+  { body, method }: { body?: object; method?: 'PATCH' | 'DELETE' } = {}
 ): Promise<unknown> {
   if (!path.startsWith('/api/')) throw new Error('BESZEL_API_PATH');
   const response = await fetch(`${url}${path}`, {
-    method: body ? 'POST' : 'GET',
+    method: method ?? (body ? 'POST' : 'GET'),
     headers: { Authorization: token, 'Content-Type': 'application/json' },
     ...(body ? { body: JSON.stringify(body) } : {}),
     redirect: 'error',
     signal: AbortSignal.timeout(10_000),
   });
   if (!response.ok) throw new Error(failure(response.status, path));
+  // PocketBase answers a deletion with no content.
+  if (response.status === 204) return null;
   return response.json() as Promise<unknown>;
 }
 

package/src/monitoring/systems.ts

@@ -0,0 +1,138 @@
+import { record } from '../config';
+import type { Check } from '../types';
+import type { HubRequest } from './enroll';
+
+/** The hub keeps one system per worker, named by its fleet ID. These change
+ * that list when a worker is replaced or retired, and find the systems no
+ * declared worker matches. Only names and hosts are read; a record's other
+ * fields (tokens, users) are never returned.
+ */
+export interface HubSystem {
+  id: string;
+  name: string;
+  host: string;
+}
+
+function systems(value: unknown): HubSystem[] {
+  if (!record(value) || !Array.isArray(value['items']))
+    throw new Error('BESZEL_PROTOCOL');
+  return value['items'].flatMap((item) =>
+    record(item) &&
+    typeof item['id'] === 'string' &&
+    /^[a-z0-9]{15}$/.test(item['id']) &&
+    typeof item['name'] === 'string'
+      ? [
+          {
+            id: item['id'],
+            name: item['name'],
+            host: typeof item['host'] === 'string' ? item['host'] : '',
+          },
+        ]
+      : []
+  );
+}
+
+export async function listSystems(request: HubRequest): Promise<HubSystem[]> {
+  return systems(
+    await request(
+      '/api/collections/systems/records?perPage=500&fields=id,name,host'
+    )
+  );
+}
+
+async function named(request: HubRequest, node: string): Promise<HubSystem[]> {
+  return systems(
+    await request(
+      `/api/collections/systems/records?filter=${encodeURIComponent(`name="${node}"`)}&perPage=2&fields=id,name,host`
+    )
+  );
+}
+
+const check = (status: Check['status'], summary: string, action?: string) => ({
+  id: 'beszel:system',
+  status,
+  summary,
+  ...(action ? { remedy: { action } } : {}),
+});
+
+/** Points a replaced worker's system at the host it is reached by now, so
+ * the hub keeps one system for the ID and its history.
+ */
+export async function moveSystem(
+  request: HubRequest,
+  node: string,
+  host: string
+): Promise<Check> {
+  const found = await named(request, node);
+  if (found.length > 1)
+    return check(
+      'fail',
+      'BESZEL_DUPLICATE_NODE',
+      `The hub holds more than one system named ${node}. Delete the extra ones in the dashboard, then run the command again.`
+    );
+  if (!found[0])
+    return check(
+      'pass',
+      `The hub holds no system for ${node}; fleet-sync enrolls the new machine.`
+    );
+  if (found[0].host === host)
+    return check('pass', `The hub's system ${node} already watches ${host}.`);
+  await request(
+    `/api/collections/systems/records/${found[0].id}`,
+    { host },
+    'PATCH'
+  );
+  return check('pass', `The hub's system ${node} now watches ${host}.`);
+}
+
+/** Deletes a retired worker's system and its agent fingerprint. */
+export async function removeSystem(
+  request: HubRequest,
+  node: string
+): Promise<Check> {
+  const found = await named(request, node);
+  for (const system of found) {
+    const prints = await request(
+      `/api/collections/fingerprints/records?filter=${encodeURIComponent(`system="${system.id}"`)}&perPage=10&fields=id`
+    );
+    for (const print of record(prints) && Array.isArray(prints['items'])
+      ? prints['items']
+      : [])
+      if (record(print) && typeof print['id'] === 'string')
+        await request(
+          `/api/collections/fingerprints/records/${print['id']}`,
+          undefined,
+          'DELETE'
+        );
+    await request(
+      `/api/collections/systems/records/${system.id}`,
+      undefined,
+      'DELETE'
+    );
+  }
+  return check(
+    'pass',
+    found.length
+      ? `The hub no longer watches ${node}.`
+      : `The hub holds no system for ${node}.`
+  );
+}
+
+/** A system no declared worker (and not the manager) is named after: what a
+ * worker removed by hand, or destroyed outside dev-fleet, leaves behind.
+ */
+export function leftoverSystems(
+  found: HubSystem[],
+  declared: string[]
+): Check[] {
+  return found
+    .filter((system) => !declared.includes(system.name))
+    .map((system) => ({
+      id: 'beszel:leftover',
+      status: 'warn' as const,
+      summary: `The hub watches ${system.name}, which is no declared worker.`,
+      remedy: {
+        action: `If ${system.name} left the fleet, retire it (devfleet fleet-node-retire --id ${system.name} --apply --yes while it is still declared) or delete the system in the dashboard.`,
+      },
+    }));
+}

package/src/network-definition.ts

@@ -1,4 +1,8 @@
-import { type FleetDefinition, type MachineDefinition } from './definition';
+import {
+  sshTarget,
+  type FleetDefinition,
+  type MachineDefinition,
+} from './definition';
 import { record } from './config';
 import { FleetUsageError } from './errors';
 
@@ -19,8 +23,11 @@ export function networkProjection(
     node.network
       ? [
           {
-            alias: node.ssh,
-            ...node.network,
+            alias: sshTarget(node),
+            tailscaleName: node.network.tailscaleName,
+            user: node.network.user,
+            sshPort: node.network.sshPort,
+            t3Port: node.network.t3Port,
             os: node.os,
             identityFile: identities[node.id] ?? '',
           },
@@ -34,7 +41,9 @@ function compatible(
 ): node is MachineDefinition {
   return Boolean(
     node?.network &&
-    Object.entries(node.network).every(([key, value]) => host[key] === value) &&
+    Object.entries(node.network)
+      .filter(([key]) => key !== 'hostKey')
+      .every(([key, value]) => host[key] === value) &&
     node.os === host['os']
   );
 }