@crvouga/dev-fleet 0.3.0 → 0.4.0

package/src/remote-operation.ts

@@ -0,0 +1,213 @@
+import { record } from './config';
+import { cleanCheck, validCheck } from './protocol';
+import { sshArgs, sshFailure } from './ssh';
+import type { Check, CommandResult, FleetNode, Runner } from './types';
+
+/** How the manager waits: a pause, and the time now. Tests answer both. */
+export interface Clock {
+  pause(ms: number): Promise<void>;
+  now(): number;
+}
+
+export const realClock: Clock = {
+  pause: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+  now: () => Date.now(),
+};
+
+/** A session that could not be opened, or was lost, is tried again after
+ * these pauses: three attempts over thirty seconds.
+ */
+export const RETRY_PAUSES_MS = [10_000, 20_000];
+/** How often an operation the manager follows is looked at again. */
+export const FOLLOW_EVERY_MS = 15_000;
+/** As long as a bootstrap may run. */
+export const FOLLOW_AT_MOST_MS = 5_400_000;
+
+/** Failures of the connection, not of the worker: worth another attempt. A
+ * changed host key, a refused login or an unknown name never are.
+ */
+export function transient(result: CommandResult): boolean {
+  if (result.ok) return false;
+  const { id } = sshFailure(result);
+  return ['ssh:disconnected', 'ssh:network', 'ssh:timeout'].includes(id);
+}
+
+/** Runs a short SSH session, again after a pause while its connection fails. */
+export async function retrying(
+  attempt: () => Promise<CommandResult>,
+  clock: Clock
+): Promise<CommandResult> {
+  let result = await attempt();
+  for (const pause of RETRY_PAUSES_MS) {
+    if (!transient(result)) return result;
+    await clock.pause(pause);
+    result = await attempt();
+  }
+  return result;
+}
+
+/** What a worker says about the fleet operation on it. */
+export interface Operation {
+  /** The operation lock is held by a running process. */
+  held: boolean;
+  /** The process that recorded the operation is still running. */
+  alive: boolean;
+  revision: string | null;
+  phase: string | null;
+  /** Seconds since the epoch, on the worker's clock. */
+  started: number | null;
+  finished: boolean;
+  /** What node convergence reported, once the operation finished. */
+  result: Check[] | null;
+}
+
+/** Read-only. Prints whether the lock is held, whether the recorded process
+ * runs, the operation record and its result, one per line. Holding the lock
+ * for a moment to test it is not possible while another holds it, and takes
+ * nothing from anyone when nobody does.
+ */
+export const OPERATION_PROBE = [
+  'r="$HOME/.local/share/personal/fleet"',
+  'l="$r/remote-operation.lock"',
+  'held=no',
+  'if [ -f "$l" ] && [ ! -L "$l" ]; then if command -v flock >/dev/null 2>&1; then flock -n "$l" true || held=yes; else python3 -c \'import fcntl, sys; fcntl.flock(open(sys.argv[1], "a"), fcntl.LOCK_EX | fcntl.LOCK_NB)\' "$l" 2>/dev/null || held=yes; fi; fi',
+  'op=$(head -c 4096 "$r/remote-operation.json" 2>/dev/null | head -n 1)',
+  'pid=$(printf %s "$op" | sed -n \'s/.*"pid":\\([0-9][0-9]*\\).*/\\1/p\')',
+  'alive=no',
+  'if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then alive=yes; fi',
+  'printf \'held=%s\\nalive=%s\\noperation=%s\\nresult=\' "$held" "$alive" "$op"',
+  'head -c 1048576 "$r/remote-operation-result.json" 2>/dev/null | head -n 1',
+  "printf '\\n'",
+].join('\n');
+
+function field(stdout: string, name: string): string | null {
+  const line = stdout.split('\n').find((entry) => entry.startsWith(`${name}=`));
+  return line === undefined ? null : line.slice(name.length + 1);
+}
+
+function json(text: string | null): unknown {
+  try {
+    return text ? JSON.parse(text) : null;
+  } catch {
+    return null;
+  }
+}
+
+function evidence(value: unknown): Check[] | null {
+  return record(value) &&
+    value['version'] === 1 &&
+    Array.isArray(value['checks']) &&
+    value['checks'].length > 0 &&
+    value['checks'].length <= 200 &&
+    value['checks'].every(validCheck)
+    ? value['checks'].map(cleanCheck)
+    : null;
+}
+
+export function parseOperation(stdout: string): Operation | null {
+  const held = field(stdout, 'held');
+  const alive = field(stdout, 'alive');
+  if (held === null || alive === null) return null;
+  const operation = json(field(stdout, 'operation'));
+  const op = record(operation) ? operation : {};
+  const text = (value: unknown, pattern: RegExp): string | null =>
+    typeof value === 'string' && pattern.test(value) ? value : null;
+  return {
+    held: held === 'yes',
+    alive: alive === 'yes',
+    revision: text(op['revision'], /^[a-f0-9]{40}$/),
+    phase: text(op['phase'], /^[a-z0-9-]{1,40}$/),
+    started: typeof op['started'] === 'number' ? op['started'] : null,
+    finished: typeof op['finished'] === 'number',
+    result: evidence(json(field(stdout, 'result'))),
+  };
+}
+
+export async function inspectOperation(
+  node: FleetNode,
+  run: Runner,
+  clock: Clock
+): Promise<Operation | null> {
+  const result = await retrying(
+    () => run([...sshArgs(node), OPERATION_PROBE], { timeoutMs: 20_000 }),
+    clock
+  );
+  return result.ok ? parseOperation(result.stdout) : null;
+}
+
+/** An operation of this revision whose process still runs: this is the run
+ * the manager lost, or one another manager run started; either way it is
+ * the run to follow, not one to start beside it.
+ */
+function followable(operation: Operation, revision: string): boolean {
+  return (
+    operation.held &&
+    operation.alive &&
+    !operation.finished &&
+    operation.revision === revision
+  );
+}
+
+function finishedWith(operation: Operation, revision: string): Check[] | null {
+  return !operation.held &&
+    operation.finished &&
+    operation.revision === revision
+    ? operation.result
+    : null;
+}
+
+export interface Follow {
+  clock?: Clock;
+  /** Told the phase and how long the operation has run, at every look. */
+  progress?: (phase: string) => void;
+}
+
+/** After a lost session or a held lock: the result of the operation of this
+ * revision on the worker, followed to its end when it still runs. Null when
+ * there is none to follow: another revision, another holder, or nothing.
+ */
+export async function followOperation(
+  node: FleetNode,
+  revision: string,
+  run: Runner,
+  options: Follow = {}
+): Promise<Check[] | null> {
+  const clock = options.clock ?? realClock;
+  const deadline = clock.now() + FOLLOW_AT_MOST_MS;
+  let operation = await inspectOperation(node, run, clock);
+  if (!operation) return null;
+  const done = finishedWith(operation, revision);
+  if (done) return [attached(operation, 'finished'), ...done];
+  if (!followable(operation, revision)) return null;
+  let last: Operation = operation;
+  while (clock.now() < deadline) {
+    options.progress?.(describe(last, clock));
+    await clock.pause(FOLLOW_EVERY_MS);
+    operation = await inspectOperation(node, run, clock);
+    if (!operation) continue;
+    last = operation;
+    const result = finishedWith(operation, revision);
+    if (result) return [attached(operation, 'followed'), ...result];
+    if (!followable(operation, revision)) return null;
+  }
+  return null;
+}
+
+function describe(operation: Operation, clock: Clock): string {
+  const elapsed =
+    operation.started === null
+      ? ''
+      : `, ${Math.max(0, Math.round(clock.now() / 1000 - operation.started))} s`;
+  return `Following the running operation (${operation.phase ?? 'unknown phase'}${elapsed})`;
+}
+
+function attached(_operation: Operation, how: 'finished' | 'followed'): Check {
+  return {
+    id: 'bootstrap:attached',
+    status: 'pass',
+    summary:
+      how === 'followed'
+        ? 'The manager lost or could not take this machine’s operation of the same revision, followed it to its end and reports its result.'
+        : 'This machine finished the operation of the same revision the manager lost; its result is reported.',
+  };
+}

package/src/schedule-check.ts

@@ -0,0 +1,40 @@
+import type { History } from './state/history';
+import type { Check } from './types';
+
+/** How often the manager's schedule runs fleet-sync. */
+export const SCHEDULE_INTERVAL_SECONDS = 1800;
+
+/** `manager:schedule`: a schedule that is on must have succeeded within two
+ * of its intervals. A schedule that was never turned on is no check at all.
+ */
+export function scheduleCheck(history: History, now: Date): Check | null {
+  const { schedule } = history;
+  if (!schedule?.enabled) return null;
+  const window = 2 * schedule.intervalSeconds * 1000;
+  const lastSuccess = history.runs
+    .filter((run) => run.trigger === 'schedule' && run.ok)
+    .map((run) => Date.parse(run.finishedAt))
+    .reduce((latest, at) => Math.max(latest, at), Number.NEGATIVE_INFINITY);
+  // A schedule turned on a moment ago has had no chance to run yet.
+  const since = Math.max(lastSuccess, Date.parse(schedule.changedAt));
+  const minutes = Math.round(schedule.intervalSeconds / 60);
+  if (now.getTime() - since <= window)
+    return {
+      id: 'manager:schedule',
+      status: 'pass',
+      summary: Number.isFinite(lastSuccess)
+        ? `The schedule last converged the fleet at ${new Date(lastSuccess).toISOString()}.`
+        : 'The schedule was turned on and has not had to run yet.',
+    };
+  return {
+    id: 'manager:schedule',
+    status: 'fail',
+    summary: Number.isFinite(lastSuccess)
+      ? `The schedule has not converged the fleet since ${new Date(lastSuccess).toISOString()}, more than two intervals of ${minutes} minutes.`
+      : `The schedule has not converged the fleet once since it was turned on, more than two intervals of ${minutes} minutes ago.`,
+    remedy: {
+      action:
+        'Run devfleet fleet-sync --schedule status on the manager to see the last scheduled runs, then devfleet fleet-report for the failing checks. Re-enable it with devfleet fleet-sync --schedule enable --apply --yes when the manager was replaced.',
+    },
+  };
+}

package/src/secrets.ts

@@ -12,7 +12,20 @@ export const SECRET_NAMES = [
   'monitoring-tunnel-token',
   'proxy-tunnel-token',
 ] as const;
-export type SecretName = (typeof SECRET_NAMES)[number];
+/** Secrets only a feature the fleet declares needs: a Vault location may
+ * leave them out, and they are then unavailable.
+ */
+export const OPTIONAL_SECRET_NAMES = [
+  // The off-machine storage of the monitoring hub's backups
+  // (services.monitoring.backups).
+  'monitoring-backup-access-key',
+  'monitoring-backup-secret-key',
+  // Where fleet-watch notifies (notify.targets).
+  'notify-webhook-url',
+  'notify-ntfy-url',
+] as const;
+export type SecretName =
+  (typeof SECRET_NAMES)[number] | (typeof OPTIONAL_SECRET_NAMES)[number];
 
 /** A secret's location in a Vault KV v2 mount. */
 export type VaultLocation = { path: string; field: string };
@@ -24,8 +37,13 @@ export type SecretsConfig =
       provider: 'vault';
       address: string;
       mount: string;
-      locations: Record<SecretName, VaultLocation>;
+      locations: VaultLocations;
     };
+export type VaultLocations = Record<
+  (typeof SECRET_NAMES)[number],
+  VaultLocation
+> &
+  Partial<Record<(typeof OPTIONAL_SECRET_NAMES)[number], VaultLocation>>;
 
 /** Runs a command and returns its stdout, or null when it fails. */
 export type SecretCommand = (
@@ -79,6 +97,7 @@ export function secretsProvider(
       );
   return async (name) => {
     const location = config.locations[name];
+    if (!location) return null;
     return present(
       await run([
         'vault',
@@ -142,13 +161,12 @@ const PARSERS: Record<string, (value: Fields) => SecretsConfig | null> = {
   },
 };
 
-function vaultLocations(
-  value: unknown
-): Record<SecretName, VaultLocation> | null {
+function vaultLocations(value: unknown): VaultLocations | null {
   if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
   const fields = value as Fields;
-  if (!only(fields, [...SECRET_NAMES])) return null;
-  const valid = SECRET_NAMES.every((name) => {
+  if (!only(fields, [...SECRET_NAMES, ...OPTIONAL_SECRET_NAMES])) return null;
+  const optional = OPTIONAL_SECRET_NAMES.filter((name) => name in fields);
+  const valid = [...SECRET_NAMES, ...optional].every((name) => {
     const location = fields[name] as Fields | undefined;
     return (
       location !== null &&
@@ -158,7 +176,7 @@ function vaultLocations(
       text(location['field'])
     );
   });
-  return valid ? (fields as Record<SecretName, VaultLocation>) : null;
+  return valid ? (fields as VaultLocations) : null;
 }
 
 /** The `secrets` block of a fleet configuration, or null when it is invalid. */

package/src/ssh.ts

@@ -20,9 +20,9 @@ export function sshArgs(node: FleetNode): string[] {
     '-o',
     'ConnectionAttempts=1',
     '-o',
-    'ServerAliveInterval=5',
+    'ServerAliveInterval=15',
     '-o',
-    'ServerAliveCountMax=2',
+    'ServerAliveCountMax=4',
     '-o',
     'ClearAllForwardings=yes',
     '-o',
@@ -66,6 +66,14 @@ const FAILURES = [
     action:
       'Use ssh-add -l and ssh -vvv ALIAS locally; check User, IdentityFile, IdentitiesOnly, agent/keychain access and authorized_keys. Do not share debug logs without review.',
   },
+  {
+    pattern:
+      /Connection (?:to \S+ )?closed|Connection reset|Broken pipe|Timeout, server \S+ not responding|client_loop: send disconnect/i,
+    id: 'ssh:disconnected',
+    summary: 'The SSH session was lost while the operation ran.',
+    action:
+      'The worker keeps running what it started. Rerun fleet-sync: it follows the operation of the same revision to its end. Check the network between the manager and the worker if it repeats.',
+  },
   {
     pattern: /Connection refused|No route to host|Network is unreachable/i,
     id: 'ssh:network',

package/src/state/history.ts

@@ -0,0 +1,172 @@
+import { record } from '../config';
+import { databasePath, stateRoot } from './paths';
+import { readDocument, writeDocument } from './store';
+
+/** Who started a fleet-sync run: a person (or an agent) at the manager, or
+ * the manager's own schedule.
+ */
+export type Trigger = 'operator' | 'schedule';
+
+/** One fleet-sync run, as the history keeps it: codes and node names only,
+ * never a remedy, a path or anything a subprocess printed.
+ */
+export interface SyncRun {
+  startedAt: string;
+  finishedAt: string;
+  durationMs: number;
+  trigger: Trigger;
+  revision: string | null;
+  ok: boolean;
+  exitCode: number;
+  /** The IDs of the checks that failed, once each. */
+  failing: string[];
+  /** The machines that were not ready when the run ended. */
+  notReady: string[];
+}
+
+/** The schedule the manager was last told to keep. */
+export interface ScheduleRecord {
+  enabled: boolean;
+  intervalSeconds: number;
+  changedAt: string;
+}
+
+export interface History {
+  version: 1;
+  schedule?: ScheduleRecord;
+  runs: SyncRun[];
+}
+
+/** The history is capped twice: by age and by count, so the document stays
+ * far below the fleet state's limit on one document.
+ */
+export const HISTORY_DAYS = 90;
+export const HISTORY_RUNS = 500;
+
+const DAY_MS = 86_400_000;
+
+function strings(value: unknown): value is string[] {
+  return Array.isArray(value) && value.every((v) => typeof v === 'string');
+}
+
+function syncRun(value: unknown): value is SyncRun {
+  return (
+    record(value) &&
+    typeof value['startedAt'] === 'string' &&
+    typeof value['finishedAt'] === 'string' &&
+    typeof value['durationMs'] === 'number' &&
+    (value['trigger'] === 'operator' || value['trigger'] === 'schedule') &&
+    (value['revision'] === null || typeof value['revision'] === 'string') &&
+    typeof value['ok'] === 'boolean' &&
+    typeof value['exitCode'] === 'number' &&
+    strings(value['failing']) &&
+    strings(value['notReady'])
+  );
+}
+
+function scheduleRecord(value: unknown): value is ScheduleRecord {
+  return (
+    record(value) &&
+    typeof value['enabled'] === 'boolean' &&
+    typeof value['intervalSeconds'] === 'number' &&
+    typeof value['changedAt'] === 'string'
+  );
+}
+
+/** A document that cannot be read is an empty history: it only ever
+ * describes the past, and the next run starts it again.
+ */
+export function parseHistory(text: string | null): History {
+  let value: unknown = null;
+  try {
+    value = text === null ? null : JSON.parse(text);
+  } catch {
+    value = null;
+  }
+  if (!record(value) || value['version'] !== 1) return { version: 1, runs: [] };
+  return {
+    version: 1,
+    ...(scheduleRecord(value['schedule'])
+      ? { schedule: value['schedule'] }
+      : {}),
+    runs: Array.isArray(value['runs']) ? value['runs'].filter(syncRun) : [],
+  };
+}
+
+export function readHistory(env = process.env): History {
+  return parseHistory(readDocument(databasePath(stateRoot(env)), 'history'));
+}
+
+function writeHistory(history: History, env: NodeJS.ProcessEnv): void {
+  writeDocument(
+    databasePath(stateRoot(env)),
+    'history',
+    JSON.stringify(history) + '\n'
+  );
+}
+
+/** Keeps the runs of the last HISTORY_DAYS days, and at most HISTORY_RUNS. */
+export function capped(runs: SyncRun[], now: Date): SyncRun[] {
+  const oldest = now.getTime() - HISTORY_DAYS * DAY_MS;
+  return runs
+    .filter((run) => Date.parse(run.finishedAt) >= oldest)
+    .slice(-HISTORY_RUNS);
+}
+
+export function appendRun(
+  run: SyncRun,
+  env = process.env,
+  now = new Date()
+): History {
+  const history = readHistory(env);
+  const next = { ...history, runs: capped([...history.runs, run], now) };
+  writeHistory(next, env);
+  return next;
+}
+
+export function recordSchedule(
+  schedule: ScheduleRecord,
+  env = process.env
+): void {
+  writeHistory({ ...readHistory(env), schedule }, env);
+}
+
+/** What a fleet-sync report says about one run. */
+export function summarizeRun(
+  data: unknown,
+  timing: { startedAt: Date; finishedAt: Date },
+  trigger: Trigger,
+  exitCode: number
+): SyncRun {
+  const report = record(data) ? data : {};
+  const checks = (value: unknown): Record<string, unknown>[] =>
+    Array.isArray(value) ? value.filter(record) : [];
+  const nodes = checks(report['nodes']);
+  const failing = new Set<string>();
+  for (const check of [
+    ...checks(report['checks']),
+    ...nodes.flatMap((node) => checks(node['checks'])),
+  ])
+    if (check['status'] === 'fail' && typeof check['id'] === 'string')
+      failing.add(check['id']);
+  const notReady = nodes
+    .filter((node) =>
+      checks(node['checks']).some((check) => check['status'] === 'fail')
+    )
+    .map((node) => node['node'])
+    .filter((node): node is string => typeof node === 'string');
+  return {
+    startedAt: timing.startedAt.toISOString(),
+    finishedAt: timing.finishedAt.toISOString(),
+    durationMs: timing.finishedAt.getTime() - timing.startedAt.getTime(),
+    trigger,
+    revision:
+      typeof report['desiredRevision'] === 'string'
+        ? report['desiredRevision']
+        : null,
+    ok: exitCode === 0 && report['ok'] !== false,
+    exitCode,
+    failing: [...failing].sort(),
+    notReady: [...new Set(notReady)].sort(),
+  };
+}

package/src/state/store.ts

@@ -9,7 +9,14 @@ import { FleetUsageError } from '../errors';
  * connection to each worker, `enrollment` the approved and applied revision.
  */
 export const DOCUMENTS = ['definition', 'roster', 'enrollment'] as const;
+/** Kept when there is something to keep; a fleet without them is complete:
+ * `history` is the record of recent fleet-sync runs and of the schedule.
+ */
+export const OPTIONAL_DOCUMENTS = ['history'] as const;
 export type DocumentName = (typeof DOCUMENTS)[number];
+/** Every document the fleet state may hold. */
+export type StoredName = DocumentName | (typeof OPTIONAL_DOCUMENTS)[number];
+const KNOWN: readonly string[] = [...DOCUMENTS, ...OPTIONAL_DOCUMENTS];
 
 /** One version of the whole database. Every change replaces it, so two copies
  * hold the same state exactly when their heads are equal.
@@ -21,7 +28,7 @@ export interface Head {
 }
 
 export interface DocumentInfo {
-  name: DocumentName;
+  name: StoredName;
   bytes: number;
   updatedAt: string;
 }
@@ -100,7 +107,7 @@ function tables(db: Database): boolean {
   );
 }
 
-export function readDocument(file: string, name: DocumentName): string | null {
+export function readDocument(file: string, name: StoredName): string | null {
   return reading(file, (db) => {
     if (!tables(db)) return null;
     const row = db
@@ -119,7 +126,7 @@ export function listDocuments(file: string): DocumentInfo[] {
           'SELECT name, length(CAST(body AS BLOB)) AS bytes, updated_at AS updatedAt FROM documents ORDER BY name'
         )
         .all() as DocumentInfo[];
-      return rows.filter((row) => DOCUMENTS.includes(row.name));
+      return rows.filter((row) => KNOWN.includes(row.name));
     }) ?? []
   );
 }
@@ -138,7 +145,7 @@ export function readHead(file: string): Head | null {
  */
 export function writeDocument(
   file: string,
-  name: DocumentName,
+  name: StoredName,
   body: string
 ): Head {
   if (Buffer.byteLength(body) > MAX_DOCUMENT_BYTES)

package/src/tailscale.ts

@@ -10,6 +10,9 @@ export interface TailnetComputer {
   ips: string[];
   online: boolean;
   self: boolean;
+  /** When the node key expires, in milliseconds since the epoch; absent when
+   * it never does (tagged nodes, or expiry disabled in the admin console). */
+  keyExpiry?: number;
 }
 export interface Tailnet {
   computers: TailnetComputer[];
@@ -103,6 +106,7 @@ function parseComputer(value: unknown, self: boolean): TailnetComputer {
     OS: os,
     TailscaleIPs: ips,
     Online: online,
+    KeyExpiry: keyExpiry,
   } = value;
   if (
     ![id, name, os].every(
@@ -129,6 +133,53 @@ function parseComputer(value: unknown, self: boolean): TailnetComputer {
     ips: ips as string[],
     online,
     self,
+    ...expiry(keyExpiry),
+  };
+}
+
+function expiry(value: unknown): { keyExpiry?: number } {
+  if (typeof value !== 'string') return {};
+  const time = Date.parse(value);
+  // Tailscale writes the zero time for a key that never expires.
+  return Number.isFinite(time) && time > 0 ? { keyExpiry: time } : {};
+}
+
+const DAY_MS = 86_400_000;
+const KEY_EXPIRY = { warnDays: 14, failDays: 3 };
+
+/** A node drops off the tailnet on the day its key expires. The check warns
+ * two weeks ahead and fails three days ahead, like a T3 link that expires.
+ */
+export function keyExpiryCheck(
+  computer: TailnetComputer | undefined,
+  now = Date.now()
+): Check {
+  const id = 'tailscale:key-expiry';
+  if (!computer)
+    return { id, status: 'skip', summary: 'TAILSCALE_KEY_EXPIRY_UNOBSERVED' };
+  if (computer.keyExpiry === undefined)
+    return {
+      id,
+      status: 'pass',
+      summary: 'The Tailscale node key of this machine does not expire.',
+    };
+  const days = Math.floor((computer.keyExpiry - now) / DAY_MS);
+  const remedy = {
+    action: computer.self
+      ? 'Renew the node key of the manager before it expires: tailscale up --force-reauth on the manager, or disable key expiry for it in the Tailscale admin console.'
+      : 'Disable key expiry for the worker in the Tailscale admin console, or tag it (tagged nodes do not expire). After an expiry, re-join it as docs/operations.md describes (Tailscale key expiry).',
+  };
+  if (days < 0)
+    return { id, status: 'fail', summary: 'TAILSCALE_KEY_EXPIRED', remedy };
+  const summary = `TAILSCALE_KEY_EXPIRES_SOON in ${days} days`;
+  if (days <= KEY_EXPIRY.failDays)
+    return { id, status: 'fail', summary, remedy };
+  if (days <= KEY_EXPIRY.warnDays)
+    return { id, status: 'warn', summary, remedy };
+  return {
+    id,
+    status: 'pass',
+    summary: `The Tailscale node key expires in ${days} days.`,
   };
 }
 

package/src/toolchain-checks.ts

@@ -1,11 +1,16 @@
 import { join } from 'node:path';
+import {
+  selectionArguments,
+  type ToolSelection,
+} from '../bootstrap/worker-tools';
 import { record } from './config';
 import type { Check, Runner } from './types';
 
 export async function toolchainChecks(
   root: string,
   run: Runner,
-  worker?: boolean
+  /** A worker's own tool selection; absent on the manager. */
+  worker?: ToolSelection
 ): Promise<Check[]> {
   const result = await run(doctorArguments(root, worker), {
     cwd: root,
@@ -97,12 +102,12 @@ function toolCheck(value: unknown): Check | null {
   };
 }
 
-function doctorArguments(root: string, worker?: boolean): string[] {
+function doctorArguments(root: string, worker?: ToolSelection): string[] {
   return [
     'bun',
     join(root, 'bootstrap/doctor.ts'),
     '--json',
     '--docker',
-    ...(worker ? ['--worker'] : []),
+    ...(worker ? ['--worker', ...selectionArguments(worker)] : []),
   ];
 }