@crvouga/dev-fleet 0.3.0 → 0.5.0

package/src/state/history.ts

@@ -0,0 +1,172 @@
+import { record } from '../config';
+import { databasePath, stateRoot } from './paths';
+import { readDocument, writeDocument } from './store';
+
+/** Who started a fleet-sync run: a person (or an agent) at the manager, or
+ * the manager's own schedule.
+ */
+export type Trigger = 'operator' | 'schedule';
+
+/** One fleet-sync run, as the history keeps it: codes and node names only,
+ * never a remedy, a path or anything a subprocess printed.
+ */
+export interface SyncRun {
+  startedAt: string;
+  finishedAt: string;
+  durationMs: number;
+  trigger: Trigger;
+  revision: string | null;
+  ok: boolean;
+  exitCode: number;
+  /** The IDs of the checks that failed, once each. */
+  failing: string[];
+  /** The machines that were not ready when the run ended. */
+  notReady: string[];
+}
+
+/** The schedule the manager was last told to keep. */
+export interface ScheduleRecord {
+  enabled: boolean;
+  intervalSeconds: number;
+  changedAt: string;
+}
+
+export interface History {
+  version: 1;
+  schedule?: ScheduleRecord;
+  runs: SyncRun[];
+}
+
+/** The history is capped twice: by age and by count, so the document stays
+ * far below the fleet state's limit on one document.
+ */
+export const HISTORY_DAYS = 90;
+export const HISTORY_RUNS = 500;
+
+const DAY_MS = 86_400_000;
+
+function strings(value: unknown): value is string[] {
+  return Array.isArray(value) && value.every((v) => typeof v === 'string');
+}
+
+function syncRun(value: unknown): value is SyncRun {
+  return (
+    record(value) &&
+    typeof value['startedAt'] === 'string' &&
+    typeof value['finishedAt'] === 'string' &&
+    typeof value['durationMs'] === 'number' &&
+    (value['trigger'] === 'operator' || value['trigger'] === 'schedule') &&
+    (value['revision'] === null || typeof value['revision'] === 'string') &&
+    typeof value['ok'] === 'boolean' &&
+    typeof value['exitCode'] === 'number' &&
+    strings(value['failing']) &&
+    strings(value['notReady'])
+  );
+}
+
+function scheduleRecord(value: unknown): value is ScheduleRecord {
+  return (
+    record(value) &&
+    typeof value['enabled'] === 'boolean' &&
+    typeof value['intervalSeconds'] === 'number' &&
+    typeof value['changedAt'] === 'string'
+  );
+}
+
+/** A document that cannot be read is an empty history: it only ever
+ * describes the past, and the next run starts it again.
+ */
+export function parseHistory(text: string | null): History {
+  let value: unknown = null;
+  try {
+    value = text === null ? null : JSON.parse(text);
+  } catch {
+    value = null;
+  }
+  if (!record(value) || value['version'] !== 1) return { version: 1, runs: [] };
+  return {
+    version: 1,
+    ...(scheduleRecord(value['schedule'])
+      ? { schedule: value['schedule'] }
+      : {}),
+    runs: Array.isArray(value['runs']) ? value['runs'].filter(syncRun) : [],
+  };
+}
+
+export function readHistory(env = process.env): History {
+  return parseHistory(readDocument(databasePath(stateRoot(env)), 'history'));
+}
+
+function writeHistory(history: History, env: NodeJS.ProcessEnv): void {
+  writeDocument(
+    databasePath(stateRoot(env)),
+    'history',
+    JSON.stringify(history) + '\n'
+  );
+}
+
+/** Keeps the runs of the last HISTORY_DAYS days, and at most HISTORY_RUNS. */
+export function capped(runs: SyncRun[], now: Date): SyncRun[] {
+  const oldest = now.getTime() - HISTORY_DAYS * DAY_MS;
+  return runs
+    .filter((run) => Date.parse(run.finishedAt) >= oldest)
+    .slice(-HISTORY_RUNS);
+}
+
+export function appendRun(
+  run: SyncRun,
+  env = process.env,
+  now = new Date()
+): History {
+  const history = readHistory(env);
+  const next = { ...history, runs: capped([...history.runs, run], now) };
+  writeHistory(next, env);
+  return next;
+}
+
+export function recordSchedule(
+  schedule: ScheduleRecord,
+  env = process.env
+): void {
+  writeHistory({ ...readHistory(env), schedule }, env);
+}
+
+/** What a fleet-sync report says about one run. */
+export function summarizeRun(
+  data: unknown,
+  timing: { startedAt: Date; finishedAt: Date },
+  trigger: Trigger,
+  exitCode: number
+): SyncRun {
+  const report = record(data) ? data : {};
+  const checks = (value: unknown): Record<string, unknown>[] =>
+    Array.isArray(value) ? value.filter(record) : [];
+  const nodes = checks(report['nodes']);
+  const failing = new Set<string>();
+  for (const check of [
+    ...checks(report['checks']),
+    ...nodes.flatMap((node) => checks(node['checks'])),
+  ])
+    if (check['status'] === 'fail' && typeof check['id'] === 'string')
+      failing.add(check['id']);
+  const notReady = nodes
+    .filter((node) =>
+      checks(node['checks']).some((check) => check['status'] === 'fail')
+    )
+    .map((node) => node['node'])
+    .filter((node): node is string => typeof node === 'string');
+  return {
+    startedAt: timing.startedAt.toISOString(),
+    finishedAt: timing.finishedAt.toISOString(),
+    durationMs: timing.finishedAt.getTime() - timing.startedAt.getTime(),
+    trigger,
+    revision:
+      typeof report['desiredRevision'] === 'string'
+        ? report['desiredRevision']
+        : null,
+    ok: exitCode === 0 && report['ok'] !== false,
+    exitCode,
+    failing: [...failing].sort(),
+    notReady: [...new Set(notReady)].sort(),
+  };
+}

package/src/state/store.ts

@@ -9,7 +9,14 @@ import { FleetUsageError } from '../errors';
  * connection to each worker, `enrollment` the approved and applied revision.
  */
 export const DOCUMENTS = ['definition', 'roster', 'enrollment'] as const;
+/** Kept when there is something to keep; a fleet without them is complete:
+ * `history` is the record of recent fleet-sync runs and of the schedule.
+ */
+export const OPTIONAL_DOCUMENTS = ['history'] as const;
 export type DocumentName = (typeof DOCUMENTS)[number];
+/** Every document the fleet state may hold. */
+export type StoredName = DocumentName | (typeof OPTIONAL_DOCUMENTS)[number];
+const KNOWN: readonly string[] = [...DOCUMENTS, ...OPTIONAL_DOCUMENTS];
 
 /** One version of the whole database. Every change replaces it, so two copies
  * hold the same state exactly when their heads are equal.
@@ -21,7 +28,7 @@ export interface Head {
 }
 
 export interface DocumentInfo {
-  name: DocumentName;
+  name: StoredName;
   bytes: number;
   updatedAt: string;
 }
@@ -100,7 +107,7 @@ function tables(db: Database): boolean {
   );
 }
 
-export function readDocument(file: string, name: DocumentName): string | null {
+export function readDocument(file: string, name: StoredName): string | null {
   return reading(file, (db) => {
     if (!tables(db)) return null;
     const row = db
@@ -119,7 +126,7 @@ export function listDocuments(file: string): DocumentInfo[] {
           'SELECT name, length(CAST(body AS BLOB)) AS bytes, updated_at AS updatedAt FROM documents ORDER BY name'
         )
         .all() as DocumentInfo[];
-      return rows.filter((row) => DOCUMENTS.includes(row.name));
+      return rows.filter((row) => KNOWN.includes(row.name));
     }) ?? []
   );
 }
@@ -138,7 +145,7 @@ export function readHead(file: string): Head | null {
  */
 export function writeDocument(
   file: string,
-  name: DocumentName,
+  name: StoredName,
   body: string
 ): Head {
   if (Buffer.byteLength(body) > MAX_DOCUMENT_BYTES)

package/src/tailscale.ts

@@ -10,6 +10,9 @@ export interface TailnetComputer {
   ips: string[];
   online: boolean;
   self: boolean;
+  /** When the node key expires, in milliseconds since the epoch; absent when
+   * it never does (tagged nodes, or expiry disabled in the admin console). */
+  keyExpiry?: number;
 }
 export interface Tailnet {
   computers: TailnetComputer[];
@@ -103,6 +106,7 @@ function parseComputer(value: unknown, self: boolean): TailnetComputer {
     OS: os,
     TailscaleIPs: ips,
     Online: online,
+    KeyExpiry: keyExpiry,
   } = value;
   if (
     ![id, name, os].every(
@@ -129,6 +133,53 @@ function parseComputer(value: unknown, self: boolean): TailnetComputer {
     ips: ips as string[],
     online,
     self,
+    ...expiry(keyExpiry),
+  };
+}
+
+function expiry(value: unknown): { keyExpiry?: number } {
+  if (typeof value !== 'string') return {};
+  const time = Date.parse(value);
+  // Tailscale writes the zero time for a key that never expires.
+  return Number.isFinite(time) && time > 0 ? { keyExpiry: time } : {};
+}
+
+const DAY_MS = 86_400_000;
+const KEY_EXPIRY = { warnDays: 14, failDays: 3 };
+
+/** A node drops off the tailnet on the day its key expires. The check warns
+ * two weeks ahead and fails three days ahead, like a T3 link that expires.
+ */
+export function keyExpiryCheck(
+  computer: TailnetComputer | undefined,
+  now = Date.now()
+): Check {
+  const id = 'tailscale:key-expiry';
+  if (!computer)
+    return { id, status: 'skip', summary: 'TAILSCALE_KEY_EXPIRY_UNOBSERVED' };
+  if (computer.keyExpiry === undefined)
+    return {
+      id,
+      status: 'pass',
+      summary: 'The Tailscale node key of this machine does not expire.',
+    };
+  const days = Math.floor((computer.keyExpiry - now) / DAY_MS);
+  const remedy = {
+    action: computer.self
+      ? 'Renew the node key of the manager before it expires: tailscale up --force-reauth on the manager, or disable key expiry for it in the Tailscale admin console.'
+      : 'Disable key expiry for the worker in the Tailscale admin console, or tag it (tagged nodes do not expire). After an expiry, re-join it as docs/operations.md describes (Tailscale key expiry).',
+  };
+  if (days < 0)
+    return { id, status: 'fail', summary: 'TAILSCALE_KEY_EXPIRED', remedy };
+  const summary = `TAILSCALE_KEY_EXPIRES_SOON in ${days} days`;
+  if (days <= KEY_EXPIRY.failDays)
+    return { id, status: 'fail', summary, remedy };
+  if (days <= KEY_EXPIRY.warnDays)
+    return { id, status: 'warn', summary, remedy };
+  return {
+    id,
+    status: 'pass',
+    summary: `The Tailscale node key expires in ${days} days.`,
   };
 }
 

package/src/toolchain-checks.ts

@@ -1,11 +1,16 @@
 import { join } from 'node:path';
+import {
+  selectionArguments,
+  type ToolSelection,
+} from '../bootstrap/worker-tools';
 import { record } from './config';
 import type { Check, Runner } from './types';
 
 export async function toolchainChecks(
   root: string,
   run: Runner,
-  worker?: boolean
+  /** A worker's own tool selection; absent on the manager. */
+  worker?: ToolSelection
 ): Promise<Check[]> {
   const result = await run(doctorArguments(root, worker), {
     cwd: root,
@@ -97,12 +102,12 @@ function toolCheck(value: unknown): Check | null {
   };
 }
 
-function doctorArguments(root: string, worker?: boolean): string[] {
+function doctorArguments(root: string, worker?: ToolSelection): string[] {
   return [
     'bun',
     join(root, 'bootstrap/doctor.ts'),
     '--json',
     '--docker',
-    ...(worker ? ['--worker'] : []),
+    ...(worker ? ['--worker', ...selectionArguments(worker)] : []),
   ];
 }

package/src/worker-capability-checks.ts

@@ -0,0 +1,167 @@
+import policy from '../ansible/ubuntu/policy.json';
+import type { Check, CommandResult, Runner } from './types';
+
+/** What an agent does on a worker, tried the way it does it: start a
+ * container, build an image, launch a headless browser. Each check runs as
+ * the worker's user, in its session, and reports a code and a remedy, never
+ * what a command printed.
+ */
+
+function failure(id: string, summary: string, action: string): Check {
+  return { id, status: 'fail', summary, remedy: { action } };
+}
+
+/** Docker's own words for the causes an agent meets most. */
+function dockerFailure(result: CommandResult): {
+  code: string;
+  action: string;
+} {
+  const said = `${result.stdout}\n${result.stderr}`;
+  if (result.reason === 'timeout')
+    return {
+      code: 'DOCKER_RUN_TIMEOUT',
+      action:
+        'A container did not start and exit in time. Inspect the engine (systemctl status docker) and its disk; rerun fleet-sync.',
+    };
+  if (/permission denied[^\n]*docker\.sock/i.test(said))
+    return {
+      code: 'DOCKER_SOCKET_ACCESS_DENIED',
+      action:
+        "This session cannot use the Docker socket. fleet-sync adds the worker's user to the docker group; a session started before that keeps its old groups. Rerun fleet-sync, then restart the session (log out and in, or restart the T3 service).",
+    };
+  if (/cannot connect to the docker daemon/i.test(said))
+    return {
+      code: 'DOCKER_DAEMON_UNREACHABLE',
+      action:
+        'The Docker engine is not running. Rerun fleet-sync, which starts and enables docker.service.',
+    };
+  if (/no such image|pull access denied|not found/i.test(said))
+    return {
+      code: 'DOCKER_SMOKE_IMAGE_MISSING',
+      action:
+        'The pinned smoke-test image is not cached on this worker. Rerun fleet-sync, which pulls it once so this check works offline.',
+    };
+  if (/no space left on device/i.test(said))
+    return {
+      code: 'DOCKER_DISK_FULL',
+      action:
+        'The image store is full. Free space under /var/lib/docker (the disk GC prunes old images) and rerun fleet-sync.',
+    };
+  return {
+    code: 'DOCKER_RUN_FAILED',
+    action:
+      'Docker answers but cannot run a container here. Inspect cgroups and the engine log (journalctl -u docker), then rerun fleet-sync.',
+  };
+}
+
+/** Runs the pinned image offline, then builds a one-line image from it. */
+export async function dockerRunCheck(run: Runner): Promise<Check> {
+  const id = 'worker:docker-run';
+  const { image, timeoutSeconds } = policy.dockerSmokeTest;
+  const timeoutMs = timeoutSeconds * 1000;
+  const started = await run(
+    [
+      'docker',
+      'run',
+      '--rm',
+      '--network',
+      'none',
+      '--pull',
+      'never',
+      image,
+      'true',
+    ],
+    { timeoutMs }
+  );
+  if (!started.ok) {
+    const { code, action } = dockerFailure(started);
+    return failure(id, code, action);
+  }
+  const built = await run(
+    ['docker', 'build', '--quiet', '--pull=false', '--network', 'none', '-'],
+    { input: `FROM ${image}\nRUN true\n`, timeoutMs }
+  );
+  if (!built.ok) {
+    const { code, action } = dockerFailure(built);
+    return failure(
+      id,
+      code === 'DOCKER_RUN_FAILED' ? 'DOCKER_BUILD_FAILED' : code,
+      code === 'DOCKER_RUN_FAILED'
+        ? 'A container runs but an image does not build. Inspect Buildx (docker buildx ls) and the image store, then rerun fleet-sync.'
+        : action
+    );
+  }
+  return {
+    id,
+    status: 'pass',
+    summary:
+      'A container of the pinned image ran and a one-line image built, offline, in this session.',
+  };
+}
+
+/** Where `ansible/ubuntu/tasks/browser.yml` unpacks the pinned build. */
+export function headlessShell(arch: string = process.arch): string {
+  const { version, amd64, arm64 } = policy.browser.headlessShell;
+  const build = arch === 'arm64' ? arm64 : amd64;
+  const directory = build.archive.split('/').pop()!;
+  return `/opt/personal-fleet/browser/${version}/${directory}/chrome-headless-shell`;
+}
+
+/** A worker with the web profile launches the pinned headless Chromium,
+ * loads a blank page and exits, as a browser test would.
+ */
+export async function browserCheck(
+  run: Runner,
+  web: boolean,
+  arch: string = process.arch
+): Promise<Check> {
+  const id = 'worker:browser';
+  if (!web)
+    return {
+      id,
+      status: 'skip',
+      summary:
+        'This worker does not select the web tool profile, so no browser is installed for tests.',
+    };
+  const launched = await run(
+    [
+      headlessShell(arch),
+      '--headless',
+      '--no-sandbox',
+      '--disable-gpu',
+      '--dump-dom',
+      'about:blank',
+    ],
+    { timeoutMs: policy.browser.launchTimeoutSeconds * 1000 }
+  );
+  if (launched.ok && /<html/i.test(launched.stdout))
+    return {
+      id,
+      status: 'pass',
+      summary: `Headless Chromium ${policy.browser.headlessShell.version} launched, loaded a blank page and exited.`,
+    };
+  const said = `${launched.stdout}\n${launched.stderr}`;
+  if (launched.reason === 'missing')
+    return failure(
+      id,
+      'BROWSER_NOT_INSTALLED',
+      'The pinned headless Chromium is not installed. Rerun fleet-sync, which installs it on workers with the web profile.'
+    );
+  if (/error while loading shared libraries/i.test(said))
+    return failure(
+      id,
+      'BROWSER_LIBRARY_MISSING',
+      'A system library headless Chromium loads is missing. Rerun fleet-sync, which installs the pinned libraries of the web profile.'
+    );
+  if (launched.reason === 'timeout')
+    return failure(
+      id,
+      'BROWSER_LAUNCH_TIMEOUT',
+      'Headless Chromium did not exit in time. Check the worker for memory pressure, then rerun fleet-sync.'
+    );
+  return failure(
+    id,
+    'BROWSER_LAUNCH_FAILED',
+    'Headless Chromium did not load a blank page. Run it by hand with --dump-dom about:blank to see why, then rerun fleet-sync.'
+  );
+}

package/src/worker-capacity.ts

@@ -1,11 +1,14 @@
 import { join } from 'node:path';
+import { clockCheck } from './clock';
 import { healthChecks, parseProbe } from './machine-health';
 import type { Check, Runner } from './types';
 
 export async function workerCapacity(
   root: string,
   node: string,
-  run: Runner
+  run: Runner,
+  /** The manager's clock in seconds since the epoch, when it sent one. */
+  reference?: number
 ): Promise<Check[]> {
   const result = await run(['sh', join(root, 'ansible/net/remote-probe.sh')], {
     timeoutMs: 10_000,
@@ -25,10 +28,19 @@ export async function workerCapacity(
         summary: 'Required CPU, memory and disk evidence is unavailable.',
       },
     ];
-  return healthChecks(sample, node).map((check) => ({
-    id: `worker:capacity:${check.id}`,
-    status: check.severity,
-    summary: check.detail,
-    ...(check.fix ? { remedy: { action: check.fix } } : {}),
-  }));
+  return [
+    ...healthChecks(sample, node).map((check) => ({
+      id: `worker:capacity:${check.id}`,
+      status: check.severity,
+      summary: check.detail,
+      ...(check.fix ? { remedy: { action: check.fix } } : {}),
+    })),
+    clockCheck(
+      'worker:clock',
+      sample,
+      reference === undefined || sample.epoch === undefined
+        ? undefined
+        : sample.epoch - reference
+    ),
+  ];
 }

package/src/worker-host-checks.ts

@@ -5,6 +5,8 @@ import { verifyResourceSafety } from './platform/resource-verify';
 import policy from '../ansible/ubuntu/policy.json';
 import { userInfo } from 'node:os';
 import { T3_HEALTH_SCRIPT, runningVersionCheck } from './platform/t3-service';
+import { compareDebianVersions } from './platform/debian-version';
+import { packagePins, ubuntuRelease } from './platform/ubuntu-release';
 import type { Check, Runner } from './types';
 
 function check(id: string, ok: boolean, summary: string): Check {
@@ -22,6 +24,34 @@ function check(id: string, ok: boolean, summary: string): Check {
       : {}),
   };
 }
+/** A pin is a minimum (docs/worker-contract.md): Ubuntu's security updates
+ * may move a package past it, which is reported but never undone. */
+export function pinnedPackage(
+  name: string,
+  pinned: string,
+  installed: string | undefined
+): Check {
+  const order =
+    installed === undefined ? -1 : compareDebianVersions(installed, pinned);
+  if (order <= 0)
+    return check(
+      `package:${name}`,
+      order === 0,
+      order === 0
+        ? 'Installed at the version the policy pins.'
+        : `PACKAGE_BELOW_PIN ${name} ${installed ?? 'missing'}, pinned ${pinned}`
+    );
+  return {
+    id: `worker:package:${name}`,
+    status: 'warn',
+    summary: `PACKAGE_NEWER_THAN_PIN ${name} ${installed}, pinned ${pinned}`,
+    remedy: {
+      action:
+        'An unattended security update moved this package past its pin, and it is kept. Raise the pin in the Ubuntu policy when the snapshot moves.',
+    },
+  };
+}
+
 /** Observed unit state for a failed service check, e.g.
  * `SERVICE_NOT_INSTALLED beszel-agent.service (inactive)`. */
 export function serviceState(name: string, show: string): string {
@@ -56,17 +86,13 @@ export async function workerHostChecks(
       audit.passwordlessSudo,
       'Designated worker user has noninteractive sudo.'
     ),
-    ...Object.entries(policy.packages).map(([name, version]) =>
-      check(
-        `package:${name}`,
-        audit.packages[name] === version,
-        'Exact declarative Ubuntu package version.'
-      )
+    ...Object.entries(packagePins(ubuntuRelease(audit.release))).map(
+      ([name, version]) => pinnedPackage(name, version, audit.packages[name])
     ),
-    check(
-      'package:tailscale',
-      audit.packages['tailscale'] === policy.tailscale.version,
-      'Exact declarative Tailscale version.'
+    pinnedPackage(
+      'tailscale',
+      policy.tailscale.version,
+      audit.packages['tailscale']
     ),
   ];
   for (const name of policy.alwaysOnServices) {
@@ -95,6 +121,7 @@ export async function workerHostChecks(
       )
     );
   }
+  checks.push(await rebootPendingCheck(audit.rebootPending, run));
   checks.push(...(await recoveryChecks(run)));
   checks.push(...(await verifyUbuntuBoot(user, run)));
   checks.push(...(await verifyResourceSafety(userInfo().homedir, run)));
@@ -202,3 +229,34 @@ async function t3HealthCheck(run: Runner): Promise<Check> {
     'T3 local API answers with its environment identity within 10 seconds.'
   );
 }
+
+const DAY_SECONDS = 86_400;
+
+/** Kernel and library updates apply only after a reboot. A pending reboot
+ * warns with its age, and fails once it is older than `policy.json` allows.
+ */
+export async function rebootPendingCheck(
+  pending: boolean,
+  run: Runner,
+  now = Date.now()
+): Promise<Check> {
+  const id = 'worker:reboot-pending';
+  if (!pending) return { id, status: 'pass', summary: 'No reboot is pending.' };
+  const since = await run(['stat', '-c', '%Y', '/var/run/reboot-required']);
+  const seconds = Number(since.stdout.trim());
+  const days =
+    since.ok && /^\d+$/.test(since.stdout.trim())
+      ? Math.max(0, Math.floor((now / 1000 - seconds) / DAY_SECONDS))
+      : null;
+  const age = days === null ? '' : ` for ${days} days`;
+  return {
+    id,
+    status:
+      days !== null && days > policy.rebootPendingFailDays ? 'fail' : 'warn',
+    summary: `REBOOT_PENDING${age}: installed updates apply only after a reboot.`,
+    remedy: {
+      action:
+        'Reboot the worker when no agent turn is running (sudo systemctl reboot on the worker), then confirm it returns to ready with devfleet fleet-status --workers.',
+    },
+  };
+}