@crvouga/dev-fleet 0.6.0 → 0.8.0

package/src/machine/state.ts

@@ -20,6 +20,12 @@ export interface Enrollment {
   approvedRevision: string;
   appliedRevision?: string;
   previousRevision?: string;
+  /** The revision each worker last converged to, by fleet id. A worker
+   * keeps its progress when the fleet as a whole is not ready.
+   */
+  workers?: Record<string, string>;
+  /** Tombstones: when each retired worker left the fleet, by fleet id. */
+  retired?: Record<string, string>;
 }
 
 export const isRevision = (value: unknown): value is string =>
@@ -156,6 +162,30 @@ export async function ensurePrivateRoot(root: string): Promise<void> {
   }
 }
 
+function validWorkers(value: unknown): boolean {
+  return (
+    value === undefined ||
+    (record(value) &&
+      Object.entries(value).every(
+        ([id, revision]) =>
+          /^[a-z][a-z0-9-]{0,62}$/.test(id) && isRevision(revision)
+      ))
+  );
+}
+
+function validRetired(value: unknown): boolean {
+  return (
+    value === undefined ||
+    (record(value) &&
+      Object.entries(value).every(
+        ([id, at]) =>
+          /^[a-z][a-z0-9-]{0,62}$/.test(id) &&
+          typeof at === 'string' &&
+          !Number.isNaN(Date.parse(at))
+      ))
+  );
+}
+
 /** A valid enrollment record, or null. */
 export function parseEnrollment(data: unknown): Enrollment | null {
   if (!record(data) || data['version'] !== 1) return null;
@@ -167,7 +197,9 @@ export function parseEnrollment(data: unknown): Enrollment | null {
   return isRevision(data['approvedRevision']) &&
     ['appliedRevision', 'previousRevision'].every(
       (key) => data[key] === undefined || isRevision(data[key])
-    )
+    ) &&
+    validWorkers(data['workers']) &&
+    validRetired(data['retired'])
     ? (data as unknown as Enrollment)
     : null;
 }

package/src/manager-sync.ts

@@ -37,6 +37,8 @@ export interface ManagerReport {
   ok: boolean;
   applied: boolean;
   desiredRevision: string | null;
+  /** The worker that proved a revision new to the fleet first, when one did. */
+  canary?: string;
   checks: Check[];
   nodes: ManagerNode[];
   candidates: number;
@@ -196,12 +198,10 @@ async function applyNodes(
   deploy: DeployNode
 ): Promise<void> {
   const workers = report.nodes.filter((node) => node.role !== 'manager');
-  // A revision new to the fleet is proven on one worker first. When the run
-  // selects only later workers, the first selected one proves it instead:
-  // narrowing a run never skips the canary.
   const canary = input.canary
-    ? (workers.find((node) => node.node === input.canary) ?? workers[0])
+    ? chooseCanary(workers, input.canary, input.nodes)
     : undefined;
+  if (canary) report.canary = canary.node;
   let blocked = report.checks.some(blocking);
   const step = async (result: ManagerNode) => {
     const deployable = mayDeploy(result, input.nodes);
@@ -219,6 +219,26 @@ async function applyNodes(
   await Promise.all(workers.filter((node) => node !== canary).map(step));
 }
 
+/** A revision new to the fleet is proven on one worker first: the preferred
+ * one, or when it cannot be reached (powered off, being rebuilt, offline)
+ * the next worker in declared order that can. An unreachable worker is
+ * reported as such and never blocks the others; only a canary that was
+ * deployed and failed stops promotion. When the run selects only later
+ * workers, the first selected one proves it instead: narrowing a run never
+ * skips the canary.
+ */
+function chooseCanary(
+  workers: ManagerNode[],
+  preferred: string,
+  nodes: FleetNode[]
+): ManagerNode | undefined {
+  const first = workers.find((node) => node.node === preferred);
+  const order = first
+    ? [first, ...workers.filter((node) => node !== first)]
+    : workers;
+  return order.find((node) => mayDeploy(node, nodes));
+}
+
 /** One node's rollout step; true when its remote operation ran. */
 async function rollOut(
   result: ManagerNode,

package/src/platform/manager-host.ts

@@ -0,0 +1,184 @@
+import { readFileSync } from 'node:fs';
+import { userInfo } from 'node:os';
+import { join } from 'node:path';
+import macos from '../../ansible/macos/policy.json';
+import { record } from '../config';
+import { hostId } from '../topology';
+import { ansibleCommand } from './ansible';
+import type { Runner } from '../types';
+
+/** One thing a manager needs to come back by itself after a power cut. */
+export interface BootItem {
+  item: string;
+  /** `fail` stops unattended recovery; `warn` is a trade-off to know about. */
+  status: 'pass' | 'warn' | 'fail';
+  detail: string;
+}
+
+/** Where the manager runs, behind one interface: how it is named, how it is
+ * kept awake, how its services are supervised and where they keep data. The
+ * macOS host is the default and the reference.
+ */
+export interface ManagerHost {
+  platform: 'macOS' | 'linux';
+  serviceManager: 'launchd' | 'systemd';
+  containerEngine: 'Docker Desktop' | 'Docker Engine';
+  /** The Tailscale command line. */
+  tailscale: string[];
+  /** Where the manager's services keep their data. */
+  dataDirectory(home: string): string;
+  /** The machine's local host name, as `manager.id` must name it. */
+  localHostName(run: Runner): Promise<string | null>;
+  /** The power settings that differ from the manager's policy. */
+  powerDrift(run: Runner): Promise<string[]>;
+  /** What brings the manager's services back after a power cut, unattended. */
+  bootRecovery(run: Runner, home: string): Promise<BootItem[]>;
+  /** How a person applies what bootRecovery measures. */
+  bootRemedy(workspace: string): string;
+}
+
+const item = (
+  name: string,
+  ok: boolean,
+  [pass, failure, status = 'fail']: [string, string, ('warn' | 'fail')?]
+): BootItem => ({
+  item: name,
+  status: ok ? 'pass' : status,
+  detail: ok ? pass : failure,
+});
+
+/** `pmset -g custom`'s AC Power section, as key and value. */
+async function acPower(
+  run: Runner
+): Promise<(key: string) => string | undefined> {
+  const result = await run(['pmset', '-g', 'custom']);
+  const ac = result.ok ? (result.stdout.split('AC Power:')[1] ?? '') : '';
+  return (key) => new RegExp(`^\\s*${key}\\s+(\\S+)`, 'm').exec(ac)?.[1];
+}
+
+function dockerSettings(home: string): Record<string, unknown> | null {
+  try {
+    const value: unknown = JSON.parse(
+      readFileSync(join(home, macos.dockerDesktop.settings), 'utf8')
+    );
+    return record(value) ? value : null;
+  } catch {
+    return null;
+  }
+}
+
+export const MACOS_MANAGER: ManagerHost = {
+  platform: 'macOS',
+  serviceManager: 'launchd',
+  containerEngine: 'Docker Desktop',
+  tailscale: ['tailscale'],
+  dataDirectory: (home) => join(home, 'Library/Application Support'),
+  async localHostName(run) {
+    const result = await run(['scutil', '--get', 'LocalHostName']);
+    return result.ok ? hostId(result.stdout.trim()) : null;
+  },
+  async powerDrift(run) {
+    const observed = await acPower(run);
+    return Object.entries(macos.pmset)
+      .filter(([key, value]) => observed(key) !== value)
+      .map(([key]) => key);
+  },
+  async bootRecovery(run, home) {
+    const observed = await acPower(run);
+    const docker = dockerSettings(home);
+    const vault = await run(['fdesetup', 'status']);
+    const filevault = vault.ok && /FileVault is On/.test(vault.stdout);
+    const login = await run([
+      'defaults',
+      'read',
+      '/Library/Preferences/com.apple.loginwindow',
+      'autoLoginUser',
+    ]);
+    const agent = await run([
+      'launchctl',
+      'print',
+      `gui/${userInfo().uid}/${macos.beszelDaemon}`,
+    ]);
+    return [
+      item('autorestart', observed('autorestart') === macos.pmset.autorestart, [
+        'The Mac starts again by itself when power returns.',
+        'The Mac stays off after a power cut (pmset autorestart is not 1).',
+      ]),
+      item('docker-autostart', docker?.[macos.dockerDesktop.key] === true, [
+        'Docker Desktop starts at login, and with it the proxy and the hub.',
+        'Docker Desktop does not start at login: the CLI proxy and the monitoring hub stay down after a restart.',
+      ]),
+      filevault
+        ? item('login', false, [
+            '',
+            'FileVault is on: after a restart the Mac waits at the login window until a person signs in, and no LaunchAgent or Docker Desktop starts before that. Turn FileVault off for unattended recovery, or accept that a power cut needs a person (docs/manager-recovery.md).',
+            'warn',
+          ])
+        : item('login', login.ok && login.stdout.trim().length > 0, [
+            'FileVault is off and a user signs in automatically after a restart.',
+            'No user signs in automatically after a restart, so no LaunchAgent starts. Turn on automatic login for the manager’s user in System Settings (it asks for that user’s password).',
+            'warn',
+          ]),
+      item('launch-agents', agent.ok, [
+        'The fleet’s LaunchAgents are loaded.',
+        `The LaunchAgent ${macos.beszelDaemon} is not loaded.`,
+      ]),
+    ];
+  },
+  bootRemedy: (workspace) =>
+    `Apply the manager policy, as an administrator: cd ${workspace} && ${ansibleCommand('darwin').join(' ')} -K ansible/macos/manager.yml (autorestart, Docker Desktop at login), then fleet-sync (LaunchAgents). FileVault and automatic login are a person’s choice: docs/manager-recovery.md.`,
+};
+
+/** The sleep states a Linux manager must never enter. */
+const SLEEP_TARGETS = [
+  'sleep.target',
+  'suspend.target',
+  'hibernate.target',
+  'hybrid-sleep.target',
+];
+
+export const LINUX_MANAGER: ManagerHost = {
+  platform: 'linux',
+  serviceManager: 'systemd',
+  containerEngine: 'Docker Engine',
+  tailscale: ['tailscale'],
+  dataDirectory: (home) => join(home, '.local/share'),
+  async localHostName(run) {
+    const result = await run(['hostnamectl', '--static']);
+    return result.ok ? hostId(result.stdout.trim()) : null;
+  },
+  async powerDrift(run) {
+    const drifted: string[] = [];
+    for (const target of SLEEP_TARGETS) {
+      const result = await run(['systemctl', 'is-enabled', target]);
+      if (result.stdout.trim() !== 'masked') drifted.push(target);
+    }
+    return drifted;
+  },
+  async bootRecovery(run) {
+    const user = userInfo().username;
+    const linger = await run([
+      'loginctl',
+      'show-user',
+      user,
+      '--property=Linger',
+    ]);
+    const docker = await run(['systemctl', 'is-enabled', 'docker.service']);
+    return [
+      item('linger', linger.stdout.trim() === 'Linger=yes', [
+        'The manager’s user services start at boot, without a login.',
+        'The manager’s user services wait for a login (loginctl linger is off).',
+      ]),
+      item('docker-autostart', docker.stdout.trim() === 'enabled', [
+        'Docker Engine starts at boot, and with it the proxy and the hub.',
+        'Docker Engine does not start at boot: the CLI proxy and the monitoring hub stay down after a restart.',
+      ]),
+    ];
+  },
+  bootRemedy: () =>
+    'Enable lingering for the manager’s user and the docker service with the Linux manager policy; a Linux manager role is not shipped yet.',
+};
+
+export function managerHost(platform: 'macOS' | 'linux'): ManagerHost {
+  return platform === 'linux' ? LINUX_MANAGER : MACOS_MANAGER;
+}

package/src/repair-catalog.ts

@@ -20,6 +20,29 @@ const login = (
   action,
   commands,
 });
+/** "Run play X with tag Y on host Z": every machine repair is a playbook the
+ * worker already applies, restricted to the tag that owns the failing check.
+ */
+const playbook = ({
+  id,
+  title,
+  path,
+  tag,
+  action,
+}: {
+  id: string;
+  title: string;
+  path: string;
+  tag: string;
+  action: string;
+}): RepairRecipe => ({
+  id,
+  title,
+  owner: `${path} (tag ${tag})`,
+  automatic: false,
+  action: `${action} Runs on the worker, in its fleet-owned checkout, with the variables fleet-sync wrote; devfleet fleet-sync --node NODE re-applies the whole policy instead.`,
+  commands: [['ansible-playbook', path, '--tags', tag]],
+});
 export const REPAIR_RECIPES: RepairRecipe[] = [
   {
     id: 'workspace',
@@ -183,6 +206,47 @@ export const REPAIR_RECIPES: RepairRecipe[] = [
       ['bun', 'run', 'password-manager', 'actions', 'list'],
     ],
   },
+  playbook({
+    id: 'recovery',
+    title: 'Re-apply the worker recovery policy',
+    path: 'ansible/ubuntu/recovery.yml',
+    tag: 'recovery',
+    action:
+      'Restores the restart and boot triggers of the worker services (worker:service-recovery). Bounded to the recovery play; it never reboots the worker.',
+  }),
+  playbook({
+    id: 'disk',
+    title: 'Re-apply the worker disk and memory safety policy',
+    path: 'ansible/ubuntu/baseline.yml',
+    tag: 'disk',
+    action:
+      'Restores the disk guard and the disk-gc timer that reclaims caches (worker:capacity:*). Bounded to the resource tasks; it deletes nothing outside the caches disk-gc owns.',
+  }),
+  playbook({
+    id: 'monitoring',
+    title: 'Re-apply the worker monitoring agent',
+    path: 'ansible/monitoring/agent.yml',
+    tag: 'monitoring',
+    action:
+      'Reinstalls the pinned Beszel agent and restarts its unit, keeping its runtime identity (beszel:agent, beszel:service).',
+  }),
+  playbook({
+    id: 'network',
+    title: 'Re-apply the worker Tailscale daemon',
+    path: 'ansible/ubuntu/tailscale.yml',
+    tag: 'network',
+    action:
+      'Reinstalls the approved Tailscale and starts its daemon, keeping the device identity. Signing in again (tailscale up) stays with a person.',
+  }),
+  {
+    id: 'state',
+    title: 'Take the fleet state from the bucket again',
+    owner: 'devfleet fleet-state',
+    automatic: false,
+    action:
+      'On the manager: devfleet fleet-state --restore replaces the local copy with the bucket’s (add --yes to give up changes the bucket does not hold). A corrupt replica is restored from an earlier time with --timestamp; see docs/state.md.',
+    commands: [['devfleet', 'fleet-state', '--restore']],
+  },
   {
     id: 'operator',
     title: 'Resolve the check with its owning tool',

package/src/repair-plan.ts

@@ -1,3 +1,4 @@
+import { checkOwner } from './check-owners';
 import { AUTOMATED_REPAIRS, recipe } from './repair-catalog';
 import type { RepairRecipe } from './repair-catalog';
 import type { Check, FleetNode, NodeReport, Fingerprint } from './types';
@@ -13,21 +14,26 @@ export interface RepairStep extends RepairRecipe {
   checks: string[];
   eligible: boolean;
   next: string[];
+  /** For work left to a person: why no recipe does it, per check. */
+  manual?: { check: string; reason: string }[];
 }
 
 export function repairId(check: Check): string {
-  if (check.id.startsWith('tool:docker-')) return 'docker';
-  if (check.id.startsWith('tool:github-auth')) return 'login:github';
-  if (['dependencies', 'auth:toolchain'].includes(check.id)) return 'workspace';
-  if (check.id.startsWith('tool:')) return 'workspace';
-  if (check.id.startsWith('workstation:')) return 'workstation';
-  if (
-    check.id.startsWith('auth:') &&
-    !['auth:vscode', 'auth:workstation'].includes(check.id)
-  )
-    return `login:${check.id.slice(5)}`;
-  if (check.id === 't3:settings' || check.id === 't3:api') return 't3';
-  return 'operator';
+  const owner = checkOwner(check.id);
+  if (!owner?.repair) return 'operator';
+  return owner.repair === 'login:*'
+    ? `login:${check.id.slice('auth:'.length)}`
+    : owner.repair;
+}
+
+/** Why a check is left to a person, or null when a recipe repairs it. */
+export function manualReason(check: Check): string | null {
+  const owner = checkOwner(check.id);
+  if (owner?.repair) return null;
+  return (
+    owner?.manual ??
+    'No repair recipe is registered for this check: follow its remedy.'
+  );
 }
 
 /** Every nonpassing check is accounted for; unknown checks remain operator work. */
@@ -44,9 +50,14 @@ export function repairPlan(
       const repair = recipe(repairId(check)).id;
       groups.set(repair, [...(groups.get(repair) ?? []), check]);
     }
-    for (const [id, checks] of groups)
+    for (const [id, checks] of groups) {
+      const manual = checks.flatMap((check) => {
+        const reason = manualReason(check);
+        return reason ? [{ check: check.id, reason }] : [];
+      });
       steps.push({
         ...recipe(id),
+        ...(manual.length ? { manual } : {}),
         node: report.node,
         eligible:
           targets.some((target) => target.name === report.node) &&
@@ -60,6 +71,7 @@ export function repairPlan(
           ),
         ],
       });
+    }
   }
   return steps;
 }

package/src/secrets.ts

@@ -26,6 +26,9 @@ export const OPTIONAL_SECRET_NAMES = [
   // Where fleet-watch notifies (notify.targets).
   'notify-webhook-url',
   'notify-ntfy-url',
+  // A short-lived, single-use, tagged Tailscale auth key a new worker joins
+  // the tailnet with (fleet-node-seed).
+  'tailnet-auth-key',
 ] as const;
 export type SecretName =
   (typeof SECRET_NAMES)[number] | (typeof OPTIONAL_SECRET_NAMES)[number];

package/src/state/check.ts

@@ -49,6 +49,13 @@ const FINDINGS: Record<
     action:
       'Check the network, the bucket named under "state" in fleet.json (or saved in bucket.json) and its credentials: DEVFLEET_BUCKET_ACCESS_KEY_ID and DEVFLEET_BUCKET_SECRET_ACCESS_KEY for this command, or the ones devfleet fleet-state --init saved on this machine. Then run devfleet fleet-state --push, which also installs the pinned Litestream when it is missing (that needs mise, which the first fleet-sync on a machine installs). Details are withheld to protect credentials.',
   },
+  corrupt: {
+    status: 'fail',
+    summary:
+      'The bucket answered, but the fleet state it holds cannot be read: the replica is damaged.',
+    action:
+      'The network and the credentials work; the latest version in the bucket does not. Restore the last good version from before the damage, which also uploads it as the current one: devfleet fleet-state --restore --timestamp 2026-11-01T00:00:00Z --yes (a time before the damage; Litestream keeps thirty days). See docs/state.md.',
+  },
   diverged: {
     status: 'fail',
     summary: 'This machine and the bucket hold different fleet states.',

package/src/state/holder.ts

@@ -0,0 +1,50 @@
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+
+/** The machine's own identifier, which survives a reinstall of dev-fleet
+ * and differs between two Macs given the same host name.
+ */
+function machineId(platform: NodeJS.Platform): string | null {
+  try {
+    if (platform === 'linux')
+      return readFileSync('/etc/machine-id', 'utf8').trim() || null;
+    if (platform === 'darwin') {
+      const ioreg = spawnSync(
+        '/usr/sbin/ioreg',
+        ['-rd1', '-c', 'IOPlatformExpertDevice'],
+        { encoding: 'utf8', timeout: 5_000 }
+      );
+      return (
+        /"IOPlatformUUID" = "([^"]+)"/.exec(ioreg.stdout ?? '')?.[1] ?? null
+      );
+    }
+  } catch {
+    // An unidentified machine neither checks nor takes the lease.
+  }
+  return null;
+}
+
+const known = new Map<string, string | null>();
+
+/** This machine, as the manager lease names it: a digest of its machine ID,
+ * never the ID itself. `DEVFLEET_MANAGER_HOLDER` names it instead (a test, a
+ * container without a machine ID). Null when it cannot be told apart.
+ */
+export function managerHolder(
+  env: NodeJS.ProcessEnv = process.env,
+  platform: NodeJS.Platform = process.platform
+): string | null {
+  const named = env['DEVFLEET_MANAGER_HOLDER'];
+  if (named) return named;
+  if (!known.has(platform)) {
+    const id = machineId(platform);
+    known.set(
+      platform,
+      id
+        ? `machine-${createHash('sha256').update(id).digest('hex').slice(0, 16)}`
+        : null
+    );
+  }
+  return known.get(platform) ?? null;
+}

package/src/state/litestream.ts

@@ -125,15 +125,18 @@ export async function replicate(target: Replication): Promise<boolean> {
   ]);
 }
 
-/** Downloads the bucket's latest version to `output`, which must not exist.
- * `absent` means the bucket is reachable and holds no state yet.
+/** Downloads the bucket's latest version to `output`, which must not exist,
+ * or the version it held at `timestamp`. `absent` means the bucket is
+ * reachable and holds no state yet.
  */
 export async function restore(
   target: Replication,
-  output: string
+  output: string,
+  timestamp?: string
 ): Promise<'restored' | 'absent' | 'failed'> {
   const ran = await invoke(target, 'restore', [
     '-if-replica-exists',
+    ...(timestamp ? ['-timestamp', timestamp] : []),
     '-o',
     output,
     target.database,

package/src/state/migrations/0001-documents.sql

@@ -0,0 +1,13 @@
+-- The fleet state as it was before it carried a schema version: the
+-- documents and the head. A database without a version has exactly these.
+CREATE TABLE IF NOT EXISTS documents (
+  name TEXT PRIMARY KEY,
+  body TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+) STRICT;
+CREATE TABLE IF NOT EXISTS head (
+  one INTEGER PRIMARY KEY CHECK (one = 1),
+  id TEXT NOT NULL,
+  sequence INTEGER NOT NULL,
+  written_at TEXT NOT NULL
+) STRICT;

package/src/state/migrations/0002-lease.sql

@@ -0,0 +1,7 @@
+-- The manager lease: which machine may change the fleet state, and until
+-- when. It travels with the state, so the bucket says who holds it.
+CREATE TABLE IF NOT EXISTS lease (
+  one INTEGER PRIMARY KEY CHECK (one = 1),
+  holder TEXT NOT NULL,
+  expires_at TEXT NOT NULL
+) STRICT;