@crvouga/dev-fleet 0.5.0 → 0.7.0

package/src/platform/manager-host.ts

@@ -0,0 +1,184 @@
+import { readFileSync } from 'node:fs';
+import { userInfo } from 'node:os';
+import { join } from 'node:path';
+import macos from '../../ansible/macos/policy.json';
+import { record } from '../config';
+import { hostId } from '../topology';
+import { ansibleCommand } from './ansible';
+import type { Runner } from '../types';
+
+/** One thing a manager needs to come back by itself after a power cut. */
+export interface BootItem {
+  item: string;
+  /** `fail` stops unattended recovery; `warn` is a trade-off to know about. */
+  status: 'pass' | 'warn' | 'fail';
+  detail: string;
+}
+
+/** Where the manager runs, behind one interface: how it is named, how it is
+ * kept awake, how its services are supervised and where they keep data. The
+ * macOS host is the default and the reference.
+ */
+export interface ManagerHost {
+  platform: 'macOS' | 'linux';
+  serviceManager: 'launchd' | 'systemd';
+  containerEngine: 'Docker Desktop' | 'Docker Engine';
+  /** The Tailscale command line. */
+  tailscale: string[];
+  /** Where the manager's services keep their data. */
+  dataDirectory(home: string): string;
+  /** The machine's local host name, as `manager.id` must name it. */
+  localHostName(run: Runner): Promise<string | null>;
+  /** The power settings that differ from the manager's policy. */
+  powerDrift(run: Runner): Promise<string[]>;
+  /** What brings the manager's services back after a power cut, unattended. */
+  bootRecovery(run: Runner, home: string): Promise<BootItem[]>;
+  /** How a person applies what bootRecovery measures. */
+  bootRemedy(workspace: string): string;
+}
+
+const item = (
+  name: string,
+  ok: boolean,
+  [pass, failure, status = 'fail']: [string, string, ('warn' | 'fail')?]
+): BootItem => ({
+  item: name,
+  status: ok ? 'pass' : status,
+  detail: ok ? pass : failure,
+});
+
+/** `pmset -g custom`'s AC Power section, as key and value. */
+async function acPower(
+  run: Runner
+): Promise<(key: string) => string | undefined> {
+  const result = await run(['pmset', '-g', 'custom']);
+  const ac = result.ok ? (result.stdout.split('AC Power:')[1] ?? '') : '';
+  return (key) => new RegExp(`^\\s*${key}\\s+(\\S+)`, 'm').exec(ac)?.[1];
+}
+
+function dockerSettings(home: string): Record<string, unknown> | null {
+  try {
+    const value: unknown = JSON.parse(
+      readFileSync(join(home, macos.dockerDesktop.settings), 'utf8')
+    );
+    return record(value) ? value : null;
+  } catch {
+    return null;
+  }
+}
+
+export const MACOS_MANAGER: ManagerHost = {
+  platform: 'macOS',
+  serviceManager: 'launchd',
+  containerEngine: 'Docker Desktop',
+  tailscale: ['tailscale'],
+  dataDirectory: (home) => join(home, 'Library/Application Support'),
+  async localHostName(run) {
+    const result = await run(['scutil', '--get', 'LocalHostName']);
+    return result.ok ? hostId(result.stdout.trim()) : null;
+  },
+  async powerDrift(run) {
+    const observed = await acPower(run);
+    return Object.entries(macos.pmset)
+      .filter(([key, value]) => observed(key) !== value)
+      .map(([key]) => key);
+  },
+  async bootRecovery(run, home) {
+    const observed = await acPower(run);
+    const docker = dockerSettings(home);
+    const vault = await run(['fdesetup', 'status']);
+    const filevault = vault.ok && /FileVault is On/.test(vault.stdout);
+    const login = await run([
+      'defaults',
+      'read',
+      '/Library/Preferences/com.apple.loginwindow',
+      'autoLoginUser',
+    ]);
+    const agent = await run([
+      'launchctl',
+      'print',
+      `gui/${userInfo().uid}/${macos.beszelDaemon}`,
+    ]);
+    return [
+      item('autorestart', observed('autorestart') === macos.pmset.autorestart, [
+        'The Mac starts again by itself when power returns.',
+        'The Mac stays off after a power cut (pmset autorestart is not 1).',
+      ]),
+      item('docker-autostart', docker?.[macos.dockerDesktop.key] === true, [
+        'Docker Desktop starts at login, and with it the proxy and the hub.',
+        'Docker Desktop does not start at login: the CLI proxy and the monitoring hub stay down after a restart.',
+      ]),
+      filevault
+        ? item('login', false, [
+            '',
+            'FileVault is on: after a restart the Mac waits at the login window until a person signs in, and no LaunchAgent or Docker Desktop starts before that. Turn FileVault off for unattended recovery, or accept that a power cut needs a person (docs/manager-recovery.md).',
+            'warn',
+          ])
+        : item('login', login.ok && login.stdout.trim().length > 0, [
+            'FileVault is off and a user signs in automatically after a restart.',
+            'No user signs in automatically after a restart, so no LaunchAgent starts. Turn on automatic login for the manager’s user in System Settings (it asks for that user’s password).',
+            'warn',
+          ]),
+      item('launch-agents', agent.ok, [
+        'The fleet’s LaunchAgents are loaded.',
+        `The LaunchAgent ${macos.beszelDaemon} is not loaded.`,
+      ]),
+    ];
+  },
+  bootRemedy: (workspace) =>
+    `Apply the manager policy, as an administrator: cd ${workspace} && ${ansibleCommand('darwin').join(' ')} -K ansible/macos/manager.yml (autorestart, Docker Desktop at login), then fleet-sync (LaunchAgents). FileVault and automatic login are a person’s choice: docs/manager-recovery.md.`,
+};
+
+/** The sleep states a Linux manager must never enter. */
+const SLEEP_TARGETS = [
+  'sleep.target',
+  'suspend.target',
+  'hibernate.target',
+  'hybrid-sleep.target',
+];
+
+export const LINUX_MANAGER: ManagerHost = {
+  platform: 'linux',
+  serviceManager: 'systemd',
+  containerEngine: 'Docker Engine',
+  tailscale: ['tailscale'],
+  dataDirectory: (home) => join(home, '.local/share'),
+  async localHostName(run) {
+    const result = await run(['hostnamectl', '--static']);
+    return result.ok ? hostId(result.stdout.trim()) : null;
+  },
+  async powerDrift(run) {
+    const drifted: string[] = [];
+    for (const target of SLEEP_TARGETS) {
+      const result = await run(['systemctl', 'is-enabled', target]);
+      if (result.stdout.trim() !== 'masked') drifted.push(target);
+    }
+    return drifted;
+  },
+  async bootRecovery(run) {
+    const user = userInfo().username;
+    const linger = await run([
+      'loginctl',
+      'show-user',
+      user,
+      '--property=Linger',
+    ]);
+    const docker = await run(['systemctl', 'is-enabled', 'docker.service']);
+    return [
+      item('linger', linger.stdout.trim() === 'Linger=yes', [
+        'The manager’s user services start at boot, without a login.',
+        'The manager’s user services wait for a login (loginctl linger is off).',
+      ]),
+      item('docker-autostart', docker.stdout.trim() === 'enabled', [
+        'Docker Engine starts at boot, and with it the proxy and the hub.',
+        'Docker Engine does not start at boot: the CLI proxy and the monitoring hub stay down after a restart.',
+      ]),
+    ];
+  },
+  bootRemedy: () =>
+    'Enable lingering for the manager’s user and the docker service with the Linux manager policy; a Linux manager role is not shipped yet.',
+};
+
+export function managerHost(platform: 'macOS' | 'linux'): ManagerHost {
+  return platform === 'linux' ? LINUX_MANAGER : MACOS_MANAGER;
+}

package/src/repair-catalog.ts

@@ -20,6 +20,29 @@ const login = (
   action,
   commands,
 });
+/** "Run play X with tag Y on host Z": every machine repair is a playbook the
+ * worker already applies, restricted to the tag that owns the failing check.
+ */
+const playbook = ({
+  id,
+  title,
+  path,
+  tag,
+  action,
+}: {
+  id: string;
+  title: string;
+  path: string;
+  tag: string;
+  action: string;
+}): RepairRecipe => ({
+  id,
+  title,
+  owner: `${path} (tag ${tag})`,
+  automatic: false,
+  action: `${action} Runs on the worker, in its fleet-owned checkout, with the variables fleet-sync wrote; devfleet fleet-sync --node NODE re-applies the whole policy instead.`,
+  commands: [['ansible-playbook', path, '--tags', tag]],
+});
 export const REPAIR_RECIPES: RepairRecipe[] = [
   {
     id: 'workspace',
@@ -141,8 +164,8 @@ export const REPAIR_RECIPES: RepairRecipe[] = [
   login(
     'copilot',
     'Sign in to Copilot',
-    [['copilot', 'login']],
-    'Complete native login; provider authorization still needs supported operator diagnostics. No model prompt is used for health checking.'
+    [],
+    'Copilot CLI uses the node’s GitHub sign-in, which fleet-sync brings from the stored credential (github-token). Run fleet-sync; store a fine-grained token with the Copilot Requests permission if auth:copilot says the token kind is refused. No per-node copilot login.'
   ),
   login(
     'gemini',
@@ -183,6 +206,47 @@ export const REPAIR_RECIPES: RepairRecipe[] = [
       ['bun', 'run', 'password-manager', 'actions', 'list'],
     ],
   },
+  playbook({
+    id: 'recovery',
+    title: 'Re-apply the worker recovery policy',
+    path: 'ansible/ubuntu/recovery.yml',
+    tag: 'recovery',
+    action:
+      'Restores the restart and boot triggers of the worker services (worker:service-recovery). Bounded to the recovery play; it never reboots the worker.',
+  }),
+  playbook({
+    id: 'disk',
+    title: 'Re-apply the worker disk and memory safety policy',
+    path: 'ansible/ubuntu/baseline.yml',
+    tag: 'disk',
+    action:
+      'Restores the disk guard and the disk-gc timer that reclaims caches (worker:capacity:*). Bounded to the resource tasks; it deletes nothing outside the caches disk-gc owns.',
+  }),
+  playbook({
+    id: 'monitoring',
+    title: 'Re-apply the worker monitoring agent',
+    path: 'ansible/monitoring/agent.yml',
+    tag: 'monitoring',
+    action:
+      'Reinstalls the pinned Beszel agent and restarts its unit, keeping its runtime identity (beszel:agent, beszel:service).',
+  }),
+  playbook({
+    id: 'network',
+    title: 'Re-apply the worker Tailscale daemon',
+    path: 'ansible/ubuntu/tailscale.yml',
+    tag: 'network',
+    action:
+      'Reinstalls the approved Tailscale and starts its daemon, keeping the device identity. Signing in again (tailscale up) stays with a person.',
+  }),
+  {
+    id: 'state',
+    title: 'Take the fleet state from the bucket again',
+    owner: 'devfleet fleet-state',
+    automatic: false,
+    action:
+      'On the manager: devfleet fleet-state --restore replaces the local copy with the bucket’s (add --yes to give up changes the bucket does not hold). A corrupt replica is restored from an earlier time with --timestamp; see docs/state.md.',
+    commands: [['devfleet', 'fleet-state', '--restore']],
+  },
   {
     id: 'operator',
     title: 'Resolve the check with its owning tool',

package/src/repair-plan.ts

@@ -1,3 +1,4 @@
+import { checkOwner } from './check-owners';
 import { AUTOMATED_REPAIRS, recipe } from './repair-catalog';
 import type { RepairRecipe } from './repair-catalog';
 import type { Check, FleetNode, NodeReport, Fingerprint } from './types';
@@ -13,21 +14,26 @@ export interface RepairStep extends RepairRecipe {
   checks: string[];
   eligible: boolean;
   next: string[];
+  /** For work left to a person: why no recipe does it, per check. */
+  manual?: { check: string; reason: string }[];
 }
 
 export function repairId(check: Check): string {
-  if (check.id.startsWith('tool:docker-')) return 'docker';
-  if (check.id.startsWith('tool:github-auth')) return 'login:github';
-  if (['dependencies', 'auth:toolchain'].includes(check.id)) return 'workspace';
-  if (check.id.startsWith('tool:')) return 'workspace';
-  if (check.id.startsWith('workstation:')) return 'workstation';
-  if (
-    check.id.startsWith('auth:') &&
-    !['auth:vscode', 'auth:workstation'].includes(check.id)
-  )
-    return `login:${check.id.slice(5)}`;
-  if (check.id === 't3:settings' || check.id === 't3:api') return 't3';
-  return 'operator';
+  const owner = checkOwner(check.id);
+  if (!owner?.repair) return 'operator';
+  return owner.repair === 'login:*'
+    ? `login:${check.id.slice('auth:'.length)}`
+    : owner.repair;
+}
+
+/** Why a check is left to a person, or null when a recipe repairs it. */
+export function manualReason(check: Check): string | null {
+  const owner = checkOwner(check.id);
+  if (owner?.repair) return null;
+  return (
+    owner?.manual ??
+    'No repair recipe is registered for this check: follow its remedy.'
+  );
 }
 
 /** Every nonpassing check is accounted for; unknown checks remain operator work. */
@@ -44,9 +50,14 @@ export function repairPlan(
       const repair = recipe(repairId(check)).id;
       groups.set(repair, [...(groups.get(repair) ?? []), check]);
     }
-    for (const [id, checks] of groups)
+    for (const [id, checks] of groups) {
+      const manual = checks.flatMap((check) => {
+        const reason = manualReason(check);
+        return reason ? [{ check: check.id, reason }] : [];
+      });
       steps.push({
         ...recipe(id),
+        ...(manual.length ? { manual } : {}),
         node: report.node,
         eligible:
           targets.some((target) => target.name === report.node) &&
@@ -60,6 +71,7 @@ export function repairPlan(
           ),
         ],
       });
+    }
   }
   return steps;
 }

package/src/secrets.ts

@@ -12,10 +12,13 @@ export const SECRET_NAMES = [
   'monitoring-tunnel-token',
   'proxy-tunnel-token',
 ] as const;
-/** Secrets only a feature the fleet declares needs: a Vault location may
- * leave them out, and they are then unavailable.
+/** Secrets a fleet may leave undeclared. A Vault fleet that names no location
+ * for the GitHub token keeps it where dev-fleet always read it (see
+ * `vaultLocation`); the others are needed only by a feature the fleet
+ * declares, and are otherwise unavailable.
  */
 export const OPTIONAL_SECRET_NAMES = [
+  'github-token',
   // The off-machine storage of the monitoring hub's backups
   // (services.monitoring.backups).
   'monitoring-backup-access-key',
@@ -52,16 +55,53 @@ export type SecretCommand = (
 ) => Promise<string | null>;
 
 /** One read of one secret for this node. Null when it is not available; the
- * value is never logged, printed or placed in argv.
+ * value is never logged, printed or placed in argv. `node` names a node's own
+ * copy of a secret that is kept once per node (`secretVariant`).
  */
-export type SecretsProvider = (name: SecretName) => Promise<string | null>;
+export type SecretsProvider = (
+  name: SecretName,
+  node?: string
+) => Promise<string | null>;
 
 const present = (value: string | null | undefined): string | null =>
   value?.trim() ? value.trim() : null;
 
 /** `proxy-client-key` is read from `DEVFLEET_SECRET_PROXY_CLIENT_KEY`. */
-export const envName = (name: SecretName): string =>
-  `DEVFLEET_SECRET_${name.toUpperCase().replaceAll('-', '_')}`;
+export const envName = (name: SecretName, node?: string): string =>
+  `DEVFLEET_SECRET_${secretVariant(name, node).toUpperCase().replaceAll('-', '_')}`;
+
+/** The name of a node's own copy of a secret: `github-token` of `worker-one`
+ * is `github-token-worker-one` (`DEVFLEET_SECRET_GITHUB_TOKEN_WORKER_ONE`, the
+ * file or sops key `github-token-worker-one`, the Vault field `FIELD_WORKER_ONE`).
+ */
+export const secretVariant = (name: SecretName, node?: string): string =>
+  node ? `${name}-${node}` : name;
+
+/** Where a Vault fleet keeps a secret. The GitHub token was always read from
+ * the field `GITHUB_TOKEN` beside the proxy client key; a fleet that names no
+ * location of its own keeps it there.
+ */
+export function vaultLocation(
+  config: Extract<SecretsConfig, { provider: 'vault' }>,
+  name: SecretName,
+  node?: string
+): VaultLocation | null {
+  const location =
+    config.locations[name as keyof typeof config.locations] ??
+    (name === 'github-token'
+      ? {
+          path: config.locations['proxy-client-key'].path,
+          field: 'GITHUB_TOKEN',
+        }
+      : null);
+  if (!location) return null;
+  return node
+    ? {
+        path: location.path,
+        field: `${location.field}_${node.toUpperCase().replaceAll('-', '_')}`,
+      }
+    : location;
+}
 
 export function secretsProvider(
   config: SecretsConfig,
@@ -74,29 +114,34 @@ export function secretsProvider(
   const env = options.env ?? process.env;
   const run = options.run ?? execute;
   if (config.provider === 'env')
-    return async (name) => present(env[envName(name)]);
+    return async (name, node) => present(env[envName(name, node)]);
   if (config.provider === 'file') {
     // One private file per secret, named after it.
     const directory =
       config.directory ??
       join(options.home ?? homedir(), '.config/dev-fleet/secrets');
-    return async (name) =>
-      present(await readFile(join(directory, name), 'utf8').catch(() => null));
+    return async (name, node) =>
+      present(
+        await readFile(
+          join(directory, secretVariant(name, node)),
+          'utf8'
+        ).catch(() => null)
+      );
   }
   if (config.provider === 'sops')
     // An age- (or KMS-) encrypted document with one top-level key per secret.
-    return async (name) =>
+    return async (name, node) =>
       present(
         await run([
           'sops',
           '--decrypt',
           '--extract',
-          `["${name}"]`,
+          `["${secretVariant(name, node)}"]`,
           config.file,
         ])
       );
-  return async (name) => {
-    const location = config.locations[name];
+  return async (name, node) => {
+    const location = vaultLocation(config, name, node);
     if (!location) return null;
     return present(
       await run([

package/src/service-auth.ts

@@ -1,4 +1,5 @@
 import { record } from './config';
+import { copilotCheck } from './github/copilot';
 import type { Check, CommandResult, Runner } from './types';
 
 interface Service {
@@ -48,8 +49,6 @@ export async function codingAuth(
 export const PROXY_ROUTED: Record<string, string> = {
   gemini:
     'gemini signs in through the CLI proxy and has no T3 launcher; provider:gemini:inference proves its route from this node.',
-  copilot:
-    'copilot is not routed through the CLI proxy and dev-fleet does not manage its GitHub sign-in; no provider account is checked.',
   opencode:
     'opencode signs in through the CLI proxy; agent:muse:session and the per-account agent:opencode sessions prove it from this node.',
 };
@@ -101,6 +100,7 @@ export async function serviceAuth(run: Runner): Promise<Check[]> {
   ];
   const checks: Check[] = [];
   for (const service of services) checks.push(await serviceCheck(service, run));
+  checks.push(await copilotCheck(run));
   for (const [id, reason] of Object.entries(PROXY_ROUTED))
     checks.push({
       id: `auth:${id}`,

package/src/state/check.ts

@@ -49,6 +49,13 @@ const FINDINGS: Record<
     action:
       'Check the network, the bucket named under "state" in fleet.json (or saved in bucket.json) and its credentials: DEVFLEET_BUCKET_ACCESS_KEY_ID and DEVFLEET_BUCKET_SECRET_ACCESS_KEY for this command, or the ones devfleet fleet-state --init saved on this machine. Then run devfleet fleet-state --push, which also installs the pinned Litestream when it is missing (that needs mise, which the first fleet-sync on a machine installs). Details are withheld to protect credentials.',
   },
+  corrupt: {
+    status: 'fail',
+    summary:
+      'The bucket answered, but the fleet state it holds cannot be read: the replica is damaged.',
+    action:
+      'The network and the credentials work; the latest version in the bucket does not. Restore the last good version from before the damage, which also uploads it as the current one: devfleet fleet-state --restore --timestamp 2026-11-01T00:00:00Z --yes (a time before the damage; Litestream keeps thirty days). See docs/state.md.',
+  },
   diverged: {
     status: 'fail',
     summary: 'This machine and the bucket hold different fleet states.',

package/src/state/holder.ts

@@ -0,0 +1,50 @@
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+
+/** The machine's own identifier, which survives a reinstall of dev-fleet
+ * and differs between two Macs given the same host name.
+ */
+function machineId(platform: NodeJS.Platform): string | null {
+  try {
+    if (platform === 'linux')
+      return readFileSync('/etc/machine-id', 'utf8').trim() || null;
+    if (platform === 'darwin') {
+      const ioreg = spawnSync(
+        '/usr/sbin/ioreg',
+        ['-rd1', '-c', 'IOPlatformExpertDevice'],
+        { encoding: 'utf8', timeout: 5_000 }
+      );
+      return (
+        /"IOPlatformUUID" = "([^"]+)"/.exec(ioreg.stdout ?? '')?.[1] ?? null
+      );
+    }
+  } catch {
+    // An unidentified machine neither checks nor takes the lease.
+  }
+  return null;
+}
+
+const known = new Map<string, string | null>();
+
+/** This machine, as the manager lease names it: a digest of its machine ID,
+ * never the ID itself. `DEVFLEET_MANAGER_HOLDER` names it instead (a test, a
+ * container without a machine ID). Null when it cannot be told apart.
+ */
+export function managerHolder(
+  env: NodeJS.ProcessEnv = process.env,
+  platform: NodeJS.Platform = process.platform
+): string | null {
+  const named = env['DEVFLEET_MANAGER_HOLDER'];
+  if (named) return named;
+  if (!known.has(platform)) {
+    const id = machineId(platform);
+    known.set(
+      platform,
+      id
+        ? `machine-${createHash('sha256').update(id).digest('hex').slice(0, 16)}`
+        : null
+    );
+  }
+  return known.get(platform) ?? null;
+}

package/src/state/litestream.ts

@@ -125,15 +125,18 @@ export async function replicate(target: Replication): Promise<boolean> {
   ]);
 }
 
-/** Downloads the bucket's latest version to `output`, which must not exist.
- * `absent` means the bucket is reachable and holds no state yet.
+/** Downloads the bucket's latest version to `output`, which must not exist,
+ * or the version it held at `timestamp`. `absent` means the bucket is
+ * reachable and holds no state yet.
  */
 export async function restore(
   target: Replication,
-  output: string
+  output: string,
+  timestamp?: string
 ): Promise<'restored' | 'absent' | 'failed'> {
   const ran = await invoke(target, 'restore', [
     '-if-replica-exists',
+    ...(timestamp ? ['-timestamp', timestamp] : []),
     '-o',
     output,
     target.database,

package/src/state/migrations/0001-documents.sql

@@ -0,0 +1,13 @@
+-- The fleet state as it was before it carried a schema version: the
+-- documents and the head. A database without a version has exactly these.
+CREATE TABLE IF NOT EXISTS documents (
+  name TEXT PRIMARY KEY,
+  body TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+) STRICT;
+CREATE TABLE IF NOT EXISTS head (
+  one INTEGER PRIMARY KEY CHECK (one = 1),
+  id TEXT NOT NULL,
+  sequence INTEGER NOT NULL,
+  written_at TEXT NOT NULL
+) STRICT;

package/src/state/migrations/0002-lease.sql

@@ -0,0 +1,7 @@
+-- The manager lease: which machine may change the fleet state, and until
+-- when. It travels with the state, so the bucket says who holds it.
+CREATE TABLE IF NOT EXISTS lease (
+  one INTEGER PRIMARY KEY CHECK (one = 1),
+  holder TEXT NOT NULL,
+  expires_at TEXT NOT NULL
+) STRICT;