@crvouga/dev-fleet 0.5.0 → 0.7.0

package/src/cli/fleet-restore.ts

@@ -0,0 +1,205 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { parseDefinition, type FleetDefinition } from '../definition';
+import { definitionSource } from '../fleet-config';
+import { managerHost } from '../platform/manager-host';
+import { makeReport, renderReport, reportExitCode } from '../report';
+import { runCommand } from '../runner';
+import { connection } from '../state/connection';
+import { managerHolder } from '../state/holder';
+import { stateRoot } from '../state/paths';
+import { pull, standing, type StateContext } from '../state/sync';
+import type { Check, Runner } from '../types';
+import type { CommandOutcome } from './contracts';
+
+export interface FleetRestoreOptions {
+  dryRun?: boolean;
+  json?: boolean;
+}
+
+export interface FleetRestoreServices {
+  run: Runner;
+  env: NodeJS.ProcessEnv;
+  home: string;
+  holder?: string | null;
+}
+
+/** What a replacement manager gets back, in the order it is restored. */
+export const RESTORES = [
+  {
+    what: 'Fleet state (roster, approved revision, configuration last applied)',
+    how: 'Downloaded from the bucket by this command.',
+  },
+  {
+    what: 'Machine policy, toolchain and checkouts',
+    how: 'Rebuilt from Git by devfleet fleet-sync.',
+  },
+  {
+    what: 'CLI proxy state (provider logins, keys, Tailscale identity)',
+    how: 'Restored from its encrypted snapshots by devfleet cliproxy install (vault secrets provider only).',
+  },
+  {
+    what: 'Monitoring hub',
+    how: 'Recreated by devfleet fleet-sync with its accounts and systems; metric history is lost.',
+  },
+  {
+    what: 'T3 ownership record',
+    how: 'Adopted again with devfleet t3-apply --import ID.',
+  },
+];
+
+const check = (
+  id: string,
+  ok: boolean,
+  [pass, failure, action, missing = 'fail']: [
+    string,
+    string,
+    string,
+    ('fail' | 'warn')?,
+  ]
+): Check =>
+  ok
+    ? { id, status: 'pass', summary: pass }
+    : { id, status: missing, summary: failure, remedy: { action } };
+
+function definition(services: FleetRestoreServices): FleetDefinition | null {
+  const source = definitionSource(services.env, services.home);
+  if (source.kind !== 'file' || !existsSync(source.path)) return null;
+  try {
+    return parseDefinition(JSON.parse(readFileSync(source.path, 'utf8')));
+  } catch {
+    return null;
+  }
+}
+
+/** Every prerequisite of a manager restore, read-only. */
+async function prerequisites(
+  services: FleetRestoreServices,
+  context: StateContext
+): Promise<Check[]> {
+  const fleet = definition(services);
+  const reached = connection(services.env, services.home);
+  const checks: Check[] = [
+    check('restore:configuration', fleet !== null, [
+      'The project’s fleet.json is found and valid.',
+      'No valid fleet.json was found.',
+      'Run this from the project that holds your fleet.json, after git clone and npm ci.',
+    ]),
+    check('restore:bucket', reached.bucket !== null, [
+      'The bucket is named, with credentials for it.',
+      'No bucket is configured.',
+      'Name it under "state" in fleet.json and pass DEVFLEET_BUCKET_ACCESS_KEY_ID and DEVFLEET_BUCKET_SECRET_ACCESS_KEY.',
+    ]),
+  ];
+  if (reached.bucket) {
+    const stands = await standing(context);
+    checks.push(
+      check(
+        'restore:state',
+        stands === 'bucket-ahead' || stands === 'in-sync',
+        [
+          'The bucket answers and holds the fleet state.',
+          `The fleet state cannot be restored from the bucket: it stands "${stands}".`,
+          'See the state:replica remedy of devfleet fleet-state; a corrupt replica is restored with --restore --timestamp.',
+        ]
+      )
+    );
+  }
+  if (fleet) {
+    const host = managerHost(fleet.manager.os);
+    const name = await host.localHostName(services.run);
+    checks.push(
+      check('restore:host-name', name === fleet.manager.id, [
+        `This machine’s local host name is the manager’s id (${fleet.manager.id}).`,
+        `This machine’s local host name is not the manager’s id (${fleet.manager.id}).`,
+        host.platform === 'macOS'
+          ? `Set it: sudo scutil --set LocalHostName ${fleet.manager.id}`
+          : `Set it: sudo hostnamectl set-hostname ${fleet.manager.id}`,
+      ])
+    );
+  }
+  const git = await services.run(['git', '--version']);
+  checks.push(
+    check('restore:git', git.ok, [
+      'Git is installed.',
+      'Git is not installed.',
+      'Install Git; fleet-sync installs everything else.',
+    ])
+  );
+  const tailscale = await services.run(['tailscale', 'status', '--json']);
+  checks.push(
+    check(
+      'restore:tailscale',
+      tailscale.ok && /"BackendState":\s*"Running"/.test(tailscale.stdout),
+      [
+        'This machine is signed in to the tailnet.',
+        'This machine is not signed in to the tailnet.',
+        'Sign in to Tailscale and approve the device: that is a person’s.',
+        'warn',
+      ]
+    )
+  );
+  const docker = await services.run(['docker', 'version']);
+  checks.push(
+    check('restore:container-engine', docker.ok, [
+      'The container engine answers.',
+      'The container engine does not answer.',
+      'Install and start Docker Desktop; the proxy and the hub run in it.',
+      'warn',
+    ])
+  );
+  return checks;
+}
+
+/** Brings a replacement manager back from the bucket: `--dry-run` verifies
+ * every prerequisite and lists what would be restored, changing nothing.
+ */
+export async function fleetRestore(
+  options: FleetRestoreOptions,
+  services: FleetRestoreServices = {
+    run: runCommand,
+    env: process.env,
+    home: homedir(),
+    holder: managerHolder(process.env),
+  }
+): Promise<CommandOutcome> {
+  const reached = connection(services.env, services.home);
+  const context: StateContext = {
+    root: stateRoot(services.env, services.home),
+    bucket: reached.bucket,
+    run: services.run,
+    holder: services.holder ?? null,
+  };
+  const checks = await prerequisites(services, context);
+  const ready = !checks.some((entry) => entry.status === 'fail');
+  const done: string[] = [];
+  if (ready && !options.dryRun) {
+    const after = await pull(context);
+    checks.push(
+      check('restore:applied', after === 'in-sync', [
+        'This machine holds the fleet state from the bucket. Next: devfleet fleet-sync (it may need --take-over while the old manager’s lease runs), then devfleet cliproxy install.',
+        `The fleet state was not restored: it stands "${after}".`,
+        'Run devfleet fleet-state for the remedy.',
+      ])
+    );
+    done.push('Restored the fleet state from the bucket.');
+  }
+  const data = {
+    ...makeReport('fleet-restore', [{ node: 'manager', checks }]),
+    dryRun: options.dryRun === true,
+    restores: RESTORES,
+  };
+  return {
+    exitCode: reportExitCode(data, false),
+    data,
+    text: [
+      ...done,
+      renderReport(data),
+      '',
+      options.dryRun ? 'Would restore, in this order:' : 'Restored in order:',
+      ...RESTORES.map(
+        (entry, index) => `${index + 1}. ${entry.what}: ${entry.how}`
+      ),
+    ].join('\n'),
+  };
+}

package/src/cli/fleet-state.ts

@@ -25,6 +25,7 @@ import {
   type Connection,
 } from '../state/connection';
 import { adoptEarlier, legacyPath, readLegacy } from '../state/legacy';
+import { managerHolder } from '../state/holder';
 import { bucketPath, databasePath, stateRoot } from '../state/paths';
 import {
   DOCUMENTS,
@@ -36,7 +37,11 @@ import {
 import {
   mutateState,
   pending,
+  leaseConflict,
+  leaseHeldMessage,
   pull,
+  pullAt,
+  refresh,
   push,
   standing,
   type Standing,
@@ -58,6 +63,8 @@ export interface FleetStateOptions {
   push?: boolean;
   restore?: boolean;
   yes?: boolean;
+  timestamp?: string;
+  takeOver?: boolean;
   json?: boolean;
   nonInteractive?: boolean;
 }
@@ -68,6 +75,10 @@ export interface FleetStateServices {
   home: string;
   /** Asks for one hidden value on the terminal; null when there is none. */
   ask: ((question: string) => Promise<string>) | null;
+  /** This machine, as the manager lease names it; without one the lease is
+   * neither checked nor taken.
+   */
+  holder?: string | null;
 }
 
 function terminalAsk(
@@ -84,11 +95,16 @@ function terminalAsk(
 const reach = (services: FleetStateServices): Connection =>
   connection(services.env, services.home);
 
-function context(services: FleetStateServices): StateContext {
+function context(
+  services: FleetStateServices,
+  options: FleetStateOptions = {}
+): StateContext {
   return {
     root: stateRoot(services.env, services.home),
     bucket: reach(services).bucket,
     run: services.run,
+    holder: services.holder ?? null,
+    takeOver: options.takeOver === true,
   };
 }
 
@@ -109,6 +125,8 @@ function action(options: FleetStateOptions): Action | 'status' {
       'Choose one of --init, --import, --export, --example, --push or --restore.'
     );
   const settings = ['bucket', 'endpoint', 'region', 'prefix', 'directory'];
+  if (options.timestamp !== undefined && chosen[0] !== 'restore')
+    throw new FleetUsageError('--timestamp belongs to --restore.');
   if (
     chosen[0] !== 'init' &&
     settings.some((name) => options[name as keyof FleetStateOptions])
@@ -367,8 +385,22 @@ async function init(
       `Saved the bucket settings to ${bucketPath(services.env, services.home)}.`
     );
   }
-  const target = context(services);
+  const target = context(services, options);
   const had = listDocuments(databasePath(target.root)).length > 0;
+  // A replacement connects to restore. While another manager holds the
+  // lease this one only takes the bucket's version and writes nothing.
+  const fresh = await refresh(target);
+  const held = leaseConflict(target);
+  if (held) {
+    if (!had && fresh.replaced)
+      done.push('Restored the fleet state from the bucket.');
+    done.push(leaseHeldMessage(held));
+    return report(services, {
+      standing: fresh.standing,
+      done,
+      missing: 'warn',
+    });
+  }
   const { value, standing: after } = await mutateState(target, (before) => {
     // A state the bucket already holds is the fleet's; this machine's
     // earlier files only ever seed an empty one.
@@ -388,7 +420,8 @@ async function init(
 
 async function importDefinition(
   path: string,
-  services: FleetStateServices
+  services: FleetStateServices,
+  options: FleetStateOptions
 ): Promise<CommandOutcome> {
   let value: unknown;
   try {
@@ -399,7 +432,7 @@ async function importDefinition(
     );
   }
   const body = JSON.stringify(parseDefinition(value), null, 2) + '\n';
-  const target = context(services);
+  const target = context(services, options);
   const { standing: after } = await mutateState(target, () => {
     adoptEarlier(services.env, services.home);
     return writeDocument(databasePath(target.root), 'definition', body);
@@ -458,6 +491,23 @@ async function restore(
     throw new FleetUsageError(
       'No bucket is configured, so there is nothing to restore from. Name it under "state" in fleet.json, or run devfleet fleet-state --init --bucket NAME --endpoint https://HOST.'
     );
+  if (options.timestamp !== undefined) {
+    // The bucket's current version is replaced too: --yes is the consent.
+    if (!options.yes)
+      throw new FleetUsageError(
+        '--restore --timestamp replaces the fleet state in the bucket and on this machine with an earlier version. Repeat with --yes.'
+      );
+    const after = await pullAt(target, options.timestamp);
+    return report(services, {
+      standing: after,
+      done:
+        after === 'in-sync'
+          ? [
+              `This machine and the bucket now hold the fleet state as it was at ${options.timestamp}.`,
+            ]
+          : [],
+    });
+  }
   if (pending(target) && !options.yes)
     throw new FleetUsageError(
       'This machine has fleet state the bucket does not hold, and --restore discards it. Upload it with --push, or repeat with --yes to discard it.'
@@ -477,6 +527,7 @@ const DEFAULTS = (options: FleetStateOptions): FleetStateServices => ({
   env: process.env,
   home: homedir(),
   ask: terminalAsk(options),
+  holder: managerHolder(process.env),
 });
 
 /** `fleet-state` alone is read-only. Each flag is one explicit action on the
@@ -488,7 +539,8 @@ export async function fleetState(
 ): Promise<CommandOutcome> {
   const chosen = action(options);
   if (chosen === 'init') return init(options, services);
-  if (chosen === 'import') return importDefinition(options.import!, services);
+  if (chosen === 'import')
+    return importDefinition(options.import!, services, options);
   if (chosen === 'export') return exportDefinition(services);
   if (chosen === 'example') return exampleDefinition();
   if (chosen === 'restore') return restore(options, services);

package/src/cli/fleet-sync-manager.ts

@@ -1,7 +1,6 @@
 import { blocking, type Check, type Runner } from '../types';
 import type { FleetDefinition } from '../definition';
 import type { ManagerNode } from '../manager-sync';
-import { hostId } from '../topology';
 import { convergeProxy } from '../proxy-converge';
 import { attempt } from '../node-converge-shared';
 import { quotaLimited, verifyFleetProviders } from '../agents/fleet-verify';
@@ -14,11 +13,17 @@ import { IDES } from '../ide';
 import { enabled, managerChecks } from '../ide/orchestrate';
 import { idesOf } from '../ide/definition';
 import { convergeGitIdentity, gitChecks } from '../git-identity';
-import macos from '../../ansible/macos/policy.json';
 import { ansibleCommand } from '../platform/ansible';
+import { convergeGithub } from '../github/converge';
 import { PACKAGE_ROOT } from '../package-info';
 import { managerHealth } from '../manager-health';
 import { readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import {
+  MACOS_MANAGER,
+  managerHost,
+  type ManagerHost,
+} from '../platform/manager-host';
 import { join } from 'node:path';
 
 /** The manager converges itself locally (never over SSH) before any worker:
@@ -36,12 +41,14 @@ export async function convergeManager(
 ): Promise<ManagerNode> {
   const started = Date.now();
   const id = definition.manager.id;
-  const identity = await managerIdentity(id, run);
+  const host = managerHost(definition.manager.os);
+  const identity = await managerIdentity(id, run, host);
   const checks: Check[] = [identity];
   if (identity.status === 'pass') {
     // Launchers and scheduled jobs on the manager read the installed copy.
     if (input.apply) installConfig();
-    checks.push(await managerPower(run, input.workspace));
+    checks.push(await managerPower(run, input.workspace, host));
+    checks.push(await managerBootRecovery(run, input.workspace, host));
     checks.push(await managerDocker(run));
     // Each enabled IDE: its settings, pinned version and policy.
     checks.push(
@@ -70,6 +77,14 @@ export async function convergeManager(
       );
       if (definition.git)
         checks.push(...(await gitChecks(definition.git, run)));
+      // The same GitHub sign-in as every worker, or only a check of a
+      // person's own login when the configuration names one. git asks gh
+      // for the token through the helper git.yml declared above.
+      checks.push(
+        ...(await attempt('github:sync', async () => [
+          await convergeGithub(input.workspace, run),
+        ]))
+      );
       const providers = await attempt('provider:verification', () =>
         verifyFleetProviders(id)
       );
@@ -184,9 +199,12 @@ export async function managerAgentProfiles(
 }
 
 /** fleet-sync runs only on the declared manager. */
-export async function managerIdentity(id: string, run: Runner): Promise<Check> {
-  const result = await run(['scutil', '--get', 'LocalHostName']);
-  const local = result.ok ? hostId(result.stdout.trim()) : '';
+export async function managerIdentity(
+  id: string,
+  run: Runner,
+  host: ManagerHost = MACOS_MANAGER
+): Promise<Check> {
+  const local = (await host.localHostName(run)) ?? '';
   return local === id
     ? {
         id: 'manager:identity',
@@ -209,15 +227,10 @@ export async function managerIdentity(id: string, run: Runner): Promise<Check> {
  */
 export async function managerPower(
   run: Runner,
-  workspace = '.'
+  workspace = '.',
+  host: ManagerHost = MACOS_MANAGER
 ): Promise<Check> {
-  const result = await run(['pmset', '-g', 'custom']);
-  const ac = result.ok ? (result.stdout.split('AC Power:')[1] ?? '') : '';
-  const observed = (key: string) =>
-    new RegExp(`^\\s*${key}\\s+(\\S+)`, 'm').exec(ac)?.[1];
-  const drifted = Object.entries(macos.pmset)
-    .filter(([key, value]) => observed(key) !== value)
-    .map(([key]) => key);
+  const drifted = await host.powerDrift(run);
   return drifted.length === 0
     ? {
         id: 'manager:power',
@@ -236,6 +249,40 @@ export async function managerPower(
       };
 }
 
+/** After a power cut the manager comes back by itself: the machine starts
+ * again, a user session exists for its per-user services, and the container
+ * engine that hosts the proxy and the hub starts with it. A FileVault Mac
+ * waits for a person; that is reported as a warning with its consequence.
+ */
+export async function managerBootRecovery(
+  run: Runner,
+  workspace = '.',
+  host: ManagerHost = MACOS_MANAGER,
+  home = homedir()
+): Promise<Check> {
+  const items = await host.bootRecovery(run, home);
+  const failing = items.filter((entry) => entry.status === 'fail');
+  const warning = items.filter((entry) => entry.status === 'warn');
+  if (!failing.length && !warning.length)
+    return {
+      id: 'manager:boot-recovery',
+      status: 'pass',
+      summary: 'The manager comes back by itself after a power cut or reboot.',
+    };
+  const named = failing.length ? failing : warning;
+  return {
+    id: 'manager:boot-recovery',
+    status: failing.length ? 'fail' : 'warn',
+    summary: `MANAGER_BOOT_RECOVERY_INCOMPLETE ${named.map((entry) => entry.item).join(' ')}`,
+    remedy: {
+      action: [
+        ...[...failing, ...warning].map((entry) => entry.detail),
+        host.bootRemedy(workspace),
+      ].join(' '),
+    },
+  };
+}
+
 /** The Docker Desktop release `mise.toml` pins, as the engine names itself. */
 export function pinnedDockerDesktop(root = PACKAGE_ROOT): string {
   const { vars } = Bun.TOML.parse(

package/src/cli/fleet-sync-state.ts

@@ -4,7 +4,14 @@ import { storeEnrollment } from '../state/enrollment';
 import { adoptEarlier } from '../state/legacy';
 import { databasePath, stateRoot } from '../state/paths';
 import { writeDocument } from '../state/store';
-import { pending, push, type Standing, type StateContext } from '../state/sync';
+import {
+  claimLease,
+  refuseHeldLease,
+  pending,
+  push,
+  type Standing,
+  type StateContext,
+} from '../state/sync';
 import type { Check } from '../types';
 
 /** What one fleet-sync run records in the fleet state: the revision it was
@@ -51,14 +58,17 @@ export function runRecord(
       : Promise.resolve(state.standing);
   return {
     approve: () => {
+      refuseHeldLease(state.context);
       storeEnrollment(approval, env);
-      if (state.definition === undefined) return;
-      adoptEarlier(env);
-      writeDocument(
-        databasePath(stateRoot(env)),
-        'definition',
-        state.definition
-      );
+      if (state.definition !== undefined) {
+        adoptEarlier(env);
+        writeDocument(
+          databasePath(stateRoot(env)),
+          'definition',
+          state.definition
+        );
+      }
+      claimLease(state.context);
     },
     async settle(converged) {
       settled = true;

package/src/cli/fleet-sync.ts

@@ -31,6 +31,8 @@ import {
   DIVERGED_REMEDY,
   mutateState,
   refresh,
+  leaseConflict,
+  leaseHeldMessage,
   stateContext,
   type Standing,
 } from '../state/sync';
@@ -72,6 +74,8 @@ export interface FleetSyncOptions {
   /** Internal: set for interactive terminal runs (sudo enrollment prompts). */
   interactiveAuthorization?: boolean;
   expectedPlanFingerprint?: string;
+  /** Take the manager lease from another manager before it expired. */
+  takeOver?: boolean;
 }
 export function machineRoot(): string {
   const root = join(homedir(), '.local/share/personal/fleet/manager');
@@ -140,6 +144,31 @@ export interface ManagerSync {
   restart: () => Promise<CommandOutcome>;
 }
 
+/** Why the fleet state stops this run before any machine or the state is
+ * changed, if it does.
+ */
+function stateRefusal(
+  standing: Standing,
+  options: FleetSyncOptions,
+  run: Runner,
+  revision: string
+): CommandOutcome | null {
+  if (standing === 'diverged')
+    return actionRequired('FLEET_STATE_DIVERGED', DIVERGED_REMEDY, revision);
+  // One manager at a time: another one's lease stops this run.
+  const held = leaseConflict({
+    ...stateContext(run),
+    takeOver: options.takeOver === true,
+  });
+  return held
+    ? actionRequired(
+        'FLEET_MANAGER_LEASE_HELD',
+        leaseHeldMessage(held),
+        revision
+      )
+    : null;
+}
+
 /** fleet-sync on the machine already known to be the manager. */
 export async function managerSync(
   options: FleetSyncOptions,
@@ -177,8 +206,8 @@ export async function managerSync(
   // version of the configuration, the roster and the approval, not from a
   // stale local copy. Refreshing never uploads.
   const fresh = await refresh(stateContext(run));
-  if (fresh.standing === 'diverged')
-    return actionRequired('FLEET_STATE_DIVERGED', DIVERGED_REMEDY, revision);
+  const refused = stateRefusal(fresh.standing, options, run, revision);
+  if (refused) return refused;
   if (fresh.replaced) {
     // Everything read so far came from the stale copy, the manager's own
     // identity included.
@@ -274,7 +303,7 @@ function applyLocked(
 ): Promise<CommandOutcome> {
   const { settings, run, revision } = context;
   const record = runRecord(state, revision, {
-    context: stateContext(run),
+    context: { ...stateContext(run), takeOver: request.takeOver === true },
     standing: context.standing,
     ...(definitionSource().kind === 'file'
       ? { definition: JSON.stringify(canonicalDefinition, null, 2) + '\n' }

package/src/cli/github-verify.ts

@@ -0,0 +1,57 @@
+import type { Command } from 'commander';
+import { hostname } from 'node:os';
+import { FleetUsageError } from '../errors';
+import { fleet } from '../fleet-config';
+import { githubPolicy } from '../github/policy';
+import { verifyGithubWorkflow } from '../github/workflow';
+import { runCommand } from '../runner';
+import { hostId } from '../topology';
+import type { Runner } from '../types';
+import type { CommandOutcome } from './contracts';
+
+/** Runs the GitHub workflow an agent depends on from this machine against
+ * the declared sandbox repository, and cleans up after itself.
+ */
+export async function githubVerify(
+  run: Runner = runCommand,
+  node = hostId(hostname())
+): Promise<CommandOutcome> {
+  const sandbox = githubPolicy(fleet).sandbox;
+  if (!sandbox)
+    throw new FleetUsageError(
+      'No sandbox repository. Declare github.sandbox ("owner/name") in fleet.json: a repository the fleet login may branch, push to and open pull requests on.'
+    );
+  const result = await verifyGithubWorkflow({ run, sandbox, node });
+  const ok = result.check.status === 'pass';
+  return {
+    exitCode: ok ? 0 : 1,
+    data: {
+      version: 1,
+      command: 'github-verify',
+      ok,
+      node,
+      checks: [result.check],
+      ...(result.failed ? { failedStep: result.failed } : {}),
+      leftBehind: result.leftBehind,
+    },
+    text: `${result.check.id} ${result.check.status}: ${result.check.summary}`,
+  };
+}
+
+export function registerGithubVerify(
+  program: Command,
+  complete: (outcome: CommandOutcome) => void
+): void {
+  program
+    .command('fleet-github-verify')
+    .description(
+      'Prove this machine can clone, push, open a pull request, read checks and comment on GitHub'
+    )
+    .addHelpText(
+      'after',
+      '\nWorks on the sandbox repository named by github.sandbox in fleet.json: a uniquely named branch and a draft pull request, both closed and deleted again by the same run, whatever failed.'
+    )
+    .action(async () => {
+      complete(await githubVerify());
+    });
+}