@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/cli/fleet-sync-remote.ts

@@ -0,0 +1,242 @@
+import { verifyT3Linked } from '../t3-linked-check';
+import { T3_RESTARTED_SUMMARY } from '../platform/t3-policy';
+import { createHash } from 'node:crypto';
+import { verifyT3Endpoint } from '../t3-endpoint';
+import { authorizePrivilege } from './fleet-sync-privilege';
+import { authorizeVault } from './fleet-sync-vault';
+import { managerFailure } from '../manager-failure';
+import { managerText, stdoutColor } from './fleet-sync-output';
+import { syncProgress, type SyncProgress } from './fleet-sync-progress';
+import { saveReport } from './fleet-sync-report';
+import { synchronizeMonitoring } from './fleet-sync-monitoring';
+import { convergeManager } from './fleet-sync-manager';
+import { FleetUsageError } from '../errors';
+import { selectNodes } from '../config';
+import { loadRoster } from './roster';
+import { enrolledRoster } from '../definition';
+import type { FleetDefinition } from '../definition';
+import { fleet as definition } from '../fleet-config';
+import { synchronizeManager } from '../manager-sync';
+import { inspectBootstrap } from '../manager-sync';
+import { prepareBootstrap, bootstrapRemote } from '../remote-bootstrap';
+import { machineRoot } from './fleet-sync';
+import type { Check, Runner } from '../types';
+import type { DeployNode, ManagerReport } from '../manager-sync';
+import type { FleetSyncOptions } from './fleet-sync';
+import type { Settings } from './settings';
+import type { CommandOutcome } from './contracts';
+
+/** `settle` runs once the fleet's result is known and before it is reported:
+ * it records the result and returns the check that says so.
+ */
+export async function selected(
+  options: FleetSyncOptions,
+  settings: Settings,
+  run: Runner,
+  settle?: (report: ManagerReport) => Promise<Check>
+): Promise<CommandOutcome> {
+  if (!options.revision)
+    throw new FleetUsageError('An approved source revision is required.');
+  const canonical = definition;
+  let roster;
+  try {
+    roster = enrolledRoster(canonical, await loadRoster(settings.config));
+  } catch {
+    return enrollmentRequired(canonical, options.revision);
+  }
+  const nodes = selectNodes(roster, options.node ?? []);
+  const scoped = scopeDefinition(canonical, options.node);
+  const ui = interfaceOptions(options);
+  const planFingerprint = createHash('sha256')
+    .update(
+      JSON.stringify({ revision: options.revision, definition: scoped, nodes })
+    )
+    .digest('hex');
+  if (
+    options.expectedPlanFingerprint &&
+    options.expectedPlanFingerprint !== planFingerprint
+  )
+    return failed(
+      managerFailure(
+        scoped,
+        options.revision,
+        'FLEET_PLAN_STALE',
+        'Manager connection bindings or target policy changed after approval. Inspect a fresh plan and approve it; no source preparation or node mutation occurred.'
+      )
+    );
+  const payload = options.apply
+    ? await prepareBootstrap(machineRoot(), options.revision, run, {
+        definition: canonical,
+        checkout: settings.workspace,
+      })
+    : null;
+  const started = Date.now();
+  const report = await synchronizeManager(
+    {
+      definition: scoped,
+      nodes,
+      revision: options.revision,
+      apply: Boolean(options.apply),
+      ...canary(canonical, options),
+      progress: (node, phase) => ui.progress.step(node, phase),
+      finished: (node) => ui.progress.finished(node),
+      manager: () =>
+        convergeManager(
+          canonical,
+          {
+            revision: options.revision!,
+            apply: Boolean(options.apply),
+            workspace: settings.workspace,
+          },
+          run
+        ),
+    },
+    run,
+    deployWithMonitoring(payload, canonical, run, ui)
+  ).finally(() => ui.progress.stop());
+  if (settle) report.checks.push(await settle(report));
+  return outcome(report, planFingerprint, started);
+}
+
+/** A revision new to the fleet is proven on the first declared worker before
+ * the rest; repeating one that already converged everywhere needs no canary,
+ * so every worker converges at once.
+ */
+function canary(
+  canonical: FleetDefinition,
+  options: FleetSyncOptions
+): { canary?: string } {
+  const first = canonical.workers[0];
+  return first && !options.rolledOut ? { canary: first.id } : {};
+}
+
+/** Saves the full JSON report, then renders the short human summary. */
+async function outcome(
+  report: Awaited<ReturnType<typeof synchronizeManager>>,
+  planFingerprint: string,
+  started: number
+): Promise<CommandOutcome> {
+  const data = { ...report, planFingerprint };
+  const reportPath = await saveReport(machineRoot(), data);
+  return {
+    exitCode: report.ok ? 0 : 1,
+    data,
+    text: managerText(report, {
+      color: stdoutColor(),
+      durationMs: Date.now() - started,
+      ...(reportPath ? { reportPath } : {}),
+    }),
+  };
+}
+
+function enrollmentRequired(
+  canonical: FleetDefinition,
+  revision: string
+): CommandOutcome {
+  const report = managerFailure(
+    canonical,
+    revision,
+    'FLEET_MANAGER_ENROLLMENT_REQUIRED',
+    'Create or migrate private manager connection overrides for the canonical fleet. Resolve conflicting IDs/SSH aliases; discovery cannot authorize membership.'
+  );
+  return failed(report);
+}
+
+function failed(report: ReturnType<typeof managerFailure>): CommandOutcome {
+  return {
+    exitCode: 1,
+    data: report,
+    text: managerText(report, { color: stdoutColor() }),
+  };
+}
+
+function deployWithMonitoring(
+  payload: Awaited<ReturnType<typeof prepareBootstrap>>,
+  canonical: FleetDefinition,
+  run: Runner,
+  ui: {
+    progress: SyncProgress;
+    interactiveAuthorization: boolean;
+  }
+): DeployNode {
+  return async (node, revision) => {
+    if (!payload || payload.revision !== revision)
+      return {
+        node: node.id,
+        checks: [
+          {
+            id: 'source:bootstrap',
+            status: 'fail' as const,
+            summary: 'FLEET_APPROVED_BOOTSTRAP_UNAVAILABLE',
+            remedy: {
+              action:
+                'Install Git on the manager and verify the approved source contains the tracked remote bootstrap and matches this package’s canonical fleet definition. Use the matching package release after a membership change. No manager credentials are forwarded.',
+            },
+          },
+        ],
+      };
+    // Password prompts need a quiet terminal: no spinner while authorizing.
+    if (ui.interactiveAuthorization) {
+      ui.progress.stop();
+      const checks = await authorizePrivilege(node, payload.authorizeSudo, run);
+      if (checks.length) return { node: node.id, checks };
+    }
+    const vault = { interactive: ui.interactiveAuthorization };
+    if (ui.interactiveAuthorization) ui.progress.stop();
+    const before = await authorizeVault(node, vault, run);
+    ui.progress.step(node.id, 'Converging (remote lock, install, verify)');
+    const result = await bootstrapRemote(node, payload, run);
+    // A fresh node gets vault from the bootstrap; log it in now. Its provider
+    // checks in this run predate the login, so the next sync verifies them.
+    result.checks.push(
+      ...(before.length ? before : await authorizeVault(node, vault, run))
+    );
+    // Workers are Linux by definition.
+    ui.progress.step(node.id, 'Verifying SSH, OS and T3');
+    result.checks.push(...(await inspectBootstrap(node, run, 'linux')));
+    result.checks.push(...(await verifyT3Endpoint(node, run)));
+    // A worker T3 restarted by this sync (idle policy heal) needs a minute or
+    // two before the manager's link reconnects.
+    const restarted = result.checks.some(
+      (check) => check.summary === T3_RESTARTED_SUMMARY
+    );
+    result.checks.push(
+      await verifyT3Linked(node, run, {
+        reconnectMs: restarted ? 180_000 : 0,
+      })
+    );
+    if (
+      result.checks.some(
+        (check) => check.id === 'configuration:apply' && check.status === 'pass'
+      )
+    ) {
+      result.checks = result.checks.filter(
+        (check) => check.id !== 'beszel:freshness'
+      );
+      ui.progress.step(node.id, 'Enrolling monitoring');
+      result.checks.push(
+        ...(await synchronizeMonitoring(node, canonical, revision, run))
+      );
+    }
+    return result;
+  };
+}
+
+function scopeDefinition(canonical: FleetDefinition, ids?: string[]) {
+  return ids?.length
+    ? {
+        ...canonical,
+        workers: canonical.workers.filter((node) => ids.includes(node.id)),
+      }
+    : canonical;
+}
+function interfaceOptions(options: FleetSyncOptions) {
+  return {
+    progress: syncProgress(!options.json),
+    interactiveAuthorization: Boolean(
+      options.interactiveAuthorization &&
+      !options.json &&
+      !options.nonInteractive
+    ),
+  };
+}

package/src/cli/fleet-sync-report.ts

@@ -0,0 +1,26 @@
+import { chmod, mkdir, rename, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+/** Saves the full JSON report of the latest run (0600, manager-private) so an
+ * agent can debug without rerunning. Reports carry codes and remedies only,
+ * never secret values. Returns null when it cannot be written.
+ */
+export async function saveReport(
+  root: string,
+  data: unknown
+): Promise<string | null> {
+  try {
+    const directory = join(root, 'reports');
+    await mkdir(directory, { recursive: true, mode: 0o700 });
+    const path = join(directory, 'last-sync.json');
+    const partial = `${path}.partial`;
+    await writeFile(partial, `${JSON.stringify(data, null, 2)}\n`, {
+      mode: 0o600,
+    });
+    await chmod(partial, 0o600);
+    await rename(partial, path);
+    return path;
+  } catch {
+    return null;
+  }
+}

package/src/cli/fleet-sync-schedule.ts

@@ -0,0 +1,138 @@
+import { join } from 'node:path';
+import { currentPlatform } from '../platform';
+import type { Platform } from '../platform/types';
+import type { Runner } from '../types';
+import type { CommandOutcome } from './contracts';
+import { machineRoot } from './fleet-sync';
+
+const LABEL = 'personal-fleet-sync';
+declare const PERSONAL_VERSION: string | undefined;
+declare const PERSONAL_SOURCE_REVISION: string | null;
+export type ScheduleAction =
+  'enable' | 'pause' | 'resume' | 'disable' | 'status';
+interface ScheduleInput {
+  action: ScheduleAction;
+  revision: string;
+  config: string;
+  authorized: boolean;
+}
+export async function managerSchedule(
+  input: ScheduleInput,
+  run: Runner,
+  platform: Platform = currentPlatform()
+): Promise<CommandOutcome> {
+  if (input.action === 'status')
+    return {
+      exitCode: 0,
+      data: {
+        version: 2,
+        command: 'fleet-sync',
+        execution: 'manager-push',
+        scheduling: scheduleStatus(platform),
+      },
+    };
+  if (!input.authorized)
+    return required('Manager scheduling changes require --apply --yes.');
+  if (input.action === 'pause' || input.action === 'disable')
+    return removeSchedule(input, platform);
+  return enableSchedule(input, run, platform);
+}
+function removeSchedule(
+  input: ScheduleInput,
+  platform: Platform
+): CommandOutcome {
+  const result = platform.removeScheduledJob(LABEL);
+  return {
+    exitCode: result.ok ? 0 : 1,
+    data: {
+      version: 2,
+      command: 'fleet-sync',
+      ok: result.ok,
+      scheduling: input.action,
+      summary: result.ok
+        ? 'Manager schedule removed; node application services are untouched.'
+        : 'Manager scheduler removal failed; inspect its platform status.',
+    },
+  };
+}
+async function enableSchedule(
+  input: ScheduleInput,
+  run: Runner,
+  platform: Platform
+): Promise<CommandOutcome> {
+  if (
+    typeof PERSONAL_VERSION !== 'string' ||
+    !/^\d+\.\d+\.\d+$/.test(PERSONAL_VERSION) ||
+    typeof PERSONAL_SOURCE_REVISION !== 'string'
+  )
+    return required(
+      'Scheduling requires a published CLI with clean source provenance; dirty review artifacts cannot enable unattended deployment.'
+    );
+  const root = machineRoot();
+  const prefix = join(root, 'runtime');
+  const installed = await run(
+    [
+      'npm',
+      'install',
+      '--prefix',
+      prefix,
+      '--ignore-scripts',
+      '--no-audit',
+      '--no-fund',
+      `dev-fleet@${PERSONAL_VERSION}`,
+    ],
+    { timeoutMs: 120_000 }
+  );
+  if (!installed.ok)
+    return required(
+      'The exact CLI package could not be installed in the managed manager runtime. No schedule was enabled.'
+    );
+  const result = platform.installScheduledJob({
+    label: LABEL,
+    intervalSeconds: 1800,
+    command: [
+      process.execPath,
+      join(prefix, 'node_modules/dev-fleet/cli.mjs'),
+      'fleet-sync',
+      '--config',
+      input.config,
+      '--revision',
+      input.revision,
+      '--apply',
+      '--yes',
+      '--non-interactive',
+      '--json',
+    ],
+    logDir: join(root, 'logs'),
+    env: { PATH: process.env['PATH'] ?? '/usr/bin:/bin' },
+  });
+  return {
+    exitCode: result.ok ? 0 : 1,
+    data: {
+      version: 2,
+      command: 'fleet-sync',
+      ok: result.ok,
+      scheduling: result.ok ? 'enabled' : 'action-required',
+      revision: input.revision,
+      summary:
+        'Manager schedule uses one exact package and approved revision. Linux user scheduling requires lingering; macOS LaunchAgent requires the manager user session. Inspect status and logs; boot acceptance is separate.',
+    },
+  };
+}
+function required(summary: string): CommandOutcome {
+  return {
+    exitCode: 1,
+    data: {
+      version: 2,
+      command: 'fleet-sync',
+      ok: false,
+      scheduling: 'action-required',
+      summary,
+    },
+  };
+}
+
+function scheduleStatus(platform: Platform) {
+  const { installed, supported } = platform.scheduledJobStatus(LABEL);
+  return { installed, supported };
+}

package/src/cli/fleet-sync-state.ts

@@ -0,0 +1,86 @@
+import type { Enrollment } from '../machine/state';
+import { replicaCheck } from '../state/check';
+import { storeEnrollment } from '../state/enrollment';
+import { adoptEarlier } from '../state/legacy';
+import { databasePath, stateRoot } from '../state/paths';
+import { writeDocument } from '../state/store';
+import { pending, push, type Standing, type StateContext } from '../state/sync';
+import type { Check } from '../types';
+
+/** What one fleet-sync run records in the fleet state: the revision it was
+ * approved for and the configuration it runs with, then what it applied, then
+ * an upload so the bucket holds all of it.
+ */
+export interface RunRecord {
+  /** Before any machine changes: this revision is the approved one. */
+  approve(): void;
+  /** Once the result is known. The check says whether the bucket holds it. */
+  settle(converged: boolean): Promise<Check>;
+  /** For a run that stopped before its result: its approval is still uploaded. */
+  abandon(): Promise<void>;
+}
+
+export function runRecord(
+  previous: Enrollment | null,
+  revision: string,
+  state: {
+    context: StateContext;
+    standing: Standing;
+    env?: NodeJS.ProcessEnv;
+    /** The configuration this run reads from a project's fleet.json. Stored
+     * so that a run started anywhere else (the schedule, a hand-off from a
+     * worker, a replacement manager) uses the one last applied.
+     */
+    definition?: string;
+  }
+): RunRecord {
+  const env = state.env ?? process.env;
+  const approval: Enrollment = {
+    ...previous,
+    version: 1,
+    id: 'manager',
+    approvedRevision: revision,
+  };
+  let settled = false;
+  /** Uploads only what this run changed; otherwise the bucket stands as it
+   * did when the run started.
+   */
+  const publish = (): Promise<Standing> =>
+    pending(state.context)
+      ? push(state.context)
+      : Promise.resolve(state.standing);
+  return {
+    approve: () => {
+      storeEnrollment(approval, env);
+      if (state.definition === undefined) return;
+      adoptEarlier(env);
+      writeDocument(
+        databasePath(stateRoot(env)),
+        'definition',
+        state.definition
+      );
+    },
+    async settle(converged) {
+      settled = true;
+      if (converged)
+        storeEnrollment(
+          {
+            ...approval,
+            appliedRevision: revision,
+            ...(previous?.appliedRevision &&
+            previous.appliedRevision !== revision
+              ? { previousRevision: previous.appliedRevision }
+              : {}),
+          },
+          env
+        );
+      // The fleet converged either way: a bucket problem is reported, not fatal.
+      return replicaCheck(await publish().catch(() => 'unreachable' as const), {
+        advisory: true,
+      });
+    },
+    async abandon() {
+      if (!settled) await publish().catch(() => undefined);
+    },
+  };
+}

package/src/cli/fleet-sync-vault.ts

@@ -0,0 +1,166 @@
+import { spawnSync } from 'node:child_process';
+import { fleet as definition } from '../fleet-config';
+import { sshArgs, shellQuote } from '../ssh';
+import { resolveVaultConfig, usesVault } from '../vault-config';
+import type { Check, FleetNode, Runner } from '../types';
+
+const TOOL_PATH =
+  'export PATH="$HOME/.local/share/mise/shims:$HOME/.local/bin:$PATH"';
+
+/** Runs an interactive command with the operator's terminal attached. */
+export type TerminalRunner = (args: string[]) => boolean;
+
+const terminal: TerminalRunner = (args) =>
+  spawnSync(args[0]!, args.slice(1), { stdio: 'inherit' }).status === 0;
+
+type VaultState = 'authenticated' | 'unauthenticated' | 'unavailable';
+
+/** Vault field holding the issued token for one worker node. */
+export function workerTokenField(node: string): string {
+  return `FLEET_WORKER_TOKEN_${node.toUpperCase().replace(/[^A-Z0-9]/g, '_')}`;
+}
+
+/**
+ * The manager reads each node's own token from Vault, so it must be logged in
+ * first. Interactive runs prompt once on this terminal; the password never
+ * passes through this process.
+ */
+export async function authorizeManager(
+  options: { interactive: boolean },
+  run: Runner,
+  attach: TerminalRunner = terminal
+): Promise<boolean> {
+  if (!usesVault()) return true;
+  if ((await run(lookup(), { timeoutMs: 20_000 })).ok) return true;
+  if (!options.interactive) return false;
+  process.stderr.write(
+    'This manager is not logged in to Vault. Log in once; each machine then gets its own token.\n'
+  );
+  attach(login());
+  return (await run(lookup(), { timeoutMs: 20_000 })).ok;
+}
+
+/**
+ * Each node keeps its own periodic Vault token (renewed by the worker job). The
+ * proxy key, GitHub token and provider credentials all come from that session,
+ * so a node without one has no working proxy providers. The node's
+ * issued token is delivered once over SSH stdin, never as an argument;
+ * interactive runs fall back to a password login on the node's own terminal.
+ */
+export async function authorizeVault(
+  node: FleetNode,
+  options: { interactive: boolean },
+  run: Runner,
+  attach: TerminalRunner = terminal
+): Promise<Check[]> {
+  // The other secrets providers need no per-node session.
+  if (!usesVault()) return [];
+  const before = await vaultState(node, run);
+  if (before === 'authenticated') return [pass(node)];
+  // Fresh node: the locked toolchain (including vault) is not installed yet.
+  if (before === 'unavailable') return [];
+  if (await deliverWorkerToken(node, run)) return [pass(node)];
+  if (!options.interactive) return [required(node)];
+  process.stderr.write(
+    `[${node.id}] No issued worker token found. Enter the Vault password to log this machine in.\n`
+  );
+  attach(remote(node, login(), '-t'));
+  return [
+    (await vaultState(node, run)) === 'authenticated'
+      ? pass(node)
+      : required(node),
+  ];
+}
+
+async function deliverWorkerToken(
+  node: FleetNode,
+  run: Runner
+): Promise<boolean> {
+  const { addr, mount, project, config } = resolveVaultConfig();
+  const secret = await run(
+    [
+      'vault',
+      'kv',
+      'get',
+      `-address=${addr}`,
+      `-mount=${mount}`,
+      `-field=${workerTokenField(node.id)}`,
+      `${project}/${config}`,
+    ],
+    { timeoutMs: 20_000 }
+  );
+  const token = secret.ok ? secret.stdout.trim() : '';
+  if (!/^[\x21-\x7e]{16,4096}$/.test(token)) return false;
+  const delivered = await run(
+    remote(node, ['vault', 'login', `-address=${addr}`, '-no-print', '-']),
+    { input: token + '\n', timeoutMs: 30_000 }
+  );
+  return delivered.ok && (await vaultState(node, run)) === 'authenticated';
+}
+
+async function vaultState(node: FleetNode, run: Runner): Promise<VaultState> {
+  const result = await run(
+    [
+      ...sshArgs(node),
+      `${TOOL_PATH}; command -v vault >/dev/null 2>&1 || { printf vault-missing; exit 1; }; ${quote(lookup())} >/dev/null 2>&1`,
+    ],
+    { timeoutMs: 20_000 }
+  );
+  if (result.ok) return 'authenticated';
+  // Unreachable nodes are reported by the bootstrap's own SSH checks.
+  if (result.reason !== 'exit') return 'unavailable';
+  return result.stdout.includes('vault-missing')
+    ? 'unavailable'
+    : 'unauthenticated';
+}
+
+function lookup(): string[] {
+  const { addr } = resolveVaultConfig();
+  return ['vault', 'token', 'lookup', `-address=${addr}`, '-format=json'];
+}
+
+function login(): string[] {
+  const { addr } = resolveVaultConfig();
+  return [
+    'vault',
+    'login',
+    `-address=${addr}`,
+    '-method=userpass',
+    '-no-print',
+    `username=${definition.worker!.vaultUser}`,
+  ];
+}
+
+/** Same verified connection options; `-t` attaches a terminal for prompts. */
+export function remote(
+  node: FleetNode,
+  command: string[],
+  tty: '-T' | '-t' = '-T'
+): string[] {
+  const args = sshArgs(node).map((arg) => (arg === '-T' ? tty : arg));
+  return [...args, `${TOOL_PATH}; ${quote(command)}`];
+}
+
+function quote(command: string[]): string {
+  return command.map(shellQuote).join(' ');
+}
+
+function pass(node: FleetNode): Check {
+  return {
+    id: 'auth:vault',
+    status: 'pass',
+    summary: `${node.id} has its own Vault session; proxy providers can read their key.`,
+  };
+}
+
+function required(node: FleetNode): Check {
+  const { addr } = resolveVaultConfig();
+  return {
+    id: 'auth:vault',
+    status: 'fail',
+    summary: 'VAULT_LOGIN_REQUIRED',
+    remedy: {
+      action: `Run devfleet fleet-sync in a terminal, or store this node's token in the shared Vault document as ${workerTokenField(node.id)}. Manual fallback on ${node.id}: vault login -address=${addr} -method=userpass username=${definition.worker!.vaultUser}. Proxy providers in T3 stay unavailable until then.`,
+    },
+  };
+}