@crvouga/dev-fleet 0.0.0-stage → 0.1.0

package/src/cli/contracts.ts

@@ -0,0 +1,25 @@
+export interface CommandSpec {
+  id: string;
+  title: string;
+  description: string;
+  group: string;
+  keywords: string[];
+  args: string[];
+  examples: string[];
+  mutation: boolean;
+}
+
+export interface CommandOutcome {
+  exitCode: number;
+  data: unknown;
+  text?: string;
+}
+
+export interface InteractiveOptions {
+  version: string;
+  releasedAt: string | null;
+  workspace: string;
+  config: string;
+  commands: CommandSpec[];
+  execute: (args: string[]) => Promise<CommandOutcome>;
+}

package/src/cli/fleet-reference.ts

@@ -0,0 +1,17 @@
+import type { Fingerprint, Snapshot } from '../types';
+
+declare const PERSONAL_FLEET_REFERENCE: Fingerprint | null | undefined;
+/** Published health carries safe input hashes; it needs no manager checkout or Bun. */
+export function fleetReference(): Snapshot | undefined {
+  if (
+    typeof PERSONAL_FLEET_REFERENCE === 'undefined' ||
+    !PERSONAL_FLEET_REFERENCE?.head
+  )
+    return undefined;
+  return {
+    version: 1,
+    platform: 'declared-worker-policy',
+    fingerprint: PERSONAL_FLEET_REFERENCE,
+    checks: [],
+  };
+}

package/src/cli/fleet-sync-checkout.ts

@@ -0,0 +1,91 @@
+import { spawn } from 'node:child_process';
+import { trackedScript } from '../remote-bootstrap';
+import type { Runner } from '../types';
+import type { CommandOutcome } from './contracts';
+import type { FleetSyncOptions } from './fleet-sync';
+import { forwardedFlags, type ForwardSpawn } from './fleet-sync-forward';
+
+/** Exported by `fleet-manager-run.sh`: this process already runs from the
+ * fleet-owned checkout of that revision.
+ */
+export const RUN_CHECKOUT_ENV = 'PERSONAL_FLEET_RUN_CHECKOUT';
+
+export const inRunCheckout = (revision: string): boolean =>
+  process.env[RUN_CHECKOUT_ENV] === revision;
+
+/** The manager converges from a clean, fleet-owned checkout of the exact
+ * revision, like every worker, never from the checkout fleet-sync was started
+ * in: that one is a working tree whose uncommitted edits, branch switches or
+ * removal would otherwise decide whether a sync (and every provider it
+ * installs) works. Null when the revision has no tracked run script.
+ */
+export async function runFromCheckout(
+  options: FleetSyncOptions,
+  source: { revision: string; workspace: string },
+  run: Runner,
+  spawnLocal: ForwardSpawn = spawnInherited
+): Promise<CommandOutcome | null> {
+  const { revision } = source;
+  const script = await trackedScript(
+    source.workspace,
+    revision,
+    'bootstrap/fleet-manager-run.sh',
+    run
+  );
+  if (!script) return null;
+  // The run script takes the revision from this clone.
+  process.env['DEVFLEET_REPO'] ||= source.workspace;
+  const result = await spawnLocal([
+    'sh',
+    '-c',
+    script,
+    'fleet-manager-run',
+    revision,
+    ...forwardedFlags(options, undefined),
+  ]);
+  const stdout = result.stdout.trim();
+  if (options.json) {
+    try {
+      return { exitCode: result.exitCode, data: JSON.parse(stdout) as unknown };
+    } catch {
+      /* Fall through: the run checkout could not start; its reason is on stderr. */
+    }
+  }
+  return {
+    exitCode: result.exitCode,
+    data: { version: 1, revision, exitCode: result.exitCode },
+    text: stdout,
+  };
+}
+
+/** Null when this process is already the run checkout (or a test, which
+ * never leaves its own process: the same rule as the worker hand-off).
+ */
+export async function rehome(
+  options: FleetSyncOptions,
+  source: { revision: string; workspace: string },
+  run: Runner
+): Promise<CommandOutcome | null> {
+  if (inRunCheckout(source.revision) || process.env['NODE_ENV'] === 'test')
+    return null;
+  return runFromCheckout(options, source, run);
+}
+
+/** Prompts and progress stay on the terminal; the report is returned. */
+function spawnInherited(
+  command: string[]
+): Promise<{ exitCode: number; stdout: string }> {
+  return new Promise((resolve) => {
+    const color = process.stdout.isTTY && !process.env['NO_COLOR'];
+    const child = spawn(command[0]!, command.slice(1), {
+      stdio: ['inherit', 'pipe', 'inherit'],
+      env: { ...process.env, ...(color ? { FORCE_COLOR: '1' } : {}) },
+    });
+    let stdout = '';
+    child.stdout.setEncoding('utf8').on('data', (chunk: string) => {
+      stdout += chunk;
+    });
+    child.on('error', () => resolve({ exitCode: 1, stdout: '' }));
+    child.on('close', (code) => resolve({ exitCode: code ?? 1, stdout }));
+  });
+}

package/src/cli/fleet-sync-enrollment.ts

@@ -0,0 +1,144 @@
+import { readFile, mkdir, writeFile, rename, rm } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import { loadConfig } from '../config';
+import { enrolledRoster } from '../definition';
+import { FleetUsageError } from '../errors';
+import { dirname } from 'node:path';
+import { fleet as definition } from '../fleet-config';
+import { discoverTailnet } from '../tailscale';
+import { matchComputers } from '../tailnet-matching';
+import { sshArgs, sshFailure } from '../ssh';
+import { ensurePrivateRoot } from '../machine/state';
+import type { FleetNode, Runner } from '../types';
+
+/** Interactive enrollment/resume only. Existing paths and explicit identities
+ * are preserved. Tailnet candidates never enter the declared membership.
+ */
+export async function enrollManager(path: string, run: Runner): Promise<void> {
+  const canonical = definition;
+  const original = await existingEnrollment(path);
+  const saved =
+    original === null
+      ? []
+      : enrolledRoster(canonical, await loadConfig(path)).nodes;
+  const nodes: FleetNode[] = canonical.workers.map((node) => ({
+    id: node.id,
+    ssh: node.ssh,
+    workspace: '/',
+    required: ['t3', 'toolchain', 'workstation'],
+  }));
+  const pending = nodes.filter(
+    (node) => !saved.find((entry) => entry.id === node.id)?.tailscaleId
+  );
+  if (!pending.length) return;
+  const tailnet = await discoverTailnet(run);
+  const matches = await matchComputers(nodes, tailnet.computers, run);
+  const enrolled: FleetNode[] = [];
+  for (const node of pending) {
+    const peers = tailnet.computers.filter((peer) =>
+      matches.get(peer.id)?.some((candidate) => candidate.id === node.id)
+    );
+    if (
+      ambiguous(
+        peers.map((peer) => peer.id),
+        matches
+      )
+    )
+      continue;
+    const connection = await inspectConnection(node, peers[0]?.id, run);
+    if (connection) enrolled.push(connection);
+  }
+  const merged = mergeEnrollment(saved, enrolled);
+  if (JSON.stringify(merged) === JSON.stringify(saved)) return;
+  // Only canonical nodes reached over host-key-verified SSH get here, so
+  // enrollment needs no extra confirmation; the list is still shown.
+  process.stderr.write(
+    `Enrolling verified SSH connections for canonical targets:\n${enrolled.map((node) => `  ${node.id} → ${node.ssh}; device ${enrollmentIdentity(node)}`).join('\n')}\nMissing targets remain visible and are never replaced by discovered peers.\n`
+  );
+  await ensurePrivateRoot(dirname(path));
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+  await persistEnrollment(path, original, merged);
+}
+
+async function inspectConnection(
+  node: FleetNode,
+  peerId: string | undefined,
+  run: Runner
+): Promise<FleetNode | null> {
+  const home = await run([...sshArgs(node), 'printf "%s" "$HOME"'], {
+    timeoutMs: 20_000,
+  });
+  if (!home.ok) {
+    const check = sshFailure(home);
+    process.stderr.write(
+      `[${node.id}] ${check.id}: ${check.summary} ${check.remedy?.action ?? ''}\n`
+    );
+    return null;
+  }
+  if (!/^\/[^\x00-\x1f\x7f]{1,4000}$/.test(home.stdout)) return null;
+  return {
+    ...node,
+    workspace: `${home.stdout}/.local/share/personal/fleet`,
+    ...(peerId ? { tailscaleId: peerId } : { bootstrapViaSsh: true }),
+  };
+}
+
+function enrollmentIdentity(node: FleetNode): string {
+  return (
+    node.tailscaleId ??
+    'initial trusted SSH bootstrap; native Tailscale login required'
+  );
+}
+
+function ambiguous(ids: string[], matches: Map<string, FleetNode[]>): boolean {
+  return (
+    ids.length > 1 || (ids.length === 1 && matches.get(ids[0]!)?.length !== 1)
+  );
+}
+async function existingEnrollment(path: string): Promise<string | null> {
+  try {
+    return await readFile(path, 'utf8');
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
+    throw error;
+  }
+}
+export function mergeEnrollment(
+  saved: FleetNode[],
+  discovered: FleetNode[]
+): FleetNode[] {
+  const result = saved.map((node) => ({ ...node }));
+  for (const candidate of discovered) {
+    const existing = result.find((node) => node.id === candidate.id);
+    if (!existing) result.push(candidate);
+    else if (!existing.tailscaleId && candidate.tailscaleId) {
+      existing.tailscaleId = candidate.tailscaleId;
+      delete existing.bootstrapViaSsh;
+    }
+  }
+  return result;
+}
+async function persistEnrollment(
+  path: string,
+  original: string | null,
+  nodes: FleetNode[]
+): Promise<void> {
+  const lock = `${path}.enroll.lock`;
+  await mkdir(lock, { mode: 0o700 });
+  const temporary = `${path}.${randomUUID()}.tmp`;
+  try {
+    if ((await existingEnrollment(path)) !== original)
+      throw new FleetUsageError(
+        'Enrollment changed during approval. Inspect it and rerun; no connection was overwritten.'
+      );
+    await writeFile(
+      temporary,
+      JSON.stringify({ version: 1, nodes }, null, 2) + '\n',
+      { flag: 'wx', mode: 0o600 }
+    );
+    await rename(temporary, path);
+  } finally {
+    await rm(temporary, { force: true });
+    await rm(lock, { recursive: true });
+  }
+}

package/src/cli/fleet-sync-forward.ts

@@ -0,0 +1,134 @@
+import { spawn } from 'node:child_process';
+import type { FleetDefinition } from '../definition';
+import { shellQuote, sshArgs } from '../ssh';
+import type { FleetNode } from '../types';
+import type { CommandOutcome } from './contracts';
+import type { FleetSyncOptions } from './fleet-sync';
+
+/** Set on the manager side of a hand-off (and in tests): never hand off again. */
+export const NO_FORWARD_ENV = 'PERSONAL_FLEET_NO_FORWARD';
+
+/** Streams the remote command: progress on stderr, the report on stdout. */
+export type ForwardSpawn = (
+  command: string[]
+) => Promise<{ exitCode: number; stdout: string }>;
+
+export function forwardingDisabled(): boolean {
+  return Boolean(process.env[NO_FORWARD_ENV]);
+}
+
+/** fleet-sync only ever runs on the manager. Started on a worker, it hands the
+ * same request to the manager over host-key-verified SSH (no agent forwarding,
+ * no credentials copied) and reports the manager's result unchanged.
+ */
+export async function forwardToManager(
+  definition: FleetDefinition,
+  options: FleetSyncOptions,
+  revision: string | undefined,
+  remote: { spawn?: ForwardSpawn | undefined; script?: string | null } = {}
+): Promise<CommandOutcome> {
+  // A test must inject the spawn: it can never reach the real manager.
+  if (!remote.spawn && process.env['NODE_ENV'] === 'test')
+    throw new Error('FLEET_FORWARD_IN_TEST');
+  const spawnRemote = remote.spawn ?? spawnInherited;
+  const runScript = remote.script ?? null;
+  const { manager } = definition;
+  process.stderr.write(
+    `fleet-sync runs on the manager; handing off to ${manager.id} over SSH…\n`
+  );
+  const result = await spawnRemote(
+    managerCommand(definition, options, revision, runScript)
+  );
+  const stdout = result.stdout.trim();
+  if (options.json) {
+    try {
+      return { exitCode: result.exitCode, data: JSON.parse(stdout) as unknown };
+    } catch {
+      /* Fall through to the structured failure below. */
+    }
+  }
+  if (result.exitCode === 255 && !stdout)
+    return {
+      exitCode: 1,
+      data: {
+        version: 1,
+        ok: false,
+        code: 'FLEET_MANAGER_UNREACHABLE',
+        error: `Could not reach the manager (${manager.id}) over SSH. Wake it, check Tailscale and SSH (Remote Login), or run fleet-sync on the manager directly.`,
+      },
+    };
+  return {
+    exitCode: result.exitCode,
+    data: { version: 1, forwardedTo: manager.id, exitCode: result.exitCode },
+    text: stdout,
+  };
+}
+
+/** With the revision's tracked `fleet-manager-run.sh`, the manager runs
+ * fleet-sync from a clean checkout of that exact revision; otherwise (no
+ * revision, or an older one without the script) its installed launcher.
+ */
+export function managerCommand(
+  definition: FleetDefinition,
+  options: FleetSyncOptions,
+  revision: string | undefined,
+  runScript: string | null = null
+): string[] {
+  const { manager } = definition;
+  const target: FleetNode = {
+    id: manager.id,
+    ssh: `${manager.network.user}@${manager.network.tailscaleName}`,
+    workspace: '~',
+    required: [],
+  };
+  const args =
+    runScript && revision
+      ? [
+          'sh',
+          '-c',
+          runScript,
+          'fleet-manager-run',
+          revision,
+          ...forwardedFlags(options, undefined),
+        ]
+      : ['personal', 'fleet-sync', ...forwardedFlags(options, revision)];
+  const color = process.stdout.isTTY && !process.env['NO_COLOR'];
+  // A login shell gives the manager's normal PATH (mise, Docker, Tailscale).
+  const script = `export ${NO_FORWARD_ENV}=1${color ? ' FORCE_COLOR=1' : ''}; exec ${args.map(shellQuote).join(' ')}`;
+  return [...sshArgs(target), `exec /bin/zsh -lc ${shellQuote(script)}`];
+}
+
+export function forwardedFlags(
+  options: FleetSyncOptions,
+  revision: string | undefined
+): string[] {
+  const flags = revision ? ['--revision', revision] : [];
+  for (const node of options.node ?? []) flags.push('--node', node);
+  if (options.schedule) flags.push('--schedule', options.schedule);
+  const switches: [boolean | undefined, string][] = [
+    [options.all, '--all'],
+    [options.dryRun, '--dry-run'],
+    [options.apply, '--apply'],
+    [options.yes, '--yes'],
+    [options.json, '--json'],
+    [options.nonInteractive, '--non-interactive'],
+  ];
+  for (const [on, flag] of switches) if (on) flags.push(flag);
+  return flags;
+}
+
+function spawnInherited(
+  command: string[]
+): Promise<{ exitCode: number; stdout: string }> {
+  return new Promise((resolve) => {
+    const child = spawn(command[0]!, command.slice(1), {
+      stdio: ['ignore', 'pipe', 'inherit'],
+    });
+    let stdout = '';
+    child.stdout.setEncoding('utf8').on('data', (chunk: string) => {
+      stdout += chunk;
+    });
+    child.on('error', () => resolve({ exitCode: 255, stdout: '' }));
+    child.on('close', (code) => resolve({ exitCode: code ?? 1, stdout }));
+  });
+}

package/src/cli/fleet-sync-manager.ts

@@ -0,0 +1,193 @@
+import { blocking, type Check, type Runner } from '../types';
+import type { FleetDefinition } from '../definition';
+import type { ManagerNode } from '../manager-sync';
+import { hostId } from '../topology';
+import { convergeProxy } from '../proxy-converge';
+import { attempt } from '../node-converge-shared';
+import { quotaLimited, verifyFleetProviders } from '../agents/fleet-verify';
+import { verifyNativeAgents } from '../agents/native-verify';
+import { convergeManagerHub } from '../monitoring/manager-hub';
+import { enrollManagerMonitoring } from './fleet-sync-monitoring';
+import { installConfig } from '../fleet-config';
+import { machineProviders } from '../t3/providers';
+import { providerNames } from '../t3/names-check';
+
+/** The manager converges itself locally (never over SSH) before any worker:
+ * it must be the declared manager, stay awake on AC power, and host every
+ * singleton service (CLI proxy, Beszel hub). Workers only consume them.
+ */
+export async function convergeManager(
+  definition: FleetDefinition,
+  input: {
+    revision: string;
+    apply: boolean;
+    workspace: string;
+  },
+  run: Runner
+): Promise<ManagerNode> {
+  const started = Date.now();
+  const id = definition.manager.id;
+  const identity = await managerIdentity(id, run);
+  const checks: Check[] = [identity];
+  if (identity.status === 'pass') {
+    // Launchers and scheduled jobs on the manager read the installed copy.
+    if (input.apply) installConfig();
+    checks.push(await managerPower(run));
+    checks.push(await machineProviders(id, input.workspace, input.apply));
+    checks.push(await providerNames(id, input.workspace, input.apply));
+    if (input.apply) {
+      // Same provider profiles and the same Vault gateway credential as every
+      // worker: the instances above launch from these profiles, and the check
+      // proves the credential works end to end through the shared proxy.
+      checks.push(
+        ...(await attempt('configuration:apply', () =>
+          managerAgentProfiles(id, input.workspace, run)
+        ))
+      );
+      const providers = await attempt('provider:verification', () =>
+        verifyFleetProviders(id)
+      );
+      checks.push(...providers);
+      // Same agents, versions and shared accounts as every worker.
+      checks.push(
+        ...(await attempt('agent:verification', () =>
+          verifyNativeAgents(id, input.workspace, run, {
+            outOfQuota: quotaLimited(providers),
+          })
+        ))
+      );
+    }
+    checks.push(
+      ...(await convergeProxy(id, input.revision, run, {
+        apply: input.apply,
+        cwd: input.workspace,
+      }))
+    );
+    if (input.apply) {
+      const hub = await convergeManagerHub(input.workspace, definition, run);
+      checks.push(...hub);
+      // The manager is always a monitored system on its own hub.
+      if (!hub.some(blocking))
+        checks.push(
+          ...(await enrollManagerMonitoring(definition, input.workspace, run))
+        );
+    }
+  }
+  return {
+    node: id,
+    role: 'manager',
+    checks,
+    reachability: 'online',
+    desiredRevision: input.revision,
+    appliedRevision:
+      input.apply && !checks.some(blocking) ? input.revision : null,
+    durationMs: Date.now() - started,
+  };
+}
+
+const PROFILE_TOOLS = [
+  'codex',
+  'claude',
+  'gemini',
+  'opencode',
+  'grok',
+  'antigravity',
+] as const;
+
+/** Applies the proxy agent profiles the managed T3 instances launch from
+ * (codex-proxy.ts refuses to start with a missing or stale profile). Workers
+ * apply the same profiles from a pinned deployment; the manager's checkout is
+ * the human's working tree, so only the workstation sources must be clean
+ * (the `agents apply` owner enforces that) and nothing else is touched.
+ */
+export async function managerAgentProfiles(
+  id: string,
+  workspace: string,
+  run: Runner
+): Promise<Check[]> {
+  for (const tool of PROFILE_TOOLS) {
+    const result = await run(
+      [
+        'mise',
+        'exec',
+        '--locked',
+        'bun',
+        '--',
+        'bun',
+        'src/agents/cli.ts',
+        '--json',
+        '--yes',
+        '--non-interactive',
+        'agents',
+        'apply',
+        '--node',
+        id,
+        '--tool',
+        tool,
+        '--profile',
+        'proxy',
+      ],
+      { cwd: workspace, timeoutMs: 120_000 }
+    );
+    if (!result.ok)
+      return [
+        {
+          id: 'configuration:apply',
+          status: 'fail',
+          summary: 'MANAGER_AGENT_PROFILES_FAILED',
+          remedy: {
+            action: `Run bun src/agents/cli.ts --yes agents apply --node ${id} --tool ${tool} --profile proxy in the dev-fleet checkout on the manager and review the result.`,
+          },
+        },
+      ];
+  }
+  return [
+    {
+      id: 'configuration:apply',
+      status: 'pass',
+      summary: `The manager has the same proxy agent profiles as every worker (${PROFILE_TOOLS.join(', ')}).`,
+    },
+  ];
+}
+
+/** fleet-sync runs only on the declared manager. */
+export async function managerIdentity(id: string, run: Runner): Promise<Check> {
+  const result = await run(['scutil', '--get', 'LocalHostName']);
+  const local = result.ok ? hostId(result.stdout.trim()) : '';
+  return local === id
+    ? {
+        id: 'manager:identity',
+        status: 'pass',
+        summary: `This machine is the declared fleet manager (${id}).`,
+      }
+    : {
+        id: 'manager:identity',
+        status: 'fail',
+        summary: 'FLEET_NOT_MANAGER',
+        remedy: {
+          action: `Run fleet-sync on the manager (${id}). Workers are never controlled from another worker; change the manager only by editing manager in fleet.json and moving its singleton services.`,
+        },
+      };
+}
+
+/** Workers depend on the manager's services, so it never sleeps on AC power. */
+export async function managerPower(run: Runner): Promise<Check> {
+  const result = await run(['pmset', '-g', 'custom']);
+  const ac = result.ok ? (result.stdout.split('AC Power:')[1] ?? '') : '';
+  const sleep = /^\s*sleep\s+(\d+)/m.exec(ac)?.[1];
+  return sleep === '0'
+    ? {
+        id: 'manager:power',
+        status: 'pass',
+        summary: 'The manager never sleeps on AC power.',
+      }
+    : {
+        id: 'manager:power',
+        status: 'fail',
+        summary: 'MANAGER_SLEEPS_ON_POWER',
+        remedy: {
+          action:
+            'Workers lose the CLI proxy and Beszel hub while the manager sleeps. Apply the manager power policy once: ansible-playbook -K ansible/macos/manager.yml',
+        },
+      };
+}