@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/cli/fleet-sync-enrollment.ts

@@ -0,0 +1,140 @@
+import { mkdir, rm } from 'node:fs/promises';
+import { loadRoster } from './roster';
+import { enrolledRoster } from '../definition';
+import { FleetUsageError } from '../errors';
+import { dirname } from 'node:path';
+import { fleet as definition } from '../fleet-config';
+import { discoverTailnet } from '../tailscale';
+import { matchComputers } from '../tailnet-matching';
+import { sshArgs, sshFailure } from '../ssh';
+import { ensurePrivateRoot } from '../machine/state';
+import type { FleetNode, Runner } from '../types';
+import { STATE_ROSTER, readRosterText, writeRosterText } from '../state/roster';
+
+/** Interactive enrollment/resume only. Existing paths and explicit identities
+ * are preserved. Tailnet candidates never enter the declared membership.
+ */
+export async function enrollManager(path: string, run: Runner): Promise<void> {
+  const canonical = definition;
+  const original = await existingEnrollment(path);
+  const saved =
+    original === null
+      ? []
+      : enrolledRoster(canonical, await loadRoster(path)).nodes;
+  const nodes: FleetNode[] = canonical.workers.map((node) => ({
+    id: node.id,
+    ssh: node.ssh,
+    workspace: '/',
+    required: ['t3', 'toolchain', 'workstation'],
+  }));
+  const pending = nodes.filter(
+    (node) => !saved.find((entry) => entry.id === node.id)?.tailscaleId
+  );
+  if (!pending.length) return;
+  const tailnet = await discoverTailnet(run);
+  const matches = await matchComputers(nodes, tailnet.computers, run);
+  const enrolled: FleetNode[] = [];
+  for (const node of pending) {
+    const peers = tailnet.computers.filter((peer) =>
+      matches.get(peer.id)?.some((candidate) => candidate.id === node.id)
+    );
+    if (
+      ambiguous(
+        peers.map((peer) => peer.id),
+        matches
+      )
+    )
+      continue;
+    const connection = await inspectConnection(node, peers[0]?.id, run);
+    if (connection) enrolled.push(connection);
+  }
+  const merged = mergeEnrollment(saved, enrolled);
+  if (JSON.stringify(merged) === JSON.stringify(saved)) return;
+  // Only canonical nodes reached over host-key-verified SSH get here, so
+  // enrollment needs no extra confirmation; the list is still shown.
+  process.stderr.write(
+    `Enrolling verified SSH connections for canonical targets:\n${enrolled.map((node) => `  ${node.id} → ${node.ssh}; device ${enrollmentIdentity(node)}`).join('\n')}\nMissing targets remain visible and are never replaced by discovered peers.\n`
+  );
+  await persistEnrollment(path, original, merged);
+}
+
+async function inspectConnection(
+  node: FleetNode,
+  peerId: string | undefined,
+  run: Runner
+): Promise<FleetNode | null> {
+  const home = await run([...sshArgs(node), 'printf "%s" "$HOME"'], {
+    timeoutMs: 20_000,
+  });
+  if (!home.ok) {
+    const check = sshFailure(home);
+    process.stderr.write(
+      `[${node.id}] ${check.id}: ${check.summary} ${check.remedy?.action ?? ''}\n`
+    );
+    return null;
+  }
+  if (!/^\/[^\x00-\x1f\x7f]{1,4000}$/.test(home.stdout)) return null;
+  return {
+    ...node,
+    workspace: `${home.stdout}/.local/share/personal/fleet`,
+    ...(peerId ? { tailscaleId: peerId } : { bootstrapViaSsh: true }),
+  };
+}
+
+function enrollmentIdentity(node: FleetNode): string {
+  return (
+    node.tailscaleId ??
+    'initial trusted SSH bootstrap; native Tailscale login required'
+  );
+}
+
+function ambiguous(ids: string[], matches: Map<string, FleetNode[]>): boolean {
+  return (
+    ids.length > 1 || (ids.length === 1 && matches.get(ids[0]!)?.length !== 1)
+  );
+}
+function existingEnrollment(path: string): Promise<string | null> {
+  return readRosterText(path);
+}
+export function mergeEnrollment(
+  saved: FleetNode[],
+  discovered: FleetNode[]
+): FleetNode[] {
+  const result = saved.map((node) => ({ ...node }));
+  for (const candidate of discovered) {
+    const existing = result.find((node) => node.id === candidate.id);
+    if (!existing) result.push(candidate);
+    else if (!existing.tailscaleId && candidate.tailscaleId) {
+      existing.tailscaleId = candidate.tailscaleId;
+      delete existing.bootstrapViaSsh;
+    }
+  }
+  return result;
+}
+async function persistEnrollment(
+  path: string,
+  original: string | null,
+  nodes: FleetNode[]
+): Promise<void> {
+  const write = async (): Promise<void> => {
+    if ((await existingEnrollment(path)) !== original)
+      throw new FleetUsageError(
+        'Enrollment changed during approval. Inspect it and rerun; no connection was overwritten.'
+      );
+    await writeRosterText(
+      path,
+      JSON.stringify({ version: 1, nodes }, null, 2) + '\n'
+    );
+  };
+  // The fleet state serializes its own writers; a roster file needs a lock.
+  if (path === STATE_ROSTER) return write();
+  await ensurePrivateRoot(dirname(path));
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+  const lock = `${path}.enroll.lock`;
+  await mkdir(lock, { mode: 0o700 });
+  try {
+    await write();
+  } finally {
+    await rm(lock, { recursive: true });
+  }
+}

package/src/cli/fleet-sync-forward.ts

@@ -0,0 +1,134 @@
+import { spawn } from 'node:child_process';
+import type { FleetDefinition } from '../definition';
+import { shellQuote, sshArgs } from '../ssh';
+import type { FleetNode } from '../types';
+import type { CommandOutcome } from './contracts';
+import type { FleetSyncOptions } from './fleet-sync';
+
+/** Set on the manager side of a hand-off (and in tests): never hand off again. */
+export const NO_FORWARD_ENV = 'PERSONAL_FLEET_NO_FORWARD';
+
+/** Streams the remote command: progress on stderr, the report on stdout. */
+export type ForwardSpawn = (
+  command: string[]
+) => Promise<{ exitCode: number; stdout: string }>;
+
+export function forwardingDisabled(): boolean {
+  return Boolean(process.env[NO_FORWARD_ENV]);
+}
+
+/** fleet-sync only ever runs on the manager. Started on a worker, it hands the
+ * same request to the manager over host-key-verified SSH (no agent forwarding,
+ * no credentials copied) and reports the manager's result unchanged.
+ */
+export async function forwardToManager(
+  definition: FleetDefinition,
+  options: FleetSyncOptions,
+  revision: string | undefined,
+  remote: { spawn?: ForwardSpawn | undefined; script?: string | null } = {}
+): Promise<CommandOutcome> {
+  // A test must inject the spawn: it can never reach the real manager.
+  if (!remote.spawn && process.env['NODE_ENV'] === 'test')
+    throw new Error('FLEET_FORWARD_IN_TEST');
+  const spawnRemote = remote.spawn ?? spawnInherited;
+  const runScript = remote.script ?? null;
+  const { manager } = definition;
+  process.stderr.write(
+    `fleet-sync runs on the manager; handing off to ${manager.id} over SSH…\n`
+  );
+  const result = await spawnRemote(
+    managerCommand(definition, options, revision, runScript)
+  );
+  const stdout = result.stdout.trim();
+  if (options.json) {
+    try {
+      return { exitCode: result.exitCode, data: JSON.parse(stdout) as unknown };
+    } catch {
+      /* Fall through to the structured failure below. */
+    }
+  }
+  if (result.exitCode === 255 && !stdout)
+    return {
+      exitCode: 1,
+      data: {
+        version: 1,
+        ok: false,
+        code: 'FLEET_MANAGER_UNREACHABLE',
+        error: `Could not reach the manager (${manager.id}) over SSH. Wake it, check Tailscale and SSH (Remote Login), or run fleet-sync on the manager directly.`,
+      },
+    };
+  return {
+    exitCode: result.exitCode,
+    data: { version: 1, forwardedTo: manager.id, exitCode: result.exitCode },
+    text: stdout,
+  };
+}
+
+/** With the revision's tracked `fleet-manager-run.sh`, the manager runs
+ * fleet-sync from a clean checkout of that exact revision; otherwise (no
+ * revision, or an older one without the script) its installed launcher.
+ */
+export function managerCommand(
+  definition: FleetDefinition,
+  options: FleetSyncOptions,
+  revision: string | undefined,
+  runScript: string | null = null
+): string[] {
+  const { manager } = definition;
+  const target: FleetNode = {
+    id: manager.id,
+    ssh: `${manager.network.user}@${manager.network.tailscaleName}`,
+    workspace: '~',
+    required: [],
+  };
+  const args =
+    runScript && revision
+      ? [
+          'sh',
+          '-c',
+          runScript,
+          'fleet-manager-run',
+          revision,
+          ...forwardedFlags(options, undefined),
+        ]
+      : ['devfleet', 'fleet-sync', ...forwardedFlags(options, revision)];
+  const color = process.stdout.isTTY && !process.env['NO_COLOR'];
+  // A login shell gives the manager's normal PATH (mise, Docker, Tailscale).
+  const script = `export ${NO_FORWARD_ENV}=1${color ? ' FORCE_COLOR=1' : ''}; exec ${args.map(shellQuote).join(' ')}`;
+  return [...sshArgs(target), `exec /bin/zsh -lc ${shellQuote(script)}`];
+}
+
+export function forwardedFlags(
+  options: FleetSyncOptions,
+  revision: string | undefined
+): string[] {
+  const flags = revision ? ['--revision', revision] : [];
+  for (const node of options.node ?? []) flags.push('--node', node);
+  if (options.schedule) flags.push('--schedule', options.schedule);
+  const switches: [boolean | undefined, string][] = [
+    [options.all, '--all'],
+    [options.dryRun, '--dry-run'],
+    [options.apply, '--apply'],
+    [options.yes, '--yes'],
+    [options.json, '--json'],
+    [options.nonInteractive, '--non-interactive'],
+  ];
+  for (const [on, flag] of switches) if (on) flags.push(flag);
+  return flags;
+}
+
+function spawnInherited(
+  command: string[]
+): Promise<{ exitCode: number; stdout: string }> {
+  return new Promise((resolve) => {
+    const child = spawn(command[0]!, command.slice(1), {
+      stdio: ['ignore', 'pipe', 'inherit'],
+    });
+    let stdout = '';
+    child.stdout.setEncoding('utf8').on('data', (chunk: string) => {
+      stdout += chunk;
+    });
+    child.on('error', () => resolve({ exitCode: 255, stdout: '' }));
+    child.on('close', (code) => resolve({ exitCode: code ?? 1, stdout }));
+  });
+}

package/src/cli/fleet-sync-manager.ts

@@ -0,0 +1,249 @@
+import { blocking, type Check, type Runner } from '../types';
+import type { FleetDefinition } from '../definition';
+import type { ManagerNode } from '../manager-sync';
+import { hostId } from '../topology';
+import { convergeProxy } from '../proxy-converge';
+import { attempt } from '../node-converge-shared';
+import { quotaLimited, verifyFleetProviders } from '../agents/fleet-verify';
+import { verifyNativeAgents } from '../agents/native-verify';
+import { convergeManagerHub } from '../monitoring/manager-hub';
+import { enrollManagerMonitoring } from './fleet-sync-monitoring';
+import { installConfig } from '../fleet-config';
+import { machineProviders } from '../t3/providers';
+import { providerNames } from '../t3/names-check';
+import macos from '../../ansible/macos/policy.json';
+import { ansibleCommand } from '../platform/ansible';
+import { PACKAGE_ROOT } from '../package-info';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** The manager converges itself locally (never over SSH) before any worker:
+ * it must be the declared manager, stay awake on AC power, and host every
+ * singleton service (CLI proxy, Beszel hub). Workers only consume them.
+ */
+export async function convergeManager(
+  definition: FleetDefinition,
+  input: {
+    revision: string;
+    apply: boolean;
+    workspace: string;
+  },
+  run: Runner
+): Promise<ManagerNode> {
+  const started = Date.now();
+  const id = definition.manager.id;
+  const identity = await managerIdentity(id, run);
+  const checks: Check[] = [identity];
+  if (identity.status === 'pass') {
+    // Launchers and scheduled jobs on the manager read the installed copy.
+    if (input.apply) installConfig();
+    checks.push(await managerPower(run, input.workspace));
+    checks.push(await managerDocker(run));
+    checks.push(await machineProviders(id, input.workspace, input.apply));
+    checks.push(await providerNames(id, input.workspace, input.apply));
+    if (input.apply) {
+      // Same provider profiles and the same Vault gateway credential as every
+      // worker: the instances above launch from these profiles, and the check
+      // proves the credential works end to end through the shared proxy.
+      checks.push(
+        ...(await attempt('configuration:apply', () =>
+          managerAgentProfiles(id, input.workspace, run)
+        ))
+      );
+      const providers = await attempt('provider:verification', () =>
+        verifyFleetProviders(id)
+      );
+      checks.push(...providers);
+      // Same agents, versions and shared accounts as every worker.
+      checks.push(
+        ...(await attempt('agent:verification', () =>
+          verifyNativeAgents(id, input.workspace, run, {
+            outOfQuota: quotaLimited(providers),
+          })
+        ))
+      );
+    }
+    checks.push(
+      ...(await convergeProxy(id, input.revision, run, {
+        apply: input.apply,
+        cwd: input.workspace,
+      }))
+    );
+    if (input.apply) {
+      const hub = await convergeManagerHub(input.workspace, definition, run);
+      checks.push(...hub);
+      // The manager is always a monitored system on its own hub.
+      if (!hub.some(blocking))
+        checks.push(
+          ...(await enrollManagerMonitoring(definition, input.workspace, run))
+        );
+    }
+  }
+  return {
+    node: id,
+    role: 'manager',
+    checks,
+    reachability: 'online',
+    desiredRevision: input.revision,
+    appliedRevision:
+      input.apply && !checks.some(blocking) ? input.revision : null,
+    durationMs: Date.now() - started,
+  };
+}
+
+const PROFILE_TOOLS = [
+  'codex',
+  'claude',
+  'gemini',
+  'opencode',
+  'grok',
+  'antigravity',
+] as const;
+
+/** Applies the proxy agent profiles the managed T3 instances launch from
+ * (codex-proxy.ts refuses to start with a missing or stale profile). Workers
+ * apply the same profiles from a pinned deployment; the manager's checkout is
+ * the human's working tree, so only the workstation sources must be clean
+ * (the `agents apply` owner enforces that) and nothing else is touched.
+ */
+export async function managerAgentProfiles(
+  id: string,
+  workspace: string,
+  run: Runner
+): Promise<Check[]> {
+  for (const tool of PROFILE_TOOLS) {
+    const result = await run(
+      [
+        'mise',
+        'exec',
+        '--locked',
+        'bun',
+        '--',
+        'bun',
+        'src/agents/cli.ts',
+        '--json',
+        '--yes',
+        '--non-interactive',
+        'agents',
+        'apply',
+        '--node',
+        id,
+        '--tool',
+        tool,
+        '--profile',
+        'proxy',
+      ],
+      { cwd: workspace, timeoutMs: 120_000 }
+    );
+    if (!result.ok)
+      return [
+        {
+          id: 'configuration:apply',
+          status: 'fail',
+          summary: 'MANAGER_AGENT_PROFILES_FAILED',
+          remedy: {
+            action: `Run bun src/agents/cli.ts --yes agents apply --node ${id} --tool ${tool} --profile proxy in the dev-fleet checkout on the manager and review the result.`,
+          },
+        },
+      ];
+  }
+  return [
+    {
+      id: 'configuration:apply',
+      status: 'pass',
+      summary: `The manager has the same proxy agent profiles as every worker (${PROFILE_TOOLS.join(', ')}).`,
+    },
+  ];
+}
+
+/** fleet-sync runs only on the declared manager. */
+export async function managerIdentity(id: string, run: Runner): Promise<Check> {
+  const result = await run(['scutil', '--get', 'LocalHostName']);
+  const local = result.ok ? hostId(result.stdout.trim()) : '';
+  return local === id
+    ? {
+        id: 'manager:identity',
+        status: 'pass',
+        summary: `This machine is the declared fleet manager (${id}).`,
+      }
+    : {
+        id: 'manager:identity',
+        status: 'fail',
+        summary: 'FLEET_NOT_MANAGER',
+        remedy: {
+          action: `Run fleet-sync on the manager (${id}). Workers are never controlled from another worker; change the manager only by editing manager in fleet.json and moving its singleton services.`,
+        },
+      };
+}
+
+/** Workers depend on the manager's services, so it never sleeps on AC power.
+ * Measures every setting `ansible/macos/manager.yml` declares, from the same
+ * policy file the playbook applies.
+ */
+export async function managerPower(
+  run: Runner,
+  workspace = '.'
+): Promise<Check> {
+  const result = await run(['pmset', '-g', 'custom']);
+  const ac = result.ok ? (result.stdout.split('AC Power:')[1] ?? '') : '';
+  const observed = (key: string) =>
+    new RegExp(`^\\s*${key}\\s+(\\S+)`, 'm').exec(ac)?.[1];
+  const drifted = Object.entries(macos.pmset)
+    .filter(([key, value]) => observed(key) !== value)
+    .map(([key]) => key);
+  return drifted.length === 0
+    ? {
+        id: 'manager:power',
+        status: 'pass',
+        summary: 'The manager never sleeps on AC power.',
+      }
+    : {
+        id: 'manager:power',
+        status: 'fail',
+        summary: drifted.includes('sleep')
+          ? 'MANAGER_SLEEPS_ON_POWER'
+          : `MANAGER_POWER_POLICY_DRIFT ${drifted.join(' ')}`,
+        remedy: {
+          action: `Workers lose the CLI proxy and Beszel hub while the manager sleeps. Apply the manager power policy once, as an administrator: cd ${workspace} && ${ansibleCommand().join(' ')} -K ansible/macos/manager.yml`,
+        },
+      };
+}
+
+/** The Docker Desktop release `mise.toml` pins, as the engine names itself. */
+export function pinnedDockerDesktop(root = PACKAGE_ROOT): string {
+  const { vars } = Bun.TOML.parse(
+    readFileSync(join(root, 'mise.toml'), 'utf8')
+  ) as { vars: Record<string, string> };
+  return `Docker Desktop ${vars['docker_desktop_version']} (${vars['docker_desktop_build']})`;
+}
+
+/** The manager's services run in Docker Desktop, which no playbook installs
+ * and which updates itself: the pinned release is measured, and a different
+ * one warns instead of failing the run.
+ */
+export async function managerDocker(
+  run: Runner,
+  pinned = pinnedDockerDesktop()
+): Promise<Check> {
+  const result = await run(
+    ['docker', 'version', '--format', '{{.Server.Platform.Name}}'],
+    { timeoutMs: 20_000 }
+  );
+  const observed = result.ok ? result.stdout.trim() : '';
+  if (observed === pinned)
+    return {
+      id: 'manager:docker',
+      status: 'pass',
+      summary: `The manager runs the pinned ${pinned}.`,
+    };
+  return {
+    id: 'manager:docker',
+    status: 'warn',
+    summary: observed
+      ? 'MANAGER_DOCKER_DESKTOP_DRIFT'
+      : 'MANAGER_DOCKER_DESKTOP_UNOBSERVED',
+    remedy: {
+      action: `The manager's services are tested on ${pinned}. Install that release and turn off its automatic updates, or expect differences; fleet-sync does not install or replace Docker Desktop.`,
+    },
+  };
+}