@crvouga/dev-fleet 0.0.0-stage → 0.1.0

package/src/node-converge-linux.ts

@@ -0,0 +1,372 @@
+import { workerReadiness } from './worker-readiness';
+import { runAnsible } from './platform/ansible';
+import { runAnsibleWithReceipt } from './platform/ansible-receipt';
+import { loadedT3Policy, stageT3Policy } from './platform/t3-policy';
+import { verifyUbuntuBoot } from './platform/ubuntu-verify';
+import { verifyResourceSafety } from './platform/resource-verify';
+import { convergeT3Route, T3_ROUTE } from './platform/t3-route';
+import { convergeProxy } from './proxy-converge';
+import { providerNames } from './t3/names-check';
+import { machineProviders } from './t3/providers';
+import { quotaLimited, verifyFleetProviders } from './agents/fleet-verify';
+import { verifyNativeAgents } from './agents/native-verify';
+import { managedT3, managedT3Desktop } from '../bootstrap/native-t3';
+import { convergeT3Service, t3StartedAt } from './platform/t3-service';
+import { userInfo } from 'node:os';
+import { resolve } from 'node:path';
+import { auditUbuntu } from './platform/ubuntu-audit';
+import {
+  applyAgentProfiles,
+  attempt,
+  configureWorkstation,
+} from './node-converge-shared';
+import type { Check, Runner } from './types';
+
+/** Ubuntu worker convergence: Ansible owns OS policy, services and recovery; the
+ * native T3, proxy and provider owners run only after the baseline passes.
+ */
+export async function convergeLinux(
+  root: string,
+  target: { node: string; revision: string },
+  run: Runner
+): Promise<Check[]> {
+  const { node, revision } = target;
+  const audit = await auditUbuntu(run);
+  if (!audit.supported || !audit.passwordlessSudo)
+    return [
+      {
+        id: 'ubuntu:prerequisites',
+        status: 'fail',
+        summary: audit.supported
+          ? 'SUDO_AUTHORIZATION_REQUIRED'
+          : 'UBUNTU_BASELINE_MISMATCH',
+        remedy: {
+          action:
+            'Authorize designated-user sudo through the manager enrollment flow; unsupported releases require an explicit migration.',
+        },
+      },
+    ];
+  const checks: Check[] = [];
+  checks.push(
+    ...(await attempt('tailscale:configuration', async () => [
+      await applyTailscale(root, run),
+    ]))
+  );
+  const { baseline, verified } = await baselineStep(root, run);
+  checks.push(...baseline, ...verified);
+  checks.push(...(await t3AfterBaseline(root, { node, baseline, run })));
+  checks.push(
+    ...(await attempt('workstation:configuration', () =>
+      configureWorkstation(root, run)
+    ))
+  );
+  checks.push(
+    ...(await attempt('configuration:apply', () =>
+      applyAgentProfiles(root, node, revision, run)
+    ))
+  );
+  checks.push(
+    ...(await attempt('proxy:configuration', () =>
+      convergeProxy(node, revision, run)
+    ))
+  );
+  const providers = await attempt('provider:verification', () =>
+    verifyFleetProviders(node)
+  );
+  checks.push(...providers);
+  checks.push(
+    ...(await attempt('agent:verification', () =>
+      verifyNativeAgents(node, root, run, {
+        outOfQuota: quotaLimited(providers),
+      })
+    ))
+  );
+  checks.push(
+    ...(await attempt('worker:readiness', () => workerReadiness(run, node)))
+  );
+  return checks;
+}
+
+/** Applies the baseline (skipped on an unchanged, verified receipt), then
+ * measures boot and resource policy. Self-healing: if a skipped baseline's
+ * effective policy no longer verifies, the baseline is applied in full and
+ * measured again in the same run.
+ */
+async function baselineStep(
+  root: string,
+  run: Runner
+): Promise<{ baseline: Check[]; verified: Check[] }> {
+  const measure = async () => [
+    ...(await attempt('ubuntu:boot-policy', () =>
+      verifyUbuntuBoot(userInfo().username, run)
+    )),
+    ...(await attempt('ubuntu:resource-safety', () =>
+      verifyResourceSafety(userInfo().homedir, run)
+    )),
+  ];
+  let cached = false;
+  let baseline = await attempt('ubuntu:baseline', async () => {
+    const applied = await applyBaseline(root, run, false);
+    cached = applied.cached;
+    return applied.checks;
+  });
+  let verified = await measure();
+  if (cached && verified.some((check) => check.status === 'fail')) {
+    baseline = await attempt('ubuntu:baseline', async () => {
+      return (await applyBaseline(root, run, true)).checks;
+    });
+    verified = await measure();
+  }
+  return { baseline, verified };
+}
+
+async function applyBaseline(
+  root: string,
+  run: Runner,
+  force: boolean
+): Promise<{ cached: boolean; checks: Check[] }> {
+  const user = userInfo();
+  const ubuntu = resolve(root, 'ansible/ubuntu');
+  const result = await runAnsibleWithReceipt(
+    run,
+    resolve(ubuntu, 'baseline.yml'),
+    {
+      cwd: ubuntu,
+      timeoutMs: 900_000,
+      input: JSON.stringify({
+        development_source: root,
+        development_user: user.username,
+        development_uid: user.uid,
+        development_home: user.homedir,
+      }),
+    },
+    {
+      name: 'ubuntu-baseline',
+      sources: [ubuntu],
+      force,
+      verify: () => baselineServicesActive(run),
+    }
+  );
+  return {
+    cached: result.cached,
+    checks: [
+      {
+        id: 'ubuntu:baseline',
+        status: result.ok ? 'pass' : 'fail',
+        summary: result.ok
+          ? result.cached
+            ? 'Ubuntu baseline unchanged since its last successful apply and its services are active (re-applied in full at least daily).'
+            : 'Ubuntu baseline tasks completed, including always-on service recovery; effective boot/session policies require separate verification.'
+          : 'UBUNTU_BASELINE_INCOMPLETE',
+      },
+    ],
+  };
+}
+
+/** Cheap probe for what the baseline owns: system services and user timers. */
+async function baselineServicesActive(run: Runner): Promise<boolean> {
+  const system = await run([
+    'systemctl',
+    'is-active',
+    '--quiet',
+    'ssh.service',
+    'tailscaled.service',
+    'docker.service',
+  ]);
+  if (!system.ok) return false;
+  const user = await run([
+    'systemctl',
+    '--user',
+    'is-active',
+    '--quiet',
+    'personal-worker-auth.timer',
+  ]);
+  return user.ok;
+}
+
+async function applyTailscale(root: string, run: Runner): Promise<Check> {
+  const ubuntu = resolve(root, 'ansible/ubuntu');
+  const tailscale = await runAnsibleWithReceipt(
+    run,
+    'tailscale.yml',
+    { cwd: ubuntu, timeoutMs: 300_000, input: '{}' },
+    {
+      name: 'ubuntu-tailscale',
+      sources: [ubuntu],
+      verify: async () => {
+        const status = await run(['tailscale', 'status', '--json']);
+        return (
+          status.ok &&
+          (JSON.parse(status.stdout) as { BackendState?: unknown })
+            .BackendState === 'Running'
+        );
+      },
+    }
+  );
+  return {
+    id: 'tailscale:configuration',
+    status: tailscale.ok ? 'pass' : 'fail',
+    summary: tailscale.ok
+      ? 'Pinned native Tailscale service is enabled, running and authenticated; manager reachability remains separate.'
+      : 'TAILSCALE_SETUP_OR_LOGIN_REQUIRED',
+  };
+}
+
+async function convergeT3(
+  root: string,
+  node: string,
+  run: Runner
+): Promise<Check[]> {
+  const artifact = managedT3();
+  const startedAt = await t3StartedAt(run);
+  const checks = await convergeT3Service(
+    artifact,
+    run,
+    async () => {
+      const staged = await stageT3Policy(root, artifact.path, run, node);
+      return staged.status === 'pass' ? [] : [staged];
+    },
+    () => installT3Desktop(root, run)
+  );
+  if (checks.every((check) => check.status === 'pass'))
+    checks.push(
+      await loadedT3Policy(artifact.path, run, {
+        restarted: startedAt !== (await t3StartedAt(run)),
+      })
+    );
+  // Settings are watched by T3: providers arrive and stale ones leave at once.
+  checks.push(await machineProviders(node, root, true));
+  // Names live in the same settings, so they conform without a restart.
+  checks.push(await providerNames(node, root, true));
+  const serviceReady = checks.some(
+    (check) => check.id === 't3:service' && check.status === 'pass'
+  );
+  if (serviceReady)
+    checks.push(
+      ...(await convergeT3Route(run, () => publishT3Route(root, run)))
+    );
+  if (serviceReady) checks.push(...(await enableWorkerRecovery(root, run)));
+  return checks;
+}
+
+async function publishT3Route(root: string, run: Runner): Promise<boolean> {
+  const result = await runAnsible(run, 't3-route.yml', {
+    cwd: resolve(root, 'ansible/ubuntu'),
+    input: JSON.stringify({ t3_route_target: T3_ROUTE.proxy }),
+    timeoutMs: 60_000,
+  });
+  return result.ok;
+}
+
+async function installT3Desktop(root: string, run: Runner): Promise<boolean> {
+  const user = userInfo();
+  const desktop = managedT3Desktop();
+  const ubuntu = resolve(root, 'ansible/ubuntu');
+  const result = await runAnsibleWithReceipt(
+    run,
+    resolve(ubuntu, 't3-desktop.yml'),
+    {
+      cwd: root,
+      timeoutMs: 600_000,
+      input: JSON.stringify({
+        development_source: root,
+        development_user: user.username,
+        development_uid: user.uid,
+        development_home: user.homedir,
+        t3_desktop_url: desktop.url,
+        t3_desktop_name: desktop.name,
+        t3_desktop_sha256: desktop.sha256,
+      }),
+    },
+    {
+      name: 'ubuntu-t3-desktop',
+      sources: [ubuntu],
+      // A stopped or disabled app reruns the play, which starts it.
+      verify: async () =>
+        (
+          await run([
+            'systemctl',
+            '--user',
+            'is-enabled',
+            '--quiet',
+            't3code-desktop.service',
+          ])
+        ).ok &&
+        (
+          await run([
+            'systemctl',
+            '--user',
+            'is-active',
+            '--quiet',
+            't3code-desktop.service',
+          ])
+        ).ok,
+    }
+  );
+  return result.ok;
+}
+
+async function t3AfterBaseline(
+  root: string,
+  context: { node: string; baseline: Check[]; run: Runner }
+): Promise<Check[]> {
+  const { node, baseline, run } = context;
+  if (baseline.some((check) => check.status !== 'pass'))
+    return [
+      {
+        id: 't3:prerequisites',
+        status: 'skip',
+        summary: 'T3_BASELINE_PREREQUISITES_REQUIRED',
+        remedy: {
+          action:
+            'Complete the designated-user baseline and lingering policy before changing native service lifecycle. Existing service state is preserved.',
+        },
+      },
+    ];
+  return attempt('t3:service', () => convergeT3(root, node, run));
+}
+
+async function enableWorkerRecovery(
+  root: string,
+  run: Runner
+): Promise<Check[]> {
+  const user = userInfo();
+  const ubuntu = resolve(root, 'ansible/ubuntu');
+  const result = await runAnsibleWithReceipt(
+    run,
+    resolve(ubuntu, 'recovery.yml'),
+    {
+      cwd: root,
+      timeoutMs: 60_000,
+      input: JSON.stringify({
+        development_source: root,
+        development_user: user.username,
+        development_uid: user.uid,
+        development_home: user.homedir,
+      }),
+    },
+    {
+      name: 'ubuntu-recovery',
+      sources: [ubuntu],
+      verify: async () =>
+        (
+          await run([
+            'systemctl',
+            '--user',
+            'is-active',
+            '--quiet',
+            'personal-worker-recovery.timer',
+          ])
+        ).ok,
+    }
+  );
+  return [
+    {
+      id: 'worker:service-recovery',
+      status: result.ok ? 'pass' : 'fail',
+      summary: result.ok
+        ? 'T3 recovery enabled: a stopped desktop app is started and an unresponsive one restarted after three failed one-minute health checks.'
+        : 'WORKER_RECOVERY_SETUP_FAILED',
+    },
+  ];
+}

package/src/node-converge-shared.ts

@@ -0,0 +1,83 @@
+import { fingerprint, writeReceipt } from './fingerprint';
+import { configureWorkspace } from './configure';
+import { stableWorkspace } from './platform/t3-policy';
+import type { Check, Runner } from './types';
+
+/** Phase wrapper: one failure code per phase, durations measured, no exception text. */
+export async function attempt(
+  id: string,
+  operation: () => Promise<Check[]>
+): Promise<Check[]> {
+  const started = Date.now();
+  try {
+    const checks = await operation();
+    const phaseDurationMs = Date.now() - started;
+    return checks.map((check) => ({ ...check, phaseDurationMs }));
+  } catch {
+    return [
+      {
+        id,
+        status: 'fail',
+        summary: 'REMOTE_PHASE_FAILED',
+        phaseDurationMs: Date.now() - started,
+        remedy: {
+          action:
+            'Inspect the named owner and approved source revision. Prior phase evidence is retained; no exception, environment or auth response is logged.',
+        },
+      },
+    ];
+  }
+}
+
+/** The `devfleet` launcher (linked to the applied revision, never copied) and
+ * this worker's own credential refresh. Identical on Linux and macOS.
+ */
+export async function configureWorkstation(
+  root: string,
+  run: Runner
+): Promise<Check[]> {
+  const installed = await run(
+    // The stable path of the applied revision: it survives pruning.
+    ['sh', 'bootstrap/install-launcher.sh', await stableWorkspace(root)],
+    { cwd: root, timeoutMs: 30_000 }
+  );
+  if (!installed.ok)
+    return [
+      {
+        id: 'workstation:installation',
+        status: 'fail',
+        summary: 'LAUNCHER_INSTALLATION_FAILED',
+      },
+    ];
+  // This receipt certifies only frozen dependency inputs, never machine readiness.
+  await writeReceipt(root, await fingerprint(root, run));
+  await run(['bun', 'bootstrap/worker-auth.ts'], {
+    cwd: root,
+    timeoutMs: 90_000,
+  });
+  return [
+    {
+      id: 'workstation:configuration',
+      status: 'pass',
+      summary:
+        'The devfleet launcher is linked to the applied revision; authentication remains a separate gate.',
+    },
+  ];
+}
+
+export function applyAgentProfiles(
+  root: string,
+  node: string,
+  revision: string,
+  run: Runner
+): Promise<Check[]> {
+  return configureWorkspace(
+    root,
+    { node, revision, tool: 'codex', profile: 'proxy' },
+    run,
+    {
+      additionalTools: ['claude', 'gemini', 'opencode', 'grok', 'antigravity'],
+      bootstrappedDeployment: true,
+    }
+  );
+}

package/src/node-converge.ts

@@ -0,0 +1,40 @@
+import { runLockedCommand as runCommand } from './runner';
+import { isRevision } from './machine/state';
+import { convergeLinux } from './node-converge-linux';
+import type { Check } from './types';
+
+/** Internal remote entry, run on a worker only. The manager owns the operation
+ * lock and source selection; this installs no pull daemon or timer. Never
+ * prints child output. Workers are Linux.
+ */
+async function main(): Promise<Check[]> {
+  const [node, revision] = process.argv.slice(2);
+  if (!node || !/^[a-z][a-z0-9-]{0,62}$/.test(node) || !isRevision(revision))
+    throw new Error();
+  const root = process.cwd();
+  const head = await runCommand(['git', 'rev-parse', 'HEAD']);
+  if (!head.ok || head.stdout.trim() !== revision) throw new Error();
+  if (process.platform === 'linux')
+    return convergeLinux(root, { node, revision }, runCommand);
+  return [
+    {
+      id: 'platform:unsupported',
+      status: 'fail',
+      summary: 'PLATFORM_UNSUPPORTED',
+      remedy: {
+        action:
+          'Workers are Ubuntu 26.04. The macOS manager converges itself locally in fleet-sync.',
+      },
+    },
+  ];
+}
+if (import.meta.main) {
+  const checks = await main().catch((): Check[] => [
+    {
+      id: 'configuration:operation',
+      status: 'fail',
+      summary: 'REMOTE_CONFIGURATION_INCOMPLETE',
+    },
+  ]);
+  process.stdout.write(JSON.stringify({ version: 1, checks }));
+}

package/src/onboard.ts

@@ -0,0 +1,35 @@
+import { shellQuote } from './ssh';
+import { syncPlan } from './sync';
+import type { FleetNode } from './types';
+
+export function onboarding(node: FleetNode, launcher = 'devfleet'): object {
+  return {
+    version: 1,
+    node: node.id,
+    applied: false,
+    steps: [
+      'On the new node, review and run bootstrap/bootstrap.sh (or bootstrap.ps1 for native Windows). Use docs/machine-setup.md. This installs the shared locked toolchain; it does not enable SSH.',
+      'Enable the OS SSH service as the operator on a trusted private network. Do not expose SSH publicly or create cloud resources.',
+      'Create a distinct per-machine SSH key and authorize its PUBLIC key on the destination. Verify host fingerprints out of band. Keep private keys and auth tokens on their original machines.',
+      'In the manager SSH config, configure this alias with the real HostName, User, IdentityFile, optional Port/ProxyJump and IdentitiesOnly yes. Fleet will not write or overwrite SSH configuration.',
+      'Test ssh ALIAS manually once after fingerprint verification. Ensure noninteractive shell startup writes no stdout. Set runtime to an absolute Node/Bun executable/shim if SSH PATH omits it.',
+      'Authenticate separately on the node: gh auth login; gh auth setup-git; vault login using the approved method. Configure coding-provider sessions independently; never copy auth directories.',
+      'For T3 Connect, review t3 status and t3 service status, then complete t3 up --connect and operator pairing/linking. T3 pairing/auth state is not an SSH credential.',
+      'For VS Code, install Remote-SSH in the LOCAL editor profile, use the same SSH alias/client/config, enable the login terminal temporarily, and read Output → Remote - SSH. Restart the editor after PATH/agent changes.',
+      'VS Code needs TCP forwarding and compatible remote server prerequisites. Test with ssh -N -L 127.0.0.1:18080:127.0.0.1:REMOTE_PORT ALIAS against a known loopback service, then curl localhost:18080 from a second terminal. Review server policy; never loosen it automatically.',
+      'Run devfleet fleet-status --node NODE --json from every intended manager. This is directed connectivity, not an all-pairs proof. Review skipped T3/VS Code end-to-end checks manually.',
+    ],
+    sshTemplate: `Host ${node.ssh}\n  HostName REPLACE_WITH_PRIVATE_HOST\n  User REPLACE_WITH_USER\n  IdentityFile ~/.ssh/REPLACE_WITH_NODE_KEY\n  IdentitiesOnly yes\n  ForwardAgent no\n`,
+    vscodeSettings: {
+      'remote.SSH.path': 'REPLACE_WITH_SAME_OPENSSH_BINARY',
+      'remote.SSH.configFile': 'REPLACE_WITH_ABSOLUTE_SSH_CONFIG',
+      'remote.SSH.showLoginTerminal': true,
+    },
+    workspaceCommand: `cd -- ${shellQuote(node.workspace)}`,
+    verifyCommand: `${launcher} fleet-status --node ${node.id} --json`,
+    sync: {
+      command: `${launcher} fleet-repair --node ${node.id} --repair workspace`,
+      steps: syncPlan(false),
+    },
+  };
+}

package/src/platform/ansible-receipt.ts

@@ -0,0 +1,128 @@
+import { createHash } from 'node:crypto';
+import { mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join, relative } from 'node:path';
+import { runAnsible } from './ansible';
+import type { CommandOptions, CommandResult, Runner } from '../types';
+
+/** Full re-apply at least this often, so drift a cheap check misses still heals. */
+export const RECEIPT_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+
+export interface Receipt {
+  /** Stable receipt name, e.g. `ubuntu-baseline`. */
+  name: string;
+  /** Files or directories whose content the playbook result depends on. */
+  sources: string[];
+  /** Cheap postcondition probe; a skip happens only while it passes. */
+  verify: () => Promise<boolean>;
+  /** Ignore any receipt and apply in full. */
+  force?: boolean;
+  /** Receipt directory; defaults to the user's fleet state. */
+  directory?: string;
+  now?: number;
+}
+
+export interface ReceiptResult extends CommandResult {
+  /** True when the playbook was skipped on an unchanged, verified receipt. */
+  cached: boolean;
+}
+
+export function receiptDirectory(home = homedir()): string {
+  return join(home, '.local/state/personal/fleet/receipts');
+}
+
+/** Ansible is idempotent but slow (fact gathering, package and service
+ * inspection). A successful apply records a digest of its exact inputs
+ * (playbook sources and variables, never the variables themselves). A later
+ * run with the same digest skips the playbook while the cheap postcondition
+ * still holds; any input change, failed probe, failed apply or a receipt
+ * older than a day applies in full again.
+ */
+export async function runAnsibleWithReceipt(
+  run: Runner,
+  playbook: string,
+  options: CommandOptions & { input: string },
+  receipt: Receipt
+): Promise<ReceiptResult> {
+  const directory = receipt.directory ?? receiptDirectory();
+  const path = join(directory, `${receipt.name}.json`);
+  const now = receipt.now ?? Date.now();
+  const key = await inputKey(playbook, options.input, receipt.sources);
+  if (
+    !receipt.force &&
+    key !== null &&
+    (await fresh(path, key, now)) &&
+    (await receipt.verify().catch(() => false))
+  )
+    return { ok: true, reason: 'exit', stdout: '', stderr: '', cached: true };
+  await rm(path, { force: true }).catch(() => undefined);
+  const result = await runAnsible(run, playbook, options);
+  if (result.ok && key !== null)
+    await record(directory, path, { version: 1, key, appliedAt: now });
+  return { ...result, cached: false };
+}
+
+/** Best effort: a missing receipt only means the next run applies in full. */
+async function record(directory: string, path: string, value: object) {
+  try {
+    await mkdir(directory, { recursive: true, mode: 0o700 });
+    await writeFile(path, JSON.stringify(value) + '\n', { mode: 0o600 });
+  } catch {
+    /* Never fail a successful apply over its receipt. */
+  }
+}
+
+async function fresh(path: string, key: string, now: number) {
+  try {
+    const value = JSON.parse(await readFile(path, 'utf8')) as {
+      version?: unknown;
+      key?: unknown;
+      appliedAt?: unknown;
+    };
+    return (
+      value.version === 1 &&
+      value.key === key &&
+      typeof value.appliedAt === 'number' &&
+      now - value.appliedAt >= 0 &&
+      now - value.appliedAt < RECEIPT_MAX_AGE_MS
+    );
+  } catch {
+    return false;
+  }
+}
+
+/** Null when any source is unreadable: then the playbook always runs. */
+export async function inputKey(
+  playbook: string,
+  input: string,
+  sources: string[]
+): Promise<string | null> {
+  const hash = createHash('sha256');
+  hash.update(`playbook\0${playbook}\0input\0${input}\0`);
+  try {
+    for (const source of [...sources].sort())
+      for (const file of await files(source)) {
+        hash.update(`${relative(source, file)}\0`);
+        hash.update(await readFile(file));
+        hash.update('\0');
+      }
+  } catch {
+    return null;
+  }
+  return hash.digest('hex');
+}
+
+async function files(source: string): Promise<string[]> {
+  const entries = await readdir(source, {
+    recursive: true,
+    withFileTypes: true,
+  }).catch((error: unknown) => {
+    if ((error as NodeJS.ErrnoException).code === 'ENOTDIR') return null;
+    throw error;
+  });
+  if (entries === null) return [source];
+  return entries
+    .filter((entry) => entry.isFile())
+    .map((entry) => join(entry.parentPath, entry.name))
+    .sort();
+}