@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/monitoring/remote.ts

@@ -0,0 +1,65 @@
+import type { Check, FleetNode, Runner } from '../types';
+import { shellQuote, sshArgs, sshFailure } from '../ssh';
+import { validCheck, cleanCheck } from '../protocol';
+import { record } from '../config';
+import { isRevision } from '../machine/state';
+import type { AgentEnrollment } from './enroll';
+import { remoteLockPrelude } from '../remote-lock';
+
+/** Installs the Beszel agent on a worker. Only node-specific agent credentials
+ * cross SSH; hub account credentials stay on the manager. Workers never host
+ * the hub (it runs only on the manager), so there is no remote hub mode.
+ * The installed source and remote operation lock are reused.
+ */
+export async function installRemoteAgent(
+  node: FleetNode,
+  revision: string,
+  enrollment: AgentEnrollment,
+  run: Runner
+): Promise<Check[]> {
+  if (!isRevision(revision)) throw new Error('BESZEL_REVISION');
+  const root = `"$HOME/.local/share/personal/fleet/revisions/${revision}"`;
+  const locked = shellQuote(
+    JSON.stringify({
+      version: 1,
+      checks: [
+        {
+          id: 'beszel:lock',
+          status: 'fail',
+          summary: 'FLEET_REMOTE_LOCKED',
+          remedy: {
+            action:
+              'Another remote operation owns the lock. Inspect its state and retry after it finishes; never delete the lock or start a competing installer.',
+          },
+        },
+      ],
+    })
+  );
+  const command = `cd ${root} && { ${remoteLockPrelude()} || { printf '%s\\n' ${locked}; exit 0; }; } && trap '' HUP && exec "$HOME/.local/bin/mise" exec --locked bun -- bun ${shellQuote('src/monitoring/apply.ts')}`;
+  const result = await run([...sshArgs(node), command], {
+    input: JSON.stringify({ ...enrollment, node: node.id, revision }),
+    timeoutMs: 360_000,
+  });
+  if (!result.ok) return [sshFailure(result)];
+  try {
+    const value: unknown = JSON.parse(result.stdout);
+    if (
+      record(value) &&
+      value['version'] === 1 &&
+      Array.isArray(value['checks']) &&
+      value['checks'].length > 0 &&
+      value['checks'].length <= 20 &&
+      value['checks'].every(validCheck)
+    )
+      return value['checks'].map(cleanCheck);
+  } catch {
+    /* Never expose native output or credentials. */
+  }
+  return [
+    {
+      id: 'beszel:protocol',
+      status: 'fail',
+      summary: 'BESZEL_REMOTE_EVIDENCE_INVALID',
+    },
+  ];
+}

package/src/network-definition.ts

@@ -0,0 +1,83 @@
+import { type FleetDefinition, type MachineDefinition } from './definition';
+import { record } from './config';
+import { FleetUsageError } from './errors';
+
+export interface NetworkHost {
+  alias: string;
+  tailscaleName: string;
+  sshPort: number;
+  user: string;
+  identityFile: string;
+  os: 'linux' | 'macOS' | 'windows';
+  t3Port: number;
+}
+export function networkProjection(
+  definition: FleetDefinition,
+  identities: Record<string, string>
+): NetworkHost[] {
+  return definition.workers.flatMap((node) =>
+    node.network
+      ? [
+          {
+            alias: node.ssh,
+            ...node.network,
+            os: node.os,
+            identityFile: identities[node.id] ?? '',
+          },
+        ]
+      : []
+  );
+}
+function compatible(
+  node: MachineDefinition | undefined,
+  host: Record<string, unknown>
+): node is MachineDefinition {
+  return Boolean(
+    node?.network &&
+    Object.entries(node.network).every(([key, value]) => host[key] === value) &&
+    node.os === host['os']
+  );
+}
+/** Immutable migration input supplies old manager paths only, never desired hosts. */
+export function legacyIdentities(
+  definition: FleetDefinition,
+  raw: unknown
+): Record<string, string> {
+  if (!record(raw) || !Array.isArray(raw['hosts']))
+    throw new FleetUsageError('Invalid legacy Workstation inventory.');
+  const result: Record<string, string> = {};
+  for (const host of raw['hosts']) {
+    if (!record(host))
+      throw new FleetUsageError('Invalid legacy network host.');
+    const node = definition.workers.find((node) => node.ssh === host['alias']);
+    // A retired host has no manager identity to migrate into this fleet.
+    if (!node) continue;
+    if (!compatible(node, host))
+      throw new FleetUsageError(
+        'Legacy Workstation and canonical Fleet definitions conflict. Review IDs, aliases and network settings before migration.'
+      );
+    const identity = host['identityFile'];
+    if (
+      typeof identity !== 'string' ||
+      !/^(~\/|\/)[^\x00-\x1f\x7f]+$/.test(identity)
+    )
+      throw new FleetUsageError('Legacy identity path is invalid.');
+    if (result[node.id] !== undefined)
+      throw new FleetUsageError('Duplicate legacy machine assignment.');
+    result[node.id] = identity;
+  }
+  return result;
+}
+/** `legacy` is the consumer's earlier inventory, when it has one: it only
+ * supplies identity paths the fleet configuration does not hold.
+ */
+export function loadNetworkDefinition(
+  definition: FleetDefinition,
+  legacy: unknown,
+  overrides?: Record<string, string>
+): NetworkHost[] {
+  return networkProjection(
+    definition,
+    overrides ?? legacyIdentities(definition, legacy)
+  );
+}

package/src/node-converge-linux.ts

@@ -0,0 +1,374 @@
+import { workerReadiness } from './worker-readiness';
+import { runAnsible } from './platform/ansible';
+import { runAnsibleWithReceipt } from './platform/ansible-receipt';
+import { loadedT3Policy, stageT3Policy } from './platform/t3-policy';
+import { verifyUbuntuBoot } from './platform/ubuntu-verify';
+import { verifyResourceSafety } from './platform/resource-verify';
+import { convergeT3Route, T3_ROUTE } from './platform/t3-route';
+import { convergeProxy } from './proxy-converge';
+import { providerNames } from './t3/names-check';
+import { machineProviders } from './t3/providers';
+import { quotaLimited, verifyFleetProviders } from './agents/fleet-verify';
+import { verifyNativeAgents } from './agents/native-verify';
+import { managedT3, managedT3Desktop } from '../bootstrap/native-t3';
+import { convergeT3Service, t3StartedAt } from './platform/t3-service';
+import { userInfo } from 'node:os';
+import { resolve } from 'node:path';
+import { auditUbuntu } from './platform/ubuntu-audit';
+import {
+  applyAgentProfiles,
+  attempt,
+  configureWorkstation,
+} from './node-converge-shared';
+import type { Check, Runner } from './types';
+
+/** Ubuntu worker convergence: Ansible owns OS policy, services and recovery; the
+ * native T3, proxy and provider owners run only after the baseline passes.
+ */
+export async function convergeLinux(
+  root: string,
+  target: { node: string; revision: string },
+  run: Runner
+): Promise<Check[]> {
+  const { node, revision } = target;
+  const audit = await auditUbuntu(run);
+  if (!audit.supported || !audit.passwordlessSudo)
+    return [
+      {
+        id: 'ubuntu:prerequisites',
+        status: 'fail',
+        summary: audit.supported
+          ? 'SUDO_AUTHORIZATION_REQUIRED'
+          : 'UBUNTU_BASELINE_MISMATCH',
+        remedy: {
+          action:
+            'Authorize designated-user sudo through the manager enrollment flow; unsupported releases require an explicit migration.',
+        },
+      },
+    ];
+  const checks: Check[] = [];
+  checks.push(
+    ...(await attempt('tailscale:configuration', async () => [
+      await applyTailscale(root, run),
+    ]))
+  );
+  const { baseline, verified } = await baselineStep(root, run);
+  checks.push(...baseline, ...verified);
+  checks.push(...(await t3AfterBaseline(root, { node, baseline, run })));
+  checks.push(
+    ...(await attempt('workstation:configuration', () =>
+      configureWorkstation(root, run)
+    ))
+  );
+  checks.push(
+    ...(await attempt('configuration:apply', () =>
+      applyAgentProfiles(root, node, revision, run)
+    ))
+  );
+  checks.push(
+    ...(await attempt('proxy:configuration', () =>
+      convergeProxy(node, revision, run)
+    ))
+  );
+  const providers = await attempt('provider:verification', () =>
+    verifyFleetProviders(node)
+  );
+  checks.push(...providers);
+  checks.push(
+    ...(await attempt('agent:verification', () =>
+      verifyNativeAgents(node, root, run, {
+        outOfQuota: quotaLimited(providers),
+        // Where T3 starts them: the environment of its user service.
+        t3Service: true,
+      })
+    ))
+  );
+  checks.push(
+    ...(await attempt('worker:readiness', () => workerReadiness(run, node)))
+  );
+  return checks;
+}
+
+/** Applies the baseline (skipped on an unchanged, verified receipt), then
+ * measures boot and resource policy. Self-healing: if a skipped baseline's
+ * effective policy no longer verifies, the baseline is applied in full and
+ * measured again in the same run.
+ */
+async function baselineStep(
+  root: string,
+  run: Runner
+): Promise<{ baseline: Check[]; verified: Check[] }> {
+  const measure = async () => [
+    ...(await attempt('ubuntu:boot-policy', () =>
+      verifyUbuntuBoot(userInfo().username, run)
+    )),
+    ...(await attempt('ubuntu:resource-safety', () =>
+      verifyResourceSafety(userInfo().homedir, run)
+    )),
+  ];
+  let cached = false;
+  let baseline = await attempt('ubuntu:baseline', async () => {
+    const applied = await applyBaseline(root, run, false);
+    cached = applied.cached;
+    return applied.checks;
+  });
+  let verified = await measure();
+  if (cached && verified.some((check) => check.status === 'fail')) {
+    baseline = await attempt('ubuntu:baseline', async () => {
+      return (await applyBaseline(root, run, true)).checks;
+    });
+    verified = await measure();
+  }
+  return { baseline, verified };
+}
+
+async function applyBaseline(
+  root: string,
+  run: Runner,
+  force: boolean
+): Promise<{ cached: boolean; checks: Check[] }> {
+  const user = userInfo();
+  const ubuntu = resolve(root, 'ansible/ubuntu');
+  const result = await runAnsibleWithReceipt(
+    run,
+    resolve(ubuntu, 'baseline.yml'),
+    {
+      cwd: ubuntu,
+      timeoutMs: 900_000,
+      input: JSON.stringify({
+        development_source: root,
+        development_user: user.username,
+        development_uid: user.uid,
+        development_home: user.homedir,
+      }),
+    },
+    {
+      name: 'ubuntu-baseline',
+      sources: [ubuntu],
+      force,
+      verify: () => baselineServicesActive(run),
+    }
+  );
+  return {
+    cached: result.cached,
+    checks: [
+      {
+        id: 'ubuntu:baseline',
+        status: result.ok ? 'pass' : 'fail',
+        summary: result.ok
+          ? result.cached
+            ? 'Ubuntu baseline unchanged since its last successful apply and its services are active (re-applied in full at least daily).'
+            : 'Ubuntu baseline tasks completed, including always-on service recovery; effective boot/session policies require separate verification.'
+          : 'UBUNTU_BASELINE_INCOMPLETE',
+      },
+    ],
+  };
+}
+
+/** Cheap probe for what the baseline owns: system services and user timers. */
+async function baselineServicesActive(run: Runner): Promise<boolean> {
+  const system = await run([
+    'systemctl',
+    'is-active',
+    '--quiet',
+    'ssh.service',
+    'tailscaled.service',
+    'docker.service',
+  ]);
+  if (!system.ok) return false;
+  const user = await run([
+    'systemctl',
+    '--user',
+    'is-active',
+    '--quiet',
+    'personal-worker-auth.timer',
+  ]);
+  return user.ok;
+}
+
+async function applyTailscale(root: string, run: Runner): Promise<Check> {
+  const ubuntu = resolve(root, 'ansible/ubuntu');
+  const tailscale = await runAnsibleWithReceipt(
+    run,
+    'tailscale.yml',
+    { cwd: ubuntu, timeoutMs: 300_000, input: '{}' },
+    {
+      name: 'ubuntu-tailscale',
+      sources: [ubuntu],
+      verify: async () => {
+        const status = await run(['tailscale', 'status', '--json']);
+        return (
+          status.ok &&
+          (JSON.parse(status.stdout) as { BackendState?: unknown })
+            .BackendState === 'Running'
+        );
+      },
+    }
+  );
+  return {
+    id: 'tailscale:configuration',
+    status: tailscale.ok ? 'pass' : 'fail',
+    summary: tailscale.ok
+      ? 'Pinned native Tailscale service is enabled, running and authenticated; manager reachability remains separate.'
+      : 'TAILSCALE_SETUP_OR_LOGIN_REQUIRED',
+  };
+}
+
+async function convergeT3(
+  root: string,
+  node: string,
+  run: Runner
+): Promise<Check[]> {
+  const artifact = managedT3();
+  const startedAt = await t3StartedAt(run);
+  const checks = await convergeT3Service(
+    artifact,
+    run,
+    async () => {
+      const staged = await stageT3Policy(root, artifact.path, run, node);
+      return staged.status === 'pass' ? [] : [staged];
+    },
+    () => installT3Desktop(root, run)
+  );
+  if (checks.every((check) => check.status === 'pass'))
+    checks.push(
+      await loadedT3Policy(artifact.path, run, {
+        restarted: startedAt !== (await t3StartedAt(run)),
+      })
+    );
+  // Settings are watched by T3: providers arrive and stale ones leave at once.
+  checks.push(await machineProviders(node, root, true));
+  // Names live in the same settings, so they conform without a restart.
+  checks.push(await providerNames(node, root, true));
+  const serviceReady = checks.some(
+    (check) => check.id === 't3:service' && check.status === 'pass'
+  );
+  if (serviceReady)
+    checks.push(
+      ...(await convergeT3Route(run, () => publishT3Route(root, run)))
+    );
+  if (serviceReady) checks.push(...(await enableWorkerRecovery(root, run)));
+  return checks;
+}
+
+async function publishT3Route(root: string, run: Runner): Promise<boolean> {
+  const result = await runAnsible(run, 't3-route.yml', {
+    cwd: resolve(root, 'ansible/ubuntu'),
+    input: JSON.stringify({ t3_route_target: T3_ROUTE.proxy }),
+    timeoutMs: 60_000,
+  });
+  return result.ok;
+}
+
+async function installT3Desktop(root: string, run: Runner): Promise<boolean> {
+  const user = userInfo();
+  const desktop = managedT3Desktop();
+  const ubuntu = resolve(root, 'ansible/ubuntu');
+  const result = await runAnsibleWithReceipt(
+    run,
+    resolve(ubuntu, 't3-desktop.yml'),
+    {
+      cwd: root,
+      timeoutMs: 600_000,
+      input: JSON.stringify({
+        development_source: root,
+        development_user: user.username,
+        development_uid: user.uid,
+        development_home: user.homedir,
+        t3_desktop_url: desktop.url,
+        t3_desktop_name: desktop.name,
+        t3_desktop_sha256: desktop.sha256,
+      }),
+    },
+    {
+      name: 'ubuntu-t3-desktop',
+      sources: [ubuntu],
+      // A stopped or disabled app reruns the play, which starts it.
+      verify: async () =>
+        (
+          await run([
+            'systemctl',
+            '--user',
+            'is-enabled',
+            '--quiet',
+            't3code-desktop.service',
+          ])
+        ).ok &&
+        (
+          await run([
+            'systemctl',
+            '--user',
+            'is-active',
+            '--quiet',
+            't3code-desktop.service',
+          ])
+        ).ok,
+    }
+  );
+  return result.ok;
+}
+
+async function t3AfterBaseline(
+  root: string,
+  context: { node: string; baseline: Check[]; run: Runner }
+): Promise<Check[]> {
+  const { node, baseline, run } = context;
+  if (baseline.some((check) => check.status !== 'pass'))
+    return [
+      {
+        id: 't3:prerequisites',
+        status: 'skip',
+        summary: 'T3_BASELINE_PREREQUISITES_REQUIRED',
+        remedy: {
+          action:
+            'Complete the designated-user baseline and lingering policy before changing native service lifecycle. Existing service state is preserved.',
+        },
+      },
+    ];
+  return attempt('t3:service', () => convergeT3(root, node, run));
+}
+
+async function enableWorkerRecovery(
+  root: string,
+  run: Runner
+): Promise<Check[]> {
+  const user = userInfo();
+  const ubuntu = resolve(root, 'ansible/ubuntu');
+  const result = await runAnsibleWithReceipt(
+    run,
+    resolve(ubuntu, 'recovery.yml'),
+    {
+      cwd: root,
+      timeoutMs: 60_000,
+      input: JSON.stringify({
+        development_source: root,
+        development_user: user.username,
+        development_uid: user.uid,
+        development_home: user.homedir,
+      }),
+    },
+    {
+      name: 'ubuntu-recovery',
+      sources: [ubuntu],
+      verify: async () =>
+        (
+          await run([
+            'systemctl',
+            '--user',
+            'is-active',
+            '--quiet',
+            'personal-worker-recovery.timer',
+          ])
+        ).ok,
+    }
+  );
+  return [
+    {
+      id: 'worker:service-recovery',
+      status: result.ok ? 'pass' : 'fail',
+      summary: result.ok
+        ? 'T3 recovery enabled: a stopped desktop app is started and an unresponsive one restarted after three failed one-minute health checks.'
+        : 'WORKER_RECOVERY_SETUP_FAILED',
+    },
+  ];
+}

package/src/node-converge-shared.ts

@@ -0,0 +1,83 @@
+import { fingerprint, writeReceipt } from './fingerprint';
+import { configureWorkspace } from './configure';
+import { stableWorkspace } from './platform/t3-policy';
+import type { Check, Runner } from './types';
+
+/** Phase wrapper: one failure code per phase, durations measured, no exception text. */
+export async function attempt(
+  id: string,
+  operation: () => Promise<Check[]>
+): Promise<Check[]> {
+  const started = Date.now();
+  try {
+    const checks = await operation();
+    const phaseDurationMs = Date.now() - started;
+    return checks.map((check) => ({ ...check, phaseDurationMs }));
+  } catch {
+    return [
+      {
+        id,
+        status: 'fail',
+        summary: 'REMOTE_PHASE_FAILED',
+        phaseDurationMs: Date.now() - started,
+        remedy: {
+          action:
+            'Inspect the named owner and approved source revision. Prior phase evidence is retained; no exception, environment or auth response is logged.',
+        },
+      },
+    ];
+  }
+}
+
+/** The `devfleet` launcher (linked to the applied revision, never copied) and
+ * this worker's own credential refresh. Identical on Linux and macOS.
+ */
+export async function configureWorkstation(
+  root: string,
+  run: Runner
+): Promise<Check[]> {
+  const installed = await run(
+    // The stable path of the applied revision: it survives pruning.
+    ['sh', 'bootstrap/install-launcher.sh', await stableWorkspace(root)],
+    { cwd: root, timeoutMs: 30_000 }
+  );
+  if (!installed.ok)
+    return [
+      {
+        id: 'workstation:installation',
+        status: 'fail',
+        summary: 'LAUNCHER_INSTALLATION_FAILED',
+      },
+    ];
+  // This receipt certifies only frozen dependency inputs, never machine readiness.
+  await writeReceipt(root, await fingerprint(root, run));
+  await run(['bun', 'bootstrap/worker-auth.ts'], {
+    cwd: root,
+    timeoutMs: 90_000,
+  });
+  return [
+    {
+      id: 'workstation:configuration',
+      status: 'pass',
+      summary:
+        'The devfleet launcher is linked to the applied revision; authentication remains a separate gate.',
+    },
+  ];
+}
+
+export function applyAgentProfiles(
+  root: string,
+  node: string,
+  revision: string,
+  run: Runner
+): Promise<Check[]> {
+  return configureWorkspace(
+    root,
+    { node, revision, tool: 'codex', profile: 'proxy' },
+    run,
+    {
+      additionalTools: ['claude', 'gemini', 'opencode', 'grok', 'antigravity'],
+      bootstrappedDeployment: true,
+    }
+  );
+}

package/src/node-converge.ts

@@ -0,0 +1,40 @@
+import { runLockedCommand as runCommand } from './runner';
+import { isRevision } from './machine/state';
+import { convergeLinux } from './node-converge-linux';
+import type { Check } from './types';
+
+/** Internal remote entry, run on a worker only. The manager owns the operation
+ * lock and source selection; this installs no pull daemon or timer. Never
+ * prints child output. Workers are Linux.
+ */
+async function main(): Promise<Check[]> {
+  const [node, revision] = process.argv.slice(2);
+  if (!node || !/^[a-z][a-z0-9-]{0,62}$/.test(node) || !isRevision(revision))
+    throw new Error();
+  const root = process.cwd();
+  const head = await runCommand(['git', 'rev-parse', 'HEAD']);
+  if (!head.ok || head.stdout.trim() !== revision) throw new Error();
+  if (process.platform === 'linux')
+    return convergeLinux(root, { node, revision }, runCommand);
+  return [
+    {
+      id: 'platform:unsupported',
+      status: 'fail',
+      summary: 'PLATFORM_UNSUPPORTED',
+      remedy: {
+        action:
+          'Workers are Ubuntu 26.04. The macOS manager converges itself locally in fleet-sync.',
+      },
+    },
+  ];
+}
+if (import.meta.main) {
+  const checks = await main().catch((): Check[] => [
+    {
+      id: 'configuration:operation',
+      status: 'fail',
+      summary: 'REMOTE_CONFIGURATION_INCOMPLETE',
+    },
+  ]);
+  process.stdout.write(JSON.stringify({ version: 1, checks }));
+}

package/src/onboard.ts

@@ -0,0 +1,35 @@
+import { shellQuote } from './ssh';
+import { syncPlan } from './sync';
+import type { FleetNode } from './types';
+
+export function onboarding(node: FleetNode, launcher = 'devfleet'): object {
+  return {
+    version: 1,
+    node: node.id,
+    applied: false,
+    steps: [
+      'On the new node, review and run bootstrap/bootstrap.sh (or bootstrap.ps1 for native Windows). Use docs/machine-setup.md. This installs the shared locked toolchain; it does not enable SSH.',
+      'Enable the OS SSH service as the operator on a trusted private network. Do not expose SSH publicly or create cloud resources.',
+      'Create a distinct per-machine SSH key and authorize its PUBLIC key on the destination. Verify host fingerprints out of band. Keep private keys and auth tokens on their original machines.',
+      'In the manager SSH config, configure this alias with the real HostName, User, IdentityFile, optional Port/ProxyJump and IdentitiesOnly yes. Fleet will not write or overwrite SSH configuration.',
+      'Test ssh ALIAS manually once after fingerprint verification. Ensure noninteractive shell startup writes no stdout. Set runtime to an absolute Node/Bun executable/shim if SSH PATH omits it.',
+      'Authenticate separately on the node: gh auth login; gh auth setup-git; vault login using the approved method. Configure coding-provider sessions independently; never copy auth directories.',
+      'For T3 Connect, review t3 status and t3 service status, then complete t3 up --connect and operator pairing/linking. T3 pairing/auth state is not an SSH credential.',
+      'For VS Code, install Remote-SSH in the LOCAL editor profile, use the same SSH alias/client/config, enable the login terminal temporarily, and read Output → Remote - SSH. Restart the editor after PATH/agent changes.',
+      'VS Code needs TCP forwarding and compatible remote server prerequisites. Test with ssh -N -L 127.0.0.1:18080:127.0.0.1:REMOTE_PORT ALIAS against a known loopback service, then curl localhost:18080 from a second terminal. Review server policy; never loosen it automatically.',
+      'Run devfleet fleet-status --node NODE --json from every intended manager. This is directed connectivity, not an all-pairs proof. Review skipped T3/VS Code end-to-end checks manually.',
+    ],
+    sshTemplate: `Host ${node.ssh}\n  HostName REPLACE_WITH_PRIVATE_HOST\n  User REPLACE_WITH_USER\n  IdentityFile ~/.ssh/REPLACE_WITH_NODE_KEY\n  IdentitiesOnly yes\n  ForwardAgent no\n`,
+    vscodeSettings: {
+      'remote.SSH.path': 'REPLACE_WITH_SAME_OPENSSH_BINARY',
+      'remote.SSH.configFile': 'REPLACE_WITH_ABSOLUTE_SSH_CONFIG',
+      'remote.SSH.showLoginTerminal': true,
+    },
+    workspaceCommand: `cd -- ${shellQuote(node.workspace)}`,
+    verifyCommand: `${launcher} fleet-status --node ${node.id} --json`,
+    sync: {
+      command: `${launcher} fleet-repair --node ${node.id} --repair workspace`,
+      steps: syncPlan(false),
+    },
+  };
+}