@crvouga/dev-fleet 0.0.0-stage → 0.1.0

package/src/test-fixtures.ts

@@ -0,0 +1,36 @@
+import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { CommandResult, Runner, FleetNode } from './types';
+
+export const HEAD = 'a'.repeat(40);
+export const NODE: FleetNode = {
+  id: 'dev-one',
+  ssh: 'dev-one',
+  workspace: '/home/dev/workspace',
+  required: [],
+};
+export const success = (stdout = ''): CommandResult => ({
+  ok: true,
+  stdout,
+  stderr: '',
+  reason: 'exit',
+});
+export const failure = (stderr = ''): CommandResult => ({
+  ok: false,
+  stdout: '',
+  stderr,
+  reason: 'exit',
+});
+export const gitRunner: Runner = async (command) =>
+  success(command.includes('rev-parse') ? HEAD : '');
+
+export async function workspace(): Promise<string> {
+  const root = await mkdtemp(join(tmpdir(), "fleet test's "));
+  for (const file of ['package.json', 'bun.lock', 'mise.toml', 'mise.lock'])
+    await writeFile(join(root, file), '{}');
+  await mkdir(join(root, 'packages/example'), { recursive: true });
+  await writeFile(join(root, 'packages/example/package.json'), '{}');
+  await mkdir(join(root, '.mise/locks'), { recursive: true });
+  return root;
+}

package/src/toolchain-checks.ts

@@ -0,0 +1,83 @@
+import { join } from 'node:path';
+import { record } from './config';
+import type { Check, Runner } from './types';
+
+export async function toolchainChecks(
+  root: string,
+  run: Runner,
+  worker?: boolean
+): Promise<Check[]> {
+  const result = await run(doctorArguments(root, worker), {
+    cwd: root,
+    timeoutMs: 90_000,
+  });
+  const fallback: Check = {
+    id: 'auth:toolchain',
+    status: 'fail',
+    summary: 'Pinned tool/dependency and Docker checks could not be verified.',
+    remedy: {
+      action:
+        'Run the checkout bootstrap/doctor.ts --json --docker on this computer. Reconcile versions only through the shared mise setup.',
+    },
+  };
+  try {
+    const data: unknown = JSON.parse(result.stdout);
+    if (
+      !record(data) ||
+      typeof data['ok'] !== 'boolean' ||
+      !Array.isArray(data['checks']) ||
+      !data['checks'].length ||
+      data['checks'].length > 150
+    )
+      return [fallback];
+    const checks = data['checks'].map(toolCheck);
+    if (checks.some((check) => check === null)) return [fallback];
+    return [
+      ...(!result.ok || !data['ok'] ? [fallback] : []),
+      ...(checks as Check[]),
+    ];
+  } catch {
+    return [fallback];
+  }
+}
+
+function toolCheck(value: unknown): Check | null {
+  if (
+    !record(value) ||
+    typeof value['id'] !== 'string' ||
+    !/^[a-zA-Z0-9@:/._-]{1,75}$/.test(value['id']) ||
+    !['pass', 'fail', 'warn', 'skip'].includes(String(value['status']))
+  )
+    return null;
+  const ok = value['status'] === 'pass';
+  const version =
+    typeof value['detail'] === 'string' &&
+    /^v?\d{1,6}(?:\.\d{1,6}){1,3}$/.test(value['detail'])
+      ? ` ${value['detail']}`
+      : '';
+  return {
+    id: `tool:${value['id'].toLowerCase().replace(/[^a-z0-9:._-]/g, '_')}`,
+    status: value['status'] as Check['status'],
+    summary: ok
+      ? `Expected tool version or runtime check succeeded${version}.`
+      : 'Expected tool version, runtime or authentication check did not pass.',
+    ...(!ok
+      ? {
+          remedy: {
+            action:
+              'Review bootstrap/doctor.ts --json --docker locally. Use shared mise/workstation setup to reconcile this computer.',
+          },
+        }
+      : {}),
+  };
+}
+
+function doctorArguments(root: string, worker?: boolean): string[] {
+  return [
+    'bun',
+    join(root, 'bootstrap/doctor.ts'),
+    '--json',
+    '--docker',
+    ...(worker ? ['--worker'] : []),
+  ];
+}

package/src/topology.ts

@@ -0,0 +1,24 @@
+import type { FleetDefinition } from './definition';
+
+/** Fleet topology: one singleton manager (the human's machine) and stateless
+ * Linux workers. Every singleton service runs on the manager; workers only
+ * consume them and can be turned off, replaced or re-imaged at will. */
+export type Role = 'manager' | 'worker';
+
+export function roleOf(definition: FleetDefinition, id: string): Role | null {
+  if (definition.manager.id === id) return 'manager';
+  return definition.workers.some((entry) => entry.id === id) ? 'worker' : null;
+}
+
+/** Private Beszel hub URL on the manager; agents never use the public name. */
+export function hubUrl(definition: FleetDefinition): string {
+  return `https://${definition.manager.network.tailscaleName}:${definition.services.monitoring.hubPort}`;
+}
+
+/** Normalizes a macOS LocalHostName / hostname to a fleet ID for identity checks. */
+export function hostId(hostname: string): string {
+  return hostname
+    .toLowerCase()
+    .replace(/\.local$/, '')
+    .replace(/[^a-z0-9-]/g, '-');
+}

package/src/transport.ts

@@ -0,0 +1,122 @@
+import { managedProbeTarget } from './managed-source';
+import type { ConfigurationRequest } from './configure';
+import { remoteCommand, sshFailure, sshArgs } from './ssh';
+import { cleanCheck, decodeSnapshot, validCheck } from './protocol';
+import { record } from './config';
+import { fileURLToPath } from 'node:url';
+import type { FleetNode, NodeReport, Runner, Fingerprint } from './types';
+import type { AutomatedRepair } from './repair-catalog';
+
+export async function bundleProbe(): Promise<string> {
+  const result = await Bun.build({
+    entrypoints: [fileURLToPath(new URL('./remote-entry.ts', import.meta.url))],
+    target: 'node',
+    minify: true,
+  });
+  if (!result.success || !result.outputs[0])
+    throw new Error('Cannot bundle the fleet probe.');
+  return result.outputs[0].text();
+}
+
+export interface RemoteOptions {
+  source: string;
+  timeoutMs: number;
+  action: 'probe' | 'sync' | 'repair' | 'configure';
+  configuration?: ConfigurationRequest;
+  repairs?: AutomatedRepair[];
+  expected?: Fingerprint;
+  pull?: boolean;
+}
+
+export async function inspectRemote(
+  node: FleetNode,
+  options: RemoteOptions,
+  run: Runner
+): Promise<NodeReport> {
+  if (node.required.includes('worker')) {
+    const preflight = await run([...sshArgs(node), 'true'], {
+      timeoutMs: 15_000,
+    });
+    if (!preflight.ok)
+      return { node: node.id, checks: [sshFailure(preflight)] };
+  }
+  const target =
+    options.action === 'probe' && options.expected
+      ? await managedProbeTarget(node, options.expected.head, run)
+      : { node, checks: [] };
+  const command = remoteCommand(target.node, {
+    action: options.action,
+    configuration: options.configuration,
+    node: target.node,
+    apply: options.action !== 'probe',
+    repairs: options.repairs,
+    expected: options.expected,
+    pull: options.pull === true,
+  });
+  const result = await run(command, {
+    input: options.source,
+    timeoutMs: options.timeoutMs,
+  });
+  if (!result.ok)
+    return { node: node.id, checks: [...target.checks, sshFailure(result)] };
+  if (options.action !== 'probe') return decodeSync(node.id, result.stdout);
+  const snapshot = decodeSnapshot(result.stdout);
+  if (!snapshot) return protocolFailure(node.id);
+  return {
+    node: node.id,
+    snapshot,
+    checks: [
+      ...target.checks,
+      {
+        id: 'ssh:session',
+        status: 'pass',
+        summary:
+          'Fresh noninteractive SSH session and Node-compatible probe succeeded with verified host keys and no agent forwarding.',
+      },
+      ...snapshot.checks,
+      {
+        id: 'vscode:forwarding',
+        status: 'skip',
+        required: node.required.includes('vscode'),
+        summary:
+          'VS Code Remote-SSH tunnel/server installation is not tested by a shell session. Use the onboarding forwarding test.',
+      },
+    ],
+  };
+}
+
+function decodeSync(node: string, output: string): NodeReport {
+  try {
+    const data: unknown = JSON.parse(output);
+    if (
+      record(data) &&
+      data['version'] === 1 &&
+      Array.isArray(data['checks']) &&
+      data['checks'].length > 0 &&
+      data['checks'].length <= 200 &&
+      data['checks'].every(validCheck)
+    ) {
+      return { node, checks: data['checks'].map(cleanCheck) };
+    }
+  } catch {
+    return protocolFailure(node);
+  }
+  return protocolFailure(node);
+}
+
+function protocolFailure(node: string): NodeReport {
+  return {
+    node,
+    checks: [
+      {
+        id: 'ssh:protocol',
+        status: 'fail',
+        summary: 'Remote response is not a supported fleet JSON document.',
+        remedy: {
+          action:
+            'Remove stdout banners from noninteractive shell startup; check the Node/Bun runtime. No raw remote output is retained.',
+        },
+      },
+    ],
+  };
+}

package/src/types.ts

@@ -0,0 +1,95 @@
+export type Status = 'pass' | 'warn' | 'fail' | 'skip';
+export type Capability =
+  | 'github'
+  | 'vault'
+  | 't3'
+  | 'vscode'
+  | 'toolchain'
+  | 'codex'
+  | 'claude'
+  | 'services'
+  | 'workstation'
+  | 'worker';
+
+export interface FleetNode {
+  id: string;
+  ssh: string;
+  workspace: string;
+  tailscaleId?: string;
+  t3EnvironmentId?: string;
+  bootstrapViaSsh?: boolean;
+  runtime?: string;
+  required: Capability[];
+}
+
+export interface FleetConfig {
+  version: 1;
+  nodes: FleetNode[];
+}
+
+export interface Check {
+  /** Explicitly out-of-scope checks do not affect required readiness. */
+  required?: boolean;
+  /** Measured enclosing phase duration; omitted when not observed. */
+  phaseDurationMs?: number;
+  id: string;
+  status: Status;
+  summary: string;
+  remedy?: { action: string; command?: string };
+}
+
+/** Only a required failure blocks readiness. Warnings (conditions outside the
+ * fleet, such as a provider quota) and skips (not observable here) are shown
+ * but never fail a run.
+ */
+export function blocking(check: Check): boolean {
+  return check.status === 'fail' && check.required !== false;
+}
+
+export interface Fingerprint {
+  head: string | null;
+  files: Record<string, string | null>;
+}
+
+export interface Snapshot {
+  version: 1;
+  platform: string;
+  fingerprint: Fingerprint;
+  checks: Check[];
+}
+
+export interface NodeReport {
+  node: string;
+  checks: Check[];
+  snapshot?: Snapshot;
+}
+
+export interface Report {
+  version: 1;
+  command: string;
+  ok: boolean;
+  generatedAt: string;
+  nodes: NodeReport[];
+  summary: Record<Status, number>;
+}
+
+export interface CommandOptions {
+  /** Internal remote worker: keep the already-held operation lock on fd 9. */
+  preserveOperationLock?: boolean;
+  cwd?: string;
+  input?: string;
+  timeoutMs?: number;
+  env?: NodeJS.ProcessEnv;
+}
+
+export interface CommandResult {
+  ok: boolean;
+  stdout: string;
+  stderr: string;
+  reason: 'exit' | 'missing' | 'timeout' | 'overflow';
+}
+
+export type Runner = (
+  command: string[],
+  options?: CommandOptions
+) => Promise<CommandResult>;

package/src/vault-config.ts

@@ -0,0 +1,38 @@
+import { fleet } from './fleet-config';
+import { FleetUsageError } from './errors';
+
+export type VaultConfig = {
+  addr: string;
+  mount: string;
+  project: string;
+  config: string;
+};
+
+/** True when this fleet keeps its secrets in Vault. Only then does each node
+ * hold its own Vault session, with the logins, renewal and checks that go
+ * with it; the other providers need none of that.
+ */
+export const usesVault = (): boolean => fleet.secrets.provider === 'vault';
+
+/** This fleet's Vault and the shared document (`<mount>/<project>/<config>`)
+ * that holds the proxy client key, from its `secrets` configuration.
+ */
+export function resolveVaultConfig(): VaultConfig {
+  const secrets = fleet.secrets;
+  if (secrets.provider !== 'vault')
+    throw new FleetUsageError(
+      'This fleet does not use the vault secrets provider.'
+    );
+  const [project = '', ...rest] =
+    secrets.locations['proxy-client-key'].path.split('/');
+  return {
+    addr: process.env['VAULT_ADDR']?.trim() || secrets.address,
+    mount: secrets.mount,
+    project,
+    config: rest.join('/'),
+  };
+}
+
+export function kvPath(vault: VaultConfig): string {
+  return `${vault.mount}/data/${vault.project}/${vault.config}`;
+}

package/src/worker-access-checks.ts

@@ -0,0 +1,115 @@
+import type { Check, Runner } from './types';
+import { fleet, secretVia } from './fleet-config';
+import { registry } from './agents/registry';
+import { resolveVaultConfig, usesVault } from './vault-config';
+import { record } from './config';
+
+/** Inject every subprocess; never put credentials in argv, reports, or logs. */
+export async function workerAccessChecks(run: Runner): Promise<Check[]> {
+  const policy = fleet.worker!;
+  const github = await run(['gh', 'api', 'user', '--jq', '.login']);
+  return [
+    {
+      id: 'worker:github-principal',
+      status:
+        github.ok && github.stdout.trim() === policy.githubLogin
+          ? 'pass'
+          : 'fail',
+      summary:
+        'Worker GitHub authentication must match the declared common account.',
+      remedy: {
+        action:
+          'Authenticate this worker with the declared GitHub account. Credentials stay on their original machines.',
+      },
+    },
+    await proxyAuthentication(run),
+    ...(usesVault() ? [await vaultRenewalCheck(run)] : []),
+  ];
+}
+async function proxyAuthentication(run: Runner): Promise<Check> {
+  const unavailable: Check = {
+    id: 'worker:proxy-authentication',
+    status: 'fail',
+    summary: 'UNATTENDED_PROXY_AUTHENTICATION_REQUIRED',
+    remedy: {
+      action:
+        'Restore this worker’s own Vault authorization and renewal; no other node’s login or gateway identity is copied.',
+    },
+  };
+  try {
+    const key = (await secretVia(run)('proxy-client-key')) ?? '';
+    if (!/^[\x21-\x7e]{16,4096}$/.test(key)) return unavailable;
+    const result = await run(
+      [
+        'curl',
+        '--fail',
+        '--silent',
+        '--show-error',
+        '--max-time',
+        '10',
+        '--header',
+        '@-',
+        `${registry.proxy.endpoint}/v1/models`,
+      ],
+      { input: `Authorization: Bearer ${key}\n`, timeoutMs: 15_000 }
+    );
+    const data: unknown = JSON.parse(result.stdout);
+    if (
+      !result.ok ||
+      !record(data) ||
+      !Array.isArray(data['data']) ||
+      !data['data'].length
+    )
+      return unavailable;
+    return {
+      ...unavailable,
+      status: 'pass',
+      summary:
+        'Worker credential authenticates to the declared singleton gateway model catalog.',
+    };
+  } catch {
+    return unavailable;
+  }
+}
+
+async function vaultRenewalCheck(run: Runner): Promise<Check> {
+  const unavailable: Check = {
+    id: 'worker:vault-renewal',
+    status: 'fail',
+    summary: 'UNATTENDED_VAULT_IDENTITY_REQUIRED',
+    remedy: {
+      action:
+        'Enroll this worker’s own periodic token or auto-auth identity. Ordinary expiring user sessions are not evidence of unattended renewal; no credentials are copied between machines.',
+    },
+  };
+  try {
+    const { addr } = resolveVaultConfig();
+    const result = await run([
+      'vault',
+      'token',
+      'lookup',
+      `-address=${addr}`,
+      '-format=json',
+    ]);
+    const data: unknown = JSON.parse(result.stdout);
+    if (!result.ok || !record(data) || !record(data['data']))
+      return unavailable;
+    const token = data['data'];
+    if (
+      token['renewable'] !== true ||
+      typeof token['period'] !== 'number' ||
+      token['period'] <= 0 ||
+      typeof token['ttl'] !== 'number' ||
+      token['ttl'] <= 1200
+    )
+      return unavailable;
+    return {
+      ...unavailable,
+      status: 'pass',
+      summary:
+        'This worker has a renewable periodic Vault identity with sufficient lifetime for the renewal timer.',
+    };
+  } catch {
+    return unavailable;
+  }
+}

package/src/worker-capacity.ts

@@ -0,0 +1,34 @@
+import { join } from 'node:path';
+import { healthChecks, parseProbe } from './machine-health';
+import type { Check, Runner } from './types';
+
+export async function workerCapacity(
+  root: string,
+  node: string,
+  run: Runner
+): Promise<Check[]> {
+  const result = await run(['sh', join(root, 'ansible/net/remote-probe.sh')], {
+    timeoutMs: 10_000,
+  });
+  const sample = parseProbe(result.stdout);
+  if (
+    !result.ok ||
+    sample.ncpu === undefined ||
+    sample.load === undefined ||
+    sample.memFreePct === undefined ||
+    sample.diskMb === undefined
+  )
+    return [
+      {
+        id: 'worker:capacity',
+        status: 'fail',
+        summary: 'Required CPU, memory and disk evidence is unavailable.',
+      },
+    ];
+  return healthChecks(sample, node).map((check) => ({
+    id: `worker:capacity:${check.id}`,
+    status: check.severity,
+    summary: check.detail,
+    ...(check.fix ? { remedy: { action: check.fix } } : {}),
+  }));
+}