@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/types.ts

@@ -0,0 +1,99 @@
+export type Status = 'pass' | 'warn' | 'fail' | 'skip';
+export type Capability =
+  | 'github'
+  | 'vault'
+  | 't3'
+  | 'vscode'
+  | 'toolchain'
+  | 'codex'
+  | 'claude'
+  | 'services'
+  | 'workstation'
+  | 'worker';
+
+export interface FleetNode {
+  id: string;
+  ssh: string;
+  workspace: string;
+  tailscaleId?: string;
+  t3EnvironmentId?: string;
+  bootstrapViaSsh?: boolean;
+  runtime?: string;
+  required: Capability[];
+}
+
+export interface FleetConfig {
+  version: 1;
+  nodes: FleetNode[];
+}
+
+export interface Check {
+  /** Explicitly out-of-scope checks do not affect required readiness. */
+  required?: boolean;
+  /** Measured enclosing phase duration; omitted when not observed. */
+  phaseDurationMs?: number;
+  id: string;
+  status: Status;
+  summary: string;
+  remedy?: { action: string; command?: string };
+}
+
+/** Only a required failure blocks readiness. Warnings (conditions outside the
+ * fleet, such as a provider quota) and skips (not observable here) are shown
+ * but never fail a run.
+ */
+export function blocking(check: Check): boolean {
+  return check.status === 'fail' && check.required !== false;
+}
+
+export interface Fingerprint {
+  head: string | null;
+  files: Record<string, string | null>;
+}
+
+export interface Snapshot {
+  version: 1;
+  platform: string;
+  fingerprint: Fingerprint;
+  checks: Check[];
+}
+
+export interface NodeReport {
+  node: string;
+  checks: Check[];
+  snapshot?: Snapshot;
+}
+
+export interface Report {
+  version: 1;
+  command: string;
+  ok: boolean;
+  generatedAt: string;
+  nodes: NodeReport[];
+  summary: Record<Status, number>;
+}
+
+export interface CommandOptions {
+  /** Internal remote worker: keep the already-held operation lock on fd 9. */
+  preserveOperationLock?: boolean;
+  cwd?: string;
+  input?: string;
+  timeoutMs?: number;
+  env?: NodeJS.ProcessEnv;
+  /** Run with `env` alone instead of on top of this process's environment:
+   * for a launch that must see only what a service would.
+   */
+  isolatedEnv?: boolean;
+}
+
+export interface CommandResult {
+  ok: boolean;
+  stdout: string;
+  stderr: string;
+  reason: 'exit' | 'missing' | 'timeout' | 'overflow';
+}
+
+export type Runner = (
+  command: string[],
+  options?: CommandOptions
+) => Promise<CommandResult>;

package/src/vault-config.ts

@@ -0,0 +1,38 @@
+import { fleet } from './fleet-config';
+import { FleetUsageError } from './errors';
+
+export type VaultConfig = {
+  addr: string;
+  mount: string;
+  project: string;
+  config: string;
+};
+
+/** True when this fleet keeps its secrets in Vault. Only then does each node
+ * hold its own Vault session, with the logins, renewal and checks that go
+ * with it; the other providers need none of that.
+ */
+export const usesVault = (): boolean => fleet.secrets.provider === 'vault';
+
+/** This fleet's Vault and the shared document (`<mount>/<project>/<config>`)
+ * that holds the proxy client key, from its `secrets` configuration.
+ */
+export function resolveVaultConfig(): VaultConfig {
+  const secrets = fleet.secrets;
+  if (secrets.provider !== 'vault')
+    throw new FleetUsageError(
+      'This fleet does not use the vault secrets provider.'
+    );
+  const [project = '', ...rest] =
+    secrets.locations['proxy-client-key'].path.split('/');
+  return {
+    addr: process.env['VAULT_ADDR']?.trim() || secrets.address,
+    mount: secrets.mount,
+    project,
+    config: rest.join('/'),
+  };
+}
+
+export function kvPath(vault: VaultConfig): string {
+  return `${vault.mount}/data/${vault.project}/${vault.config}`;
+}

package/src/worker-access-checks.ts

@@ -0,0 +1,168 @@
+import type { Check, Runner } from './types';
+import { fleet } from './fleet-config';
+import { secretsProvider } from './secrets';
+import { registry } from './agents/registry';
+import { resolveVaultConfig, usesVault } from './vault-config';
+import { record } from './config';
+
+/** Inject every subprocess; never put credentials in argv, reports, or logs. */
+export async function workerAccessChecks(run: Runner): Promise<Check[]> {
+  const policy = fleet.worker!;
+  const github = await run(['gh', 'api', 'user', '--jq', '.login']);
+  return [
+    {
+      id: 'worker:github-principal',
+      status:
+        github.ok && github.stdout.trim() === policy.githubLogin
+          ? 'pass'
+          : 'fail',
+      summary:
+        'Worker GitHub authentication must match the declared common account.',
+      remedy: {
+        action:
+          'Authenticate this worker with the declared GitHub account. Credentials stay on their original machines.',
+      },
+    },
+    await proxyAuthentication(run),
+    ...(usesVault() ? [await vaultRenewalCheck(run)] : []),
+  ];
+}
+/** Vault refuses a read the token's policy does not allow with a 403. */
+const PERMISSION_DENIED = /permission denied|Code: 403/i;
+
+async function proxyAuthentication(run: Runner): Promise<Check> {
+  // Read through the declared provider, remembering only whether Vault said
+  // the policy forbids it: never what it printed.
+  let denied = false;
+  const secret = secretsProvider(fleet.secrets, {
+    run: async (argv) => {
+      const result = await run(argv, { timeoutMs: 20_000 });
+      if (!result.ok && PERMISSION_DENIED.test(result.stderr)) denied = true;
+      return result.ok ? result.stdout : null;
+    },
+  });
+  const forbidden: Check = {
+    id: 'worker:proxy-authentication',
+    status: 'fail',
+    summary: 'VAULT_POLICY_PERMISSION_REQUIRED',
+    remedy: {
+      action:
+        'This worker’s Vault identity was refused the proxy client key by its policy. Grant that identity read access to the declared location in the Vault policy you manage. fleet-sync seeds no secret, writes no policy and adds no secret store.',
+    },
+  };
+  const unavailable: Check = {
+    id: 'worker:proxy-authentication',
+    status: 'fail',
+    summary: 'UNATTENDED_PROXY_AUTHENTICATION_REQUIRED',
+    remedy: {
+      action:
+        'Restore this worker’s own Vault authorization and renewal; no other node’s login or gateway identity is copied.',
+    },
+  };
+  try {
+    const key = (await secret('proxy-client-key')) ?? '';
+    if (denied) return forbidden;
+    if (!/^[\x21-\x7e]{16,4096}$/.test(key)) return unavailable;
+    const result = await run(
+      [
+        'curl',
+        '--fail',
+        '--silent',
+        '--show-error',
+        '--max-time',
+        '10',
+        '--header',
+        '@-',
+        `${registry.proxy.endpoint}/v1/models`,
+      ],
+      { input: `Authorization: Bearer ${key}\n`, timeoutMs: 15_000 }
+    );
+    const data: unknown = JSON.parse(result.stdout);
+    if (
+      !result.ok ||
+      !record(data) ||
+      !Array.isArray(data['data']) ||
+      !data['data'].length
+    )
+      return unavailable;
+    return {
+      ...unavailable,
+      status: 'pass',
+      summary:
+        'Worker credential authenticates to the declared singleton gateway model catalog.',
+    };
+  } catch {
+    return unavailable;
+  }
+}
+
+async function vaultRenewalCheck(run: Runner): Promise<Check> {
+  const unavailable: Check = {
+    id: 'worker:vault-renewal',
+    status: 'fail',
+    summary: 'UNATTENDED_VAULT_IDENTITY_REQUIRED',
+    remedy: {
+      action:
+        'Enroll this worker’s own periodic token or auto-auth identity. Ordinary expiring user sessions are not evidence of unattended renewal; no credentials are copied between machines.',
+    },
+  };
+  try {
+    const { addr } = resolveVaultConfig();
+    const result = await run([
+      'vault',
+      'token',
+      'lookup',
+      `-address=${addr}`,
+      '-format=json',
+    ]);
+    const data: unknown = JSON.parse(result.stdout);
+    if (!result.ok || !record(data) || !record(data['data']))
+      return unavailable;
+    const token = data['data'];
+    if (
+      token['renewable'] !== true ||
+      typeof token['period'] !== 'number' ||
+      token['period'] <= 0 ||
+      typeof token['ttl'] !== 'number' ||
+      token['ttl'] <= 1200
+    )
+      return unavailable;
+    if (!(await lastRenewalSucceeded(run)))
+      return {
+        ...unavailable,
+        summary: 'VAULT_RENEWAL_FAILING',
+        remedy: {
+          action:
+            'The identity is renewable, but the last run of the renewal job failed. Inspect it on this worker with: journalctl --user -u personal-worker-auth.service. Its token expires unless a renewal succeeds; no credentials are copied between machines.',
+        },
+      };
+    return {
+      ...unavailable,
+      status: 'pass',
+      summary:
+        'This worker has a renewable periodic Vault identity with sufficient lifetime for the renewal timer, and its last renewal succeeded.',
+    };
+  } catch {
+    return unavailable;
+  }
+}
+
+/** An active timer is not evidence that renewal works: the result of the
+ * job's last run is. A job that has not run yet has nothing to say.
+ */
+async function lastRenewalSucceeded(run: Runner): Promise<boolean> {
+  const shown = await run([
+    'systemctl',
+    '--user',
+    'show',
+    'personal-worker-auth.service',
+    '--property=Result,ExecMainStatus',
+  ]);
+  if (!shown.ok) return true;
+  const result = /^Result=(.*)$/m.exec(shown.stdout)?.[1];
+  const status = /^ExecMainStatus=(.*)$/m.exec(shown.stdout)?.[1];
+  return (
+    (result === undefined || result === 'success') &&
+    (status === undefined || status === '0')
+  );
+}

package/src/worker-capacity.ts

@@ -0,0 +1,34 @@
+import { join } from 'node:path';
+import { healthChecks, parseProbe } from './machine-health';
+import type { Check, Runner } from './types';
+
+export async function workerCapacity(
+  root: string,
+  node: string,
+  run: Runner
+): Promise<Check[]> {
+  const result = await run(['sh', join(root, 'ansible/net/remote-probe.sh')], {
+    timeoutMs: 10_000,
+  });
+  const sample = parseProbe(result.stdout);
+  if (
+    !result.ok ||
+    sample.ncpu === undefined ||
+    sample.load === undefined ||
+    sample.memFreePct === undefined ||
+    sample.diskMb === undefined
+  )
+    return [
+      {
+        id: 'worker:capacity',
+        status: 'fail',
+        summary: 'Required CPU, memory and disk evidence is unavailable.',
+      },
+    ];
+  return healthChecks(sample, node).map((check) => ({
+    id: `worker:capacity:${check.id}`,
+    status: check.severity,
+    summary: check.detail,
+    ...(check.fix ? { remedy: { action: check.fix } } : {}),
+  }));
+}

package/src/worker-host-checks.ts

@@ -0,0 +1,181 @@
+import { powerRecoveryCheck } from './power-recovery';
+import { auditUbuntu } from './platform/ubuntu-audit';
+import { verifyUbuntuBoot } from './platform/ubuntu-verify';
+import { verifyResourceSafety } from './platform/resource-verify';
+import policy from '../ansible/ubuntu/policy.json';
+import { userInfo } from 'node:os';
+import type { Check, Runner } from './types';
+
+function check(id: string, ok: boolean, summary: string): Check {
+  return {
+    id: `worker:${id}`,
+    status: ok ? 'pass' : 'fail',
+    summary,
+    ...(!ok
+      ? {
+          remedy: {
+            action:
+              'Apply the approved fleet-sync revision over verified SSH, then rerun fleet-status --workers. No password prompt or state reset is attempted by health.',
+          },
+        }
+      : {}),
+  };
+}
+/** Observed unit state for a failed service check, e.g.
+ * `SERVICE_NOT_INSTALLED beszel-agent.service (inactive)`. */
+export function serviceState(name: string, show: string): string {
+  const field = (key: string) =>
+    new RegExp(`^${key}=(.*)$`, 'm').exec(show)?.[1] ?? '';
+  const file = field('UnitFileState');
+  const state = field('ActiveState') || 'unknown';
+  const code = !file
+    ? 'SERVICE_NOT_INSTALLED'
+    : state !== 'active'
+      ? 'SERVICE_NOT_RUNNING'
+      : file !== 'enabled'
+        ? 'SERVICE_NOT_ENABLED'
+        : 'SERVICE_RESTART_POLICY';
+  return `${code} ${name}.service (${state}, unit file ${file || 'missing'}, restart ${field('Restart') || 'unknown'})`;
+}
+
+/** Measured runtime evidence, independent of installer receipts. */
+export async function workerHostChecks(
+  user: string,
+  run: Runner
+): Promise<Check[]> {
+  const audit = await auditUbuntu(run);
+  const checks = [
+    check(
+      'ubuntu',
+      audit.supported,
+      'Required Ubuntu release and architecture.'
+    ),
+    check(
+      'sudo',
+      audit.passwordlessSudo,
+      'Designated worker user has noninteractive sudo.'
+    ),
+    ...Object.entries(policy.packages).map(([name, version]) =>
+      check(
+        `package:${name}`,
+        audit.packages[name] === version,
+        'Exact declarative Ubuntu package version.'
+      )
+    ),
+    check(
+      'package:tailscale',
+      audit.packages['tailscale'] === policy.tailscale.version,
+      'Exact declarative Tailscale version.'
+    ),
+  ];
+  for (const name of policy.alwaysOnServices) {
+    const unit = await run([
+      'systemctl',
+      'show',
+      `${name}.service`,
+      '--property=UnitFileState,ActiveState,Restart',
+    ]);
+    const active =
+      name === 'beszel-agent'
+        ? /^ActiveState=active$/m.test(unit.stdout)
+        : audit.services[name] === true;
+    const ok =
+      unit.ok &&
+      active &&
+      /^UnitFileState=enabled$/m.test(unit.stdout) &&
+      /^Restart=always$/m.test(unit.stdout);
+    checks.push(
+      check(
+        `service:${name}`,
+        ok,
+        ok
+          ? 'Required system service is active, enabled for boot and restarts unconditionally.'
+          : serviceState(name, unit.stdout)
+      )
+    );
+  }
+  checks.push(...(await recoveryChecks(run)));
+  checks.push(...(await verifyUbuntuBoot(user, run)));
+  checks.push(...(await verifyResourceSafety(userInfo().homedir, run)));
+  return checks;
+}
+
+async function recoveryChecks(run: Runner): Promise<Check[]> {
+  const checks: Check[] = [];
+  const t3 = await run([
+    'systemctl',
+    '--user',
+    'show',
+    't3code-desktop.service',
+    '--property=ActiveState,UnitFileState,Restart',
+  ]);
+  checks.push(
+    check(
+      't3-recovery',
+      t3.ok &&
+        /^ActiveState=active$/m.test(t3.stdout) &&
+        /^UnitFileState=enabled$/m.test(t3.stdout) &&
+        /^Restart=(always|on-failure)$/m.test(t3.stdout),
+      'The T3 desktop app runs as an enabled session service with automatic crash recovery.'
+    )
+  );
+  checks.push(await t3HealthCheck(run));
+  const renewal = await run([
+    'systemctl',
+    '--user',
+    'is-active',
+    'personal-worker-auth.timer',
+  ]);
+  const recovery = await run([
+    'systemctl',
+    '--user',
+    'is-active',
+    'personal-worker-recovery.timer',
+  ]);
+  checks.push(
+    check(
+      'credential-renewal',
+      renewal.ok,
+      'Worker credential renewal timer is active.'
+    )
+  );
+  checks.push(
+    check(
+      'service-recovery',
+      recovery.ok,
+      'Worker service recovery timer is active.'
+    )
+  );
+  checks.push(await powerRecoveryCheck(run));
+  return checks;
+}
+
+/** The server answers its local API now, not merely that a process exists. */
+async function t3HealthCheck(run: Runner): Promise<Check> {
+  const probe = await run(
+    [
+      'curl',
+      '--fail',
+      '--silent',
+      '--max-time',
+      '10',
+      'http://127.0.0.1:3773/.well-known/t3/environment',
+    ],
+    { timeoutMs: 15_000 }
+  );
+  let identified = false;
+  try {
+    const value: unknown = JSON.parse(probe.stdout);
+    identified =
+      typeof value === 'object' &&
+      value !== null &&
+      typeof (value as Record<string, unknown>)['environmentId'] === 'string';
+  } catch {
+    /* Raw responses are never reported. */
+  }
+  return check(
+    't3-health',
+    probe.ok && identified,
+    'T3 local API answers with its environment identity within 10 seconds.'
+  );
+}

package/src/worker-policy.ts

@@ -0,0 +1,64 @@
+import { CAPABILITIES, record } from './config';
+import { FleetUsageError } from './errors';
+import type { Capability } from './types';
+
+export interface WorkerPolicy {
+  requiredCapabilities: Capability[];
+  githubLogin: string;
+  /** Vault userpass login each node uses for its own periodic token. */
+  vaultUser: string;
+  toolchain: 'mise.lock';
+  agentProfile: 'proxy';
+  unattendedSudo: true;
+  persistentT3: true;
+  neverSleep: true;
+}
+const REQUIRED = [
+  'github',
+  'vault',
+  't3',
+  'toolchain',
+  'workstation',
+  'worker',
+];
+function validCapabilities(value: unknown): boolean {
+  return (
+    Array.isArray(value) &&
+    new Set(value).size === value.length &&
+    value.every((item) => CAPABILITIES.includes(item)) &&
+    REQUIRED.every((item) => value.includes(item))
+  );
+}
+function validPolicy(value: Record<string, unknown>): boolean {
+  return (
+    Object.keys(value).every((key) =>
+      [
+        'requiredCapabilities',
+        'githubLogin',
+        'vaultUser',
+        'toolchain',
+        'agentProfile',
+        'unattendedSudo',
+        'persistentT3',
+        'neverSleep',
+      ].includes(key)
+    ) &&
+    validCapabilities(value['requiredCapabilities']) &&
+    typeof value['githubLogin'] === 'string' &&
+    /^[a-zA-Z0-9-]{1,39}$/.test(value['githubLogin']) &&
+    typeof value['vaultUser'] === 'string' &&
+    /^[a-zA-Z0-9_.-]{1,64}$/.test(value['vaultUser']) &&
+    value['toolchain'] === 'mise.lock' &&
+    value['agentProfile'] === 'proxy' &&
+    ['unattendedSudo', 'persistentT3', 'neverSleep'].every(
+      (key) => value[key] === true
+    )
+  );
+}
+export function parseWorkerPolicy(value: unknown): WorkerPolicy {
+  if (!record(value) || !validPolicy(value))
+    throw new FleetUsageError(
+      'Invalid mandatory worker policy. Workers require the locked toolchain, common services, unattended sudo, persistent T3 and no sleep.'
+    );
+  return value as unknown as WorkerPolicy;
+}

package/src/worker-readiness.ts

@@ -0,0 +1,64 @@
+import { workerCapacity } from './worker-capacity';
+import { userInfo, hostname } from 'node:os';
+import { hostId } from './topology';
+import { configPath } from './fleet-config';
+import type { Check, Runner } from './types';
+import { workerHostChecks } from './worker-host-checks';
+import { workerAccessChecks } from './worker-access-checks';
+import { record } from './config';
+
+export async function workerReadiness(
+  run: Runner,
+  nodeId = hostId(hostname()),
+  root = process.cwd()
+): Promise<Check[]> {
+  const host = await workerHostChecks(userInfo().username, run);
+  const access = await workerAccessChecks(run);
+  const revision = await run(['git', 'rev-parse', 'HEAD']);
+  const singleton = await run(
+    [
+      'python3',
+      '-B',
+      'cliproxy/scripts/fleet_sync.py',
+      '--definition',
+      configPath(),
+      '--node',
+      nodeId,
+      '--revision',
+      revision.stdout.trim(),
+    ],
+    { timeoutMs: 180_000 }
+  );
+  let proxy: Check = {
+    id: 'worker:proxy-singleton',
+    status: 'fail',
+    summary:
+      'Singleton gateway ownership or client-only role could not be verified.',
+  };
+  try {
+    const value: unknown = JSON.parse(singleton.stdout);
+    if (
+      singleton.ok &&
+      record(value) &&
+      Array.isArray(value['phases']) &&
+      value['phases'].length &&
+      value['phases'].every(
+        (item) => record(item) && item['status'] === 'unchanged'
+      )
+    )
+      proxy = {
+        ...proxy,
+        status: 'pass',
+        summary:
+          'Declared gateway host/client-only role verified without changing runtime identity.',
+      };
+  } catch {
+    /* Raw proxy output is never exposed. */
+  }
+  return [
+    ...host,
+    ...access,
+    proxy,
+    ...(await workerCapacity(root, nodeId, run)),
+  ];
+}

package/src/workstation-checks.ts

@@ -0,0 +1,29 @@
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import type { Check, Runner } from './types';
+
+/** This machine has the `devfleet` launcher, linked into a checkout. */
+export async function workstationChecks(
+  _root: string,
+  run: Runner
+): Promise<Check[]> {
+  const launcher = join(homedir(), '.local/bin/devfleet');
+  const result = await run(['test', '-x', launcher], { timeoutMs: 5_000 });
+  return [
+    result.ok
+      ? {
+          id: 'workstation:configuration',
+          status: 'pass',
+          summary: 'The devfleet launcher is installed.',
+        }
+      : {
+          id: 'workstation:configuration',
+          status: 'fail',
+          summary: 'The devfleet launcher is missing.',
+          remedy: {
+            action:
+              'Run fleet-sync from the manager: it links the launcher to the checkout of the revision it applies.',
+          },
+        },
+  ];
+}