@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/tailnet-health.ts

@@ -0,0 +1,231 @@
+import { compareFingerprints } from './fingerprint';
+import { probe } from './probe';
+import { inspectRemote } from './transport';
+import { mapBounded } from './report';
+import { matchComputers } from './tailnet-matching';
+import type { Tailnet, TailnetComputer } from './tailscale';
+import type {
+  Capability,
+  Fingerprint,
+  Check,
+  FleetNode,
+  NodeReport,
+  Runner,
+  Snapshot,
+} from './types';
+
+export const HEALTH_CAPABILITIES: Capability[] = [
+  't3',
+  'toolchain',
+  'workstation',
+];
+export interface TailnetHealthOptions {
+  includeManagerReport?: boolean;
+  reference?: Snapshot | undefined;
+  root: string;
+  timeoutMs: number;
+  source: () => Promise<string>;
+}
+
+export async function tailnetHealth(
+  tailnet: Tailnet,
+  nodes: FleetNode[],
+  options: TailnetHealthOptions,
+  run: Runner
+): Promise<NodeReport[]> {
+  const matches = await matchComputers(nodes, tailnet.computers, run);
+  const self = tailnet.computers.find((peer) => peer.self);
+  const candidates = self ? (matches.get(self.id) ?? []) : [];
+  const required = uniqueMapping(candidates, matches)
+    ? candidates[0]!.required
+    : HEALTH_CAPABILITIES;
+  const manager =
+    options.reference ??
+    (await probe(
+      options.root,
+      options.includeManagerReport === false && !self ? [] : required,
+      run
+    ));
+  let source: Promise<string> | undefined;
+  const reports = await mapBounded(tailnet.computers, async (peer) => {
+    const checks = await connectivity(peer, run);
+    if (peer.self)
+      return {
+        node: peer.dns,
+        snapshot: manager,
+        checks: [...checks, ...manager.checks],
+      };
+    const candidates = matches.get(peer.id)!;
+    if (!uniqueMapping(candidates, matches))
+      return missingMapping(peer, checks, candidates.length);
+    if (!peer.online) return { node: peer.dns, checks };
+    const node = candidates[0]!;
+    const report = await inspectComputer(
+      node,
+      {
+        action: 'probe',
+        expected: manager.fingerprint,
+        source: (source ??= Promise.resolve().then(options.source)),
+        timeoutMs: options.timeoutMs,
+      },
+      run
+    );
+    return combine(peer, checks, report, manager);
+  });
+  const absent = nodes.filter(
+    (node) => ![...matches.values()].some((matched) => matched.includes(node))
+  );
+  return [
+    { node: 'tailscale', checks: tailnet.checks },
+    ...(options.includeManagerReport !== false &&
+    !tailnet.computers.some((peer) => peer.self)
+      ? [
+          {
+            node: 'manager',
+            snapshot: manager,
+            checks: manager.checks,
+          },
+        ]
+      : []),
+    ...reports,
+    ...absent.map(absentNode),
+  ];
+}
+
+export function uniqueMapping(
+  candidates: FleetNode[],
+  matches: Map<string, FleetNode[]>
+): boolean {
+  return (
+    candidates.length === 1 &&
+    [...matches.values()].filter((nodes) => nodes.includes(candidates[0]!))
+      .length === 1
+  );
+}
+
+async function inspectComputer(
+  node: FleetNode,
+  options: {
+    action: 'probe';
+    source: Promise<string>;
+    timeoutMs: number;
+    expected?: Fingerprint;
+  },
+  run: Runner
+): Promise<NodeReport> {
+  try {
+    return await inspectRemote(
+      node,
+      { ...options, source: await options.source },
+      run
+    );
+  } catch {
+    return {
+      node: node.id,
+      checks: [
+        {
+          id: 'fleet:probe',
+          status: 'fail',
+          summary:
+            'The current remote probe could not be prepared or completed; this computer remains unverified.',
+          remedy: {
+            action:
+              'Reinstall the validated Personal CLI package and retry. No raw exception or remote output is retained.',
+          },
+        },
+      ],
+    };
+  }
+}
+
+async function connectivity(
+  peer: TailnetComputer,
+  run: Runner
+): Promise<Check[]> {
+  const online: Check = {
+    id: 'tailscale:online',
+    status: peer.online ? 'pass' : 'fail',
+    summary: peer.online
+      ? 'Computer is online in Tailscale status.'
+      : 'Computer is offline or Tailscale cannot confirm its online state.',
+  };
+  if (peer.self || !peer.online) return [online];
+  const ping = await run(
+    ['tailscale', 'ping', '--c=1', '--timeout=5s', peer.ips[0]!],
+    { timeoutMs: 8_000 }
+  );
+  return [
+    online,
+    {
+      id: 'tailscale:reachability',
+      status: ping.ok ? 'pass' : 'fail',
+      summary: ping.ok
+        ? 'Tailscale peer answered a bounded reachability probe.'
+        : 'Tailscale peer did not answer; online inventory is not proof of connectivity.',
+    },
+  ];
+}
+
+function missingMapping(
+  peer: TailnetComputer,
+  checks: Check[],
+  count: number
+): NodeReport {
+  return {
+    node: peer.dns,
+    checks: [
+      ...checks,
+      {
+        id: 'fleet:registration',
+        status: 'fail',
+        summary: count
+          ? 'Roster mapping is ambiguous between entries or computers; no SSH probe was run.'
+          : 'Computer is not registered with a verified SSH alias and checkout path; dependencies and authentication are unverified.',
+        remedy: {
+          action:
+            'Run fleet-node-add with its existing SSH alias and actual checkout path, then rerun fleet-status --tailnet.',
+        },
+      },
+    ],
+  };
+}
+
+function combine(
+  peer: TailnetComputer,
+  checks: Check[],
+  report: NodeReport,
+  reference: Snapshot
+): NodeReport {
+  return {
+    ...report,
+    node: peer.dns,
+    checks: [
+      ...checks,
+      ...report.checks,
+      ...(report.snapshot
+        ? compareFingerprints(
+            reference.fingerprint,
+            report.snapshot.fingerprint
+          )
+        : []),
+    ],
+  };
+}
+
+function absentNode(node: FleetNode): NodeReport {
+  return {
+    node: node.id,
+    checks: [
+      {
+        id: 'tailscale:membership',
+        status: 'fail',
+        summary:
+          'Registered node is absent from visible Tailscale inventory. It was not silently omitted.',
+        remedy: {
+          action:
+            'Check the node account, tailnet visibility, and the registered SSH alias.',
+        },
+      },
+    ],
+  };
+}

package/src/tailnet-matching.ts

@@ -0,0 +1,37 @@
+import type { FleetNode, Runner } from './types';
+import type { TailnetComputer } from './tailscale';
+
+export function matchesComputer(host: string, peer: TailnetComputer): boolean {
+  const name = host.toLowerCase().replace(/\.$/, '');
+  return [peer.dns, peer.dns.split('.')[0], ...peer.ips].includes(name);
+}
+
+export async function matchComputers(
+  nodes: FleetNode[],
+  peers: TailnetComputer[],
+  run: Runner
+): Promise<Map<string, FleetNode[]>> {
+  const matches = new Map(peers.map((peer) => [peer.id, [] as FleetNode[]]));
+  for (const node of nodes) {
+    const resolved = await run(
+      [
+        'ssh',
+        '-G',
+        '-o',
+        'CanonicalizeHostname=no',
+        '-o',
+        'PermitLocalCommand=no',
+        node.ssh,
+      ],
+      { timeoutMs: 5_000 }
+    );
+    const hostname = resolved.ok
+      ? /^hostname ([^\r\n]+)$/im.exec(resolved.stdout)?.[1]?.trim()
+      : undefined;
+    const targets = peers.filter((peer) =>
+      matchesComputer(hostname ?? node.ssh, peer)
+    );
+    for (const peer of targets) matches.get(peer.id)!.push(node);
+  }
+  return matches;
+}

package/src/tailscale.ts

@@ -0,0 +1,141 @@
+import { isIP } from 'node:net';
+import { record } from './config';
+import type { Check, Runner } from './types';
+
+export interface TailnetComputer {
+  id: string;
+  name: string;
+  dns: string;
+  os: string;
+  ips: string[];
+  online: boolean;
+  self: boolean;
+}
+export interface Tailnet {
+  computers: TailnetComputer[];
+  excludedMobile: number;
+  checks: Check[];
+}
+
+export async function discoverTailnet(run: Runner): Promise<Tailnet> {
+  const result = await run(['tailscale', 'status', '--json'], {
+    timeoutMs: 15_000,
+  });
+  if (!result.ok)
+    return unavailable(
+      'Tailscale status is unavailable; check the CLI, daemon and account login.'
+    );
+  try {
+    return parseTailnet(JSON.parse(result.stdout) as unknown);
+  } catch {
+    return unavailable(
+      'Tailscale returned an unsupported status document. Update the CLI and check its daemon.'
+    );
+  }
+}
+
+export function parseTailnet(data: unknown): Tailnet {
+  if (!record(data) || data['BackendState'] !== 'Running')
+    return unavailable(
+      'Tailscale is not running and signed in. Review tailscale status locally.'
+    );
+  validateStatusFields(data);
+  const peers = Object.values(
+    (data['Peer'] as Record<string, unknown> | null) ?? {}
+  );
+  if (peers.length > 1024) throw new Error('Too many peers');
+  const all = [
+    parseComputer(data['Self'], true),
+    ...peers.map((peer) => parseComputer(peer, false)),
+  ];
+  if (new Set(all.map((peer) => peer.id)).size !== all.length)
+    throw new Error('Duplicate peer');
+  const mobile = (peer: TailnetComputer) =>
+    ['ios', 'android', 'tvos'].includes(peer.os.toLowerCase());
+  const computers = all
+    .filter((peer) => !mobile(peer))
+    .sort((a, b) => a.name.localeCompare(b.name, 'en'));
+  const healthy = healthyDiscovery(data, computers.length);
+  return {
+    computers,
+    excludedMobile: all.filter(mobile).length,
+    checks: [
+      {
+        id: 'tailscale:discovery',
+        status: healthy ? 'pass' : 'fail',
+        summary: healthy
+          ? 'Signed-in Tailscale daemon returned its visible computer inventory.'
+          : 'Tailscale reports a daemon/coordination health problem or no computers; discovery may be incomplete.',
+        ...(!healthy
+          ? {
+              remedy: {
+                action:
+                  'Review tailscale status and network connectivity. Resolve coordination/daemon health before claiming complete coverage.',
+              },
+            }
+          : {}),
+      },
+    ],
+  };
+}
+
+function healthyDiscovery(
+  data: Record<string, unknown>,
+  count: number
+): boolean {
+  const health = (data['Health'] as unknown[] | null | undefined) ?? [];
+  return health.length === 0 && count > 0;
+}
+
+function validateStatusFields(data: Record<string, unknown>): void {
+  const health = data['Health'] ?? [];
+  const peers = data['Peer'] ?? {};
+  if (!record(data['Self']) || !record(peers) || !Array.isArray(health))
+    throw new Error('Unsupported Tailscale status');
+}
+
+function parseComputer(value: unknown, self: boolean): TailnetComputer {
+  if (!record(value)) throw new Error('Invalid peer');
+  const {
+    ID: id,
+    HostName: name,
+    DNSName: dns,
+    OS: os,
+    TailscaleIPs: ips,
+    Online: online,
+  } = value;
+  if (
+    ![id, name, os].every(
+      (field) =>
+        typeof field === 'string' &&
+        field.length > 0 &&
+        field.length <= 256 &&
+        !/[\x00-\x1f\x7f]/.test(field)
+    ) ||
+    typeof dns !== 'string' ||
+    !/^[a-zA-Z0-9][a-zA-Z0-9.-]{0,252}\.?$/.test(dns) ||
+    !Array.isArray(ips) ||
+    ips.length < 1 ||
+    ips.length > 16 ||
+    !ips.every((ip) => typeof ip === 'string' && isIP(ip) !== 0) ||
+    typeof online !== 'boolean'
+  )
+    throw new Error('Invalid peer fields');
+  return {
+    id: id as string,
+    name: name as string,
+    dns: dns.replace(/\.$/, '').toLowerCase(),
+    os: os as string,
+    ips: ips as string[],
+    online,
+    self,
+  };
+}
+
+function unavailable(summary: string): Tailnet {
+  return {
+    computers: [],
+    excludedMobile: 0,
+    checks: [{ id: 'tailscale:discovery', status: 'fail', summary }],
+  };
+}

package/src/toolchain-checks.ts

@@ -0,0 +1,108 @@
+import { join } from 'node:path';
+import { record } from './config';
+import type { Check, Runner } from './types';
+
+export async function toolchainChecks(
+  root: string,
+  run: Runner,
+  worker?: boolean
+): Promise<Check[]> {
+  const result = await run(doctorArguments(root, worker), {
+    cwd: root,
+    timeoutMs: 90_000,
+  });
+  const fallback: Check = {
+    id: 'auth:toolchain',
+    status: 'fail',
+    summary: 'Pinned tool/dependency and Docker checks could not be verified.',
+    remedy: {
+      action:
+        'Run the checkout bootstrap/doctor.ts --json --docker on this computer. Reconcile versions only through the shared mise setup.',
+    },
+  };
+  try {
+    const data: unknown = JSON.parse(result.stdout);
+    if (
+      !record(data) ||
+      typeof data['ok'] !== 'boolean' ||
+      !Array.isArray(data['checks']) ||
+      !data['checks'].length ||
+      data['checks'].length > 150
+    )
+      return [fallback];
+    const checks = data['checks'].map(toolCheck);
+    if (checks.some((check) => check === null)) return [fallback];
+    return [
+      ...(!result.ok || !data['ok'] ? [fallback] : []),
+      ...(checks as Check[]),
+    ];
+  } catch {
+    return [fallback];
+  }
+}
+
+/** Checks of the doctor that are not a tool version, with their own words. */
+const OWN: Record<string, { pass: string; fail: string; action: string }> = {
+  'global-mise-config': {
+    pass: 'The global mise configuration is the one fleet-sync publishes.',
+    fail: 'The global mise configuration is missing, was replaced by another project or tool, or is not the one this revision publishes.',
+    action:
+      'Run devfleet fleet-sync from the project that holds fleet.json. On a fleet machine the global mise configuration belongs to dev-fleet: a project pins its own tools in a project mise.toml and never publishes a global one.',
+  },
+};
+
+/** A doctor check with its own words, or null for a tool version. */
+function ownCheck(id: string, status: Check['status']): Check | null {
+  if (!Object.hasOwn(OWN, id)) return null;
+  const own = OWN[id]!;
+  const ok = status === 'pass';
+  return {
+    id: `tool:${id}`,
+    status,
+    summary: ok ? own.pass : own.fail,
+    ...(!ok ? { remedy: { action: own.action } } : {}),
+  };
+}
+
+function toolCheck(value: unknown): Check | null {
+  if (
+    !record(value) ||
+    typeof value['id'] !== 'string' ||
+    !/^[a-zA-Z0-9@:/._-]{1,75}$/.test(value['id']) ||
+    !['pass', 'fail', 'warn', 'skip'].includes(String(value['status']))
+  )
+    return null;
+  const own = ownCheck(value['id'], value['status'] as Check['status']);
+  if (own) return own;
+  const ok = value['status'] === 'pass';
+  const version =
+    typeof value['detail'] === 'string' &&
+    /^v?\d{1,6}(?:\.\d{1,6}){1,3}$/.test(value['detail'])
+      ? ` ${value['detail']}`
+      : '';
+  return {
+    id: `tool:${value['id'].toLowerCase().replace(/[^a-z0-9:._-]/g, '_')}`,
+    status: value['status'] as Check['status'],
+    summary: ok
+      ? `Expected tool version or runtime check succeeded${version}.`
+      : 'Expected tool version, runtime or authentication check did not pass.',
+    ...(!ok
+      ? {
+          remedy: {
+            action:
+              'Review bootstrap/doctor.ts --json --docker locally. Use shared mise/workstation setup to reconcile this computer.',
+          },
+        }
+      : {}),
+  };
+}
+
+function doctorArguments(root: string, worker?: boolean): string[] {
+  return [
+    'bun',
+    join(root, 'bootstrap/doctor.ts'),
+    '--json',
+    '--docker',
+    ...(worker ? ['--worker'] : []),
+  ];
+}

package/src/topology.ts

@@ -0,0 +1,24 @@
+import type { FleetDefinition } from './definition';
+
+/** Fleet topology: one singleton manager (the human's machine) and stateless
+ * Linux workers. Every singleton service runs on the manager; workers only
+ * consume them and can be turned off, replaced or re-imaged at will. */
+export type Role = 'manager' | 'worker';
+
+export function roleOf(definition: FleetDefinition, id: string): Role | null {
+  if (definition.manager.id === id) return 'manager';
+  return definition.workers.some((entry) => entry.id === id) ? 'worker' : null;
+}
+
+/** Private Beszel hub URL on the manager; agents never use the public name. */
+export function hubUrl(definition: FleetDefinition): string {
+  return `https://${definition.manager.network.tailscaleName}:${definition.services.monitoring.hubPort}`;
+}
+
+/** Normalizes a macOS LocalHostName / hostname to a fleet ID for identity checks. */
+export function hostId(hostname: string): string {
+  return hostname
+    .toLowerCase()
+    .replace(/\.local$/, '')
+    .replace(/[^a-z0-9-]/g, '-');
+}

package/src/transport.ts

@@ -0,0 +1,122 @@
+import { managedProbeTarget } from './managed-source';
+import type { ConfigurationRequest } from './configure';
+import { remoteCommand, sshFailure, sshArgs } from './ssh';
+import { cleanCheck, decodeSnapshot, validCheck } from './protocol';
+import { record } from './config';
+import { fileURLToPath } from 'node:url';
+import type { FleetNode, NodeReport, Runner, Fingerprint } from './types';
+import type { AutomatedRepair } from './repair-catalog';
+
+export async function bundleProbe(): Promise<string> {
+  const result = await Bun.build({
+    entrypoints: [fileURLToPath(new URL('./remote-entry.ts', import.meta.url))],
+    target: 'node',
+    minify: true,
+  });
+  if (!result.success || !result.outputs[0])
+    throw new Error('Cannot bundle the fleet probe.');
+  return result.outputs[0].text();
+}
+
+export interface RemoteOptions {
+  source: string;
+  timeoutMs: number;
+  action: 'probe' | 'sync' | 'repair' | 'configure';
+  configuration?: ConfigurationRequest;
+  repairs?: AutomatedRepair[];
+  expected?: Fingerprint;
+  pull?: boolean;
+}
+
+export async function inspectRemote(
+  node: FleetNode,
+  options: RemoteOptions,
+  run: Runner
+): Promise<NodeReport> {
+  if (node.required.includes('worker')) {
+    const preflight = await run([...sshArgs(node), 'true'], {
+      timeoutMs: 15_000,
+    });
+    if (!preflight.ok)
+      return { node: node.id, checks: [sshFailure(preflight)] };
+  }
+  const target =
+    options.action === 'probe' && options.expected
+      ? await managedProbeTarget(node, options.expected.head, run)
+      : { node, checks: [] };
+  const command = remoteCommand(target.node, {
+    action: options.action,
+    configuration: options.configuration,
+    node: target.node,
+    apply: options.action !== 'probe',
+    repairs: options.repairs,
+    expected: options.expected,
+    pull: options.pull === true,
+  });
+  const result = await run(command, {
+    input: options.source,
+    timeoutMs: options.timeoutMs,
+  });
+  if (!result.ok)
+    return { node: node.id, checks: [...target.checks, sshFailure(result)] };
+  if (options.action !== 'probe') return decodeSync(node.id, result.stdout);
+  const snapshot = decodeSnapshot(result.stdout);
+  if (!snapshot) return protocolFailure(node.id);
+  return {
+    node: node.id,
+    snapshot,
+    checks: [
+      ...target.checks,
+      {
+        id: 'ssh:session',
+        status: 'pass',
+        summary:
+          'Fresh noninteractive SSH session and Node-compatible probe succeeded with verified host keys and no agent forwarding.',
+      },
+      ...snapshot.checks,
+      {
+        id: 'vscode:forwarding',
+        status: 'skip',
+        required: node.required.includes('vscode'),
+        summary:
+          'VS Code Remote-SSH tunnel/server installation is not tested by a shell session. Use the onboarding forwarding test.',
+      },
+    ],
+  };
+}
+
+function decodeSync(node: string, output: string): NodeReport {
+  try {
+    const data: unknown = JSON.parse(output);
+    if (
+      record(data) &&
+      data['version'] === 1 &&
+      Array.isArray(data['checks']) &&
+      data['checks'].length > 0 &&
+      data['checks'].length <= 200 &&
+      data['checks'].every(validCheck)
+    ) {
+      return { node, checks: data['checks'].map(cleanCheck) };
+    }
+  } catch {
+    return protocolFailure(node);
+  }
+  return protocolFailure(node);
+}
+
+function protocolFailure(node: string): NodeReport {
+  return {
+    node,
+    checks: [
+      {
+        id: 'ssh:protocol',
+        status: 'fail',
+        summary: 'Remote response is not a supported fleet JSON document.',
+        remedy: {
+          action:
+            'Remove stdout banners from noninteractive shell startup; check the Node/Bun runtime. No raw remote output is retained.',
+        },
+      },
+    ],
+  };
+}