@crvouga/dev-fleet 0.0.0-stage → 0.1.0

package/src/t3/reconcile.ts

@@ -0,0 +1,123 @@
+import { mkdir, rm } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { FleetUsageError } from '../errors';
+import { canonical, object, read, safePath, write } from './files';
+import { ownership, planT3, readOwnership, type T3Paths } from './plan';
+
+export async function reconcileT3(
+  paths: T3Paths,
+  desired: Record<string, unknown>,
+  options: { apply: boolean; planHash?: string }
+) {
+  if (!options.apply) return report(await planT3(paths, desired), false);
+  return locked(paths, async () => {
+    const plan = await planT3(paths, desired);
+    if (options.planHash && options.planHash !== plan.planHash)
+      throw new FleetUsageError(
+        'T3 plan is stale. Run devfleet t3-apply --dry-run again.'
+      );
+    if (plan.changes.some((change) => change.action === 'conflict'))
+      throw new FleetUsageError(
+        'T3 instance already exists without ownership. Review it and use devfleet t3-apply --import ID.'
+      );
+    if (!plan.settingsChanged && !plan.stateChanged) return report(plan, true);
+    await persist(paths, desired, plan);
+    const verified = await planT3(paths, desired);
+    if (verified.settingsChanged || verified.stateChanged)
+      throw new FleetUsageError(
+        'T3 changed after apply; reconciliation is incomplete. Replan and retry.'
+      );
+    return report(plan, true);
+  });
+}
+
+export async function importT3(paths: T3Paths, id: string) {
+  return locked(paths, async () => {
+    const raw = await read(paths.settings);
+    const instances = object(
+      JSON.stringify(object(raw)['providerInstances'] ?? {})
+    );
+    if (!Object.hasOwn(instances, id))
+      throw new FleetUsageError('T3 import requires an existing instance ID.');
+    const state = await read(paths.state);
+    const owned = readOwnership(state, paths.settings);
+    if (!owned.includes(id)) {
+      if (state !== null) await write(`${paths.state}.backup`, state);
+      if ((await read(paths.settings)) !== raw)
+        throw new FleetUsageError('T3 changed during import; retry.');
+      await write(
+        paths.state,
+        canonical(ownership(paths.settings, [...owned, id])) + '\n'
+      );
+    }
+    return { version: 1, ok: true, imported: id, changed: !owned.includes(id) };
+  });
+}
+
+async function locked<T>(paths: T3Paths, action: () => Promise<T>): Promise<T> {
+  await safePath(paths.state);
+  await mkdir(dirname(paths.state), { recursive: true, mode: 0o700 });
+  const lock = `${paths.state}.lock`;
+  try {
+    await mkdir(lock, { mode: 0o700 });
+  } catch {
+    throw new FleetUsageError(
+      'Another personal T3 operation holds the lock. Inspect interrupted operations before removing a stale lock.'
+    );
+  }
+  try {
+    return await action();
+  } finally {
+    await rm(lock, { recursive: true });
+  }
+}
+
+function report(plan: Awaited<ReturnType<typeof planT3>>, applied: boolean) {
+  return {
+    version: 1,
+    ok: !plan.changes.some((change) => change.action === 'conflict'),
+    applied,
+    changed: plan.settingsChanged || plan.stateChanged,
+    planHash: plan.planHash,
+    changes: plan.changes,
+    runtime:
+      'T3 watches settings.json; new sessions use the reconciled instances. Active sessions are not restarted.',
+  };
+}
+
+async function persist(
+  paths: T3Paths,
+  desired: Record<string, unknown>,
+  plan: Awaited<ReturnType<typeof planT3>>
+) {
+  // Keep a private recovery copy. Contents, environment and secret references never enter reports.
+  if (plan.raw !== null && plan.settingsChanged)
+    await write(`${paths.state}.settings-backup`, plan.raw);
+  if (plan.stateRaw !== null)
+    await write(`${paths.state}.backup`, plan.stateRaw);
+  if (
+    (await read(paths.settings)) !== plan.raw ||
+    (await read(paths.state)) !== plan.stateRaw
+  )
+    throw new FleetUsageError(
+      'T3 configuration changed during apply. Replan and retry.'
+    );
+  // Intent first: retry after interruption can finish creates/deletes without importing them.
+  const previous = readOwnership(plan.stateRaw, paths.settings);
+  await write(
+    paths.state,
+    canonical(
+      ownership(paths.settings, [
+        ...new Set([...previous, ...Object.keys(desired)]),
+      ])
+    ) + '\n'
+  );
+  if (plan.settingsChanged) {
+    if ((await read(paths.settings)) !== plan.raw)
+      throw new FleetUsageError(
+        'T3 changed during apply. Ownership intent was saved; replan and retry.'
+      );
+    await write(paths.settings, JSON.stringify(plan.settings, null, 2) + '\n');
+  }
+  await write(paths.state, canonical(plan.state) + '\n');
+}

package/src/t3-endpoint.ts

@@ -0,0 +1,48 @@
+import { discoverTailnet } from './tailscale';
+import type { Check, FleetNode, Runner } from './types';
+
+/** A fresh manager HTTPS request is transport evidence only. Never read
+ * pairing material or treat HTTP health as an authenticated tool session.
+ */
+export async function verifyT3Endpoint(
+  node: FleetNode,
+  run: Runner,
+  request: (url: string, options: RequestInit) => Promise<Response> = fetch
+): Promise<Check[]> {
+  const tailnet = await discoverTailnet(run);
+  const peer = tailnet.computers.find((entry) => entry.id === node.tailscaleId);
+  if (!peer || !/^[a-z0-9.-]+\.ts\.net$/.test(peer.dns))
+    return pending('T3_MANAGER_IDENTITY_UNVERIFIED');
+  try {
+    const response = await request('https://' + peer.dns, {
+      method: 'GET',
+      redirect: 'manual',
+      signal: AbortSignal.timeout(10_000),
+    });
+    await response.body?.cancel();
+    if (response.status !== 200) return pending('T3_MANAGER_ENDPOINT_REJECTED');
+    return [
+      {
+        id: 't3:manager-endpoint',
+        status: 'pass',
+        summary:
+          'A fresh HTTPS request from the manager reached the enrolled private endpoint with valid TLS and HTTP 200. Pairing, server compatibility and tool execution remain unverified.',
+      },
+    ];
+  } catch {
+    return pending('T3_MANAGER_ENDPOINT_UNREACHABLE');
+  }
+}
+function pending(code: string): Check[] {
+  return [
+    {
+      id: 't3:manager-endpoint',
+      status: 'fail',
+      summary: code,
+      remedy: {
+        action:
+          'Inspect the enrolled node’s private Serve route, T3 listener, Tailscale reachability and HTTPS capability. Restore the existing private route in source; never bypass TLS verification or interpret a local listener as manager access.',
+      },
+    },
+  ];
+}

package/src/t3-linked-check.ts

@@ -0,0 +1,101 @@
+import { sshArgs } from './ssh';
+import { record } from './config';
+import type { Check, FleetNode, Runner } from './types';
+
+/** Correlate a live SSH-verified server identity with the native manager's authenticated link. */
+export async function verifyT3Linked(
+  node: FleetNode,
+  run: Runner,
+  options: {
+    now?: () => number;
+    /** After this sync restarted the node's T3, give its link time to reconnect. */
+    reconnectMs?: number;
+    pause?: () => Promise<void>;
+  } = {}
+): Promise<Check> {
+  const now = options.now ?? Date.now;
+  const deadline = now() + (options.reconnectMs ?? 0);
+  let check = await linkedOnce(node, run, now());
+  while (check.status === 'fail' && now() < deadline) {
+    await (
+      options.pause ?? (() => new Promise((done) => setTimeout(done, 10_000)))
+    )();
+    check = await linkedOnce(node, run, now());
+  }
+  return check;
+}
+
+async function linkedOnce(
+  node: FleetNode,
+  run: Runner,
+  now: number
+): Promise<Check> {
+  const unavailable: Check = {
+    id: 't3:paired-access',
+    status: 'fail',
+    summary: 'MANAGER_T3_LINK_UNVERIFIED',
+    remedy: {
+      action:
+        'Link this exact worker environment to the manager using T3 Connections once. Existing pairing identities are preserved; passwords and pairing tokens are never copied by Fleet.',
+    },
+  };
+  try {
+    const identity =
+      node.t3EnvironmentId ?? (await observedIdentity(node, run));
+    if (!identity) return unavailable;
+    const result = await run(['t3', 'environment', 'list', '--json'], {
+      timeoutMs: 30_000,
+    });
+    const links: unknown = JSON.parse(result.stdout);
+    if (!result.ok || !Array.isArray(links)) return unavailable;
+    return linkedEvidence(links, identity, unavailable, now);
+  } catch {
+    return unavailable;
+  }
+}
+
+function linkedEvidence(
+  links: unknown[],
+  environmentId: string,
+  unavailable: Check,
+  now: number
+): Check {
+  const link = links.find(
+    (value) => record(value) && value['environmentId'] === environmentId
+  );
+  if (
+    !record(link) ||
+    link['status'] !== 'reachable' ||
+    typeof link['expiresAt'] !== 'string'
+  )
+    return unavailable;
+  const expires = Date.parse(link['expiresAt']);
+  if (!Number.isFinite(expires) || expires <= now) return unavailable;
+  return {
+    ...unavailable,
+    status: expires - now < 7 * 86400000 ? 'warn' : 'pass',
+    summary:
+      expires - now < 7 * 86400000
+        ? 'Exact worker link is reachable but its manager authorization expires within seven days.'
+        : 'Native T3 authenticated link reaches the exact enrolled worker environment. This checks connected access, not a model task.',
+  };
+}
+
+async function observedIdentity(
+  node: FleetNode,
+  run: Runner
+): Promise<string | undefined> {
+  const server = await run(
+    [
+      ...sshArgs(node),
+      'curl --fail --silent --max-time 5 http://127.0.0.1:3773/.well-known/t3/environment',
+    ],
+    { timeoutMs: 15_000 }
+  );
+  const identity: unknown = JSON.parse(server.stdout);
+  return server.ok &&
+    record(identity) &&
+    typeof identity['environmentId'] === 'string'
+    ? identity['environmentId']
+    : undefined;
+}

package/src/tailnet-health.ts

@@ -0,0 +1,231 @@
+import { compareFingerprints } from './fingerprint';
+import { probe } from './probe';
+import { inspectRemote } from './transport';
+import { mapBounded } from './report';
+import { matchComputers } from './tailnet-matching';
+import type { Tailnet, TailnetComputer } from './tailscale';
+import type {
+  Capability,
+  Fingerprint,
+  Check,
+  FleetNode,
+  NodeReport,
+  Runner,
+  Snapshot,
+} from './types';
+
+export const HEALTH_CAPABILITIES: Capability[] = [
+  't3',
+  'toolchain',
+  'workstation',
+];
+export interface TailnetHealthOptions {
+  includeManagerReport?: boolean;
+  reference?: Snapshot | undefined;
+  root: string;
+  timeoutMs: number;
+  source: () => Promise<string>;
+}
+
+export async function tailnetHealth(
+  tailnet: Tailnet,
+  nodes: FleetNode[],
+  options: TailnetHealthOptions,
+  run: Runner
+): Promise<NodeReport[]> {
+  const matches = await matchComputers(nodes, tailnet.computers, run);
+  const self = tailnet.computers.find((peer) => peer.self);
+  const candidates = self ? (matches.get(self.id) ?? []) : [];
+  const required = uniqueMapping(candidates, matches)
+    ? candidates[0]!.required
+    : HEALTH_CAPABILITIES;
+  const manager =
+    options.reference ??
+    (await probe(
+      options.root,
+      options.includeManagerReport === false && !self ? [] : required,
+      run
+    ));
+  let source: Promise<string> | undefined;
+  const reports = await mapBounded(tailnet.computers, async (peer) => {
+    const checks = await connectivity(peer, run);
+    if (peer.self)
+      return {
+        node: peer.dns,
+        snapshot: manager,
+        checks: [...checks, ...manager.checks],
+      };
+    const candidates = matches.get(peer.id)!;
+    if (!uniqueMapping(candidates, matches))
+      return missingMapping(peer, checks, candidates.length);
+    if (!peer.online) return { node: peer.dns, checks };
+    const node = candidates[0]!;
+    const report = await inspectComputer(
+      node,
+      {
+        action: 'probe',
+        expected: manager.fingerprint,
+        source: (source ??= Promise.resolve().then(options.source)),
+        timeoutMs: options.timeoutMs,
+      },
+      run
+    );
+    return combine(peer, checks, report, manager);
+  });
+  const absent = nodes.filter(
+    (node) => ![...matches.values()].some((matched) => matched.includes(node))
+  );
+  return [
+    { node: 'tailscale', checks: tailnet.checks },
+    ...(options.includeManagerReport !== false &&
+    !tailnet.computers.some((peer) => peer.self)
+      ? [
+          {
+            node: 'manager',
+            snapshot: manager,
+            checks: manager.checks,
+          },
+        ]
+      : []),
+    ...reports,
+    ...absent.map(absentNode),
+  ];
+}
+
+export function uniqueMapping(
+  candidates: FleetNode[],
+  matches: Map<string, FleetNode[]>
+): boolean {
+  return (
+    candidates.length === 1 &&
+    [...matches.values()].filter((nodes) => nodes.includes(candidates[0]!))
+      .length === 1
+  );
+}
+
+async function inspectComputer(
+  node: FleetNode,
+  options: {
+    action: 'probe';
+    source: Promise<string>;
+    timeoutMs: number;
+    expected?: Fingerprint;
+  },
+  run: Runner
+): Promise<NodeReport> {
+  try {
+    return await inspectRemote(
+      node,
+      { ...options, source: await options.source },
+      run
+    );
+  } catch {
+    return {
+      node: node.id,
+      checks: [
+        {
+          id: 'fleet:probe',
+          status: 'fail',
+          summary:
+            'The current remote probe could not be prepared or completed; this computer remains unverified.',
+          remedy: {
+            action:
+              'Reinstall the validated Personal CLI package and retry. No raw exception or remote output is retained.',
+          },
+        },
+      ],
+    };
+  }
+}
+
+async function connectivity(
+  peer: TailnetComputer,
+  run: Runner
+): Promise<Check[]> {
+  const online: Check = {
+    id: 'tailscale:online',
+    status: peer.online ? 'pass' : 'fail',
+    summary: peer.online
+      ? 'Computer is online in Tailscale status.'
+      : 'Computer is offline or Tailscale cannot confirm its online state.',
+  };
+  if (peer.self || !peer.online) return [online];
+  const ping = await run(
+    ['tailscale', 'ping', '--c=1', '--timeout=5s', peer.ips[0]!],
+    { timeoutMs: 8_000 }
+  );
+  return [
+    online,
+    {
+      id: 'tailscale:reachability',
+      status: ping.ok ? 'pass' : 'fail',
+      summary: ping.ok
+        ? 'Tailscale peer answered a bounded reachability probe.'
+        : 'Tailscale peer did not answer; online inventory is not proof of connectivity.',
+    },
+  ];
+}
+
+function missingMapping(
+  peer: TailnetComputer,
+  checks: Check[],
+  count: number
+): NodeReport {
+  return {
+    node: peer.dns,
+    checks: [
+      ...checks,
+      {
+        id: 'fleet:registration',
+        status: 'fail',
+        summary: count
+          ? 'Roster mapping is ambiguous between entries or computers; no SSH probe was run.'
+          : 'Computer is not registered with a verified SSH alias and checkout path; dependencies and authentication are unverified.',
+        remedy: {
+          action:
+            'Run fleet-node-add with its existing SSH alias and actual checkout path, then rerun fleet-status --tailnet.',
+        },
+      },
+    ],
+  };
+}
+
+function combine(
+  peer: TailnetComputer,
+  checks: Check[],
+  report: NodeReport,
+  reference: Snapshot
+): NodeReport {
+  return {
+    ...report,
+    node: peer.dns,
+    checks: [
+      ...checks,
+      ...report.checks,
+      ...(report.snapshot
+        ? compareFingerprints(
+            reference.fingerprint,
+            report.snapshot.fingerprint
+          )
+        : []),
+    ],
+  };
+}
+
+function absentNode(node: FleetNode): NodeReport {
+  return {
+    node: node.id,
+    checks: [
+      {
+        id: 'tailscale:membership',
+        status: 'fail',
+        summary:
+          'Registered node is absent from visible Tailscale inventory. It was not silently omitted.',
+        remedy: {
+          action:
+            'Check the node account, tailnet visibility, and the registered SSH alias.',
+        },
+      },
+    ],
+  };
+}

package/src/tailnet-matching.ts

@@ -0,0 +1,37 @@
+import type { FleetNode, Runner } from './types';
+import type { TailnetComputer } from './tailscale';
+
+export function matchesComputer(host: string, peer: TailnetComputer): boolean {
+  const name = host.toLowerCase().replace(/\.$/, '');
+  return [peer.dns, peer.dns.split('.')[0], ...peer.ips].includes(name);
+}
+
+export async function matchComputers(
+  nodes: FleetNode[],
+  peers: TailnetComputer[],
+  run: Runner
+): Promise<Map<string, FleetNode[]>> {
+  const matches = new Map(peers.map((peer) => [peer.id, [] as FleetNode[]]));
+  for (const node of nodes) {
+    const resolved = await run(
+      [
+        'ssh',
+        '-G',
+        '-o',
+        'CanonicalizeHostname=no',
+        '-o',
+        'PermitLocalCommand=no',
+        node.ssh,
+      ],
+      { timeoutMs: 5_000 }
+    );
+    const hostname = resolved.ok
+      ? /^hostname ([^\r\n]+)$/im.exec(resolved.stdout)?.[1]?.trim()
+      : undefined;
+    const targets = peers.filter((peer) =>
+      matchesComputer(hostname ?? node.ssh, peer)
+    );
+    for (const peer of targets) matches.get(peer.id)!.push(node);
+  }
+  return matches;
+}

package/src/tailscale.ts

@@ -0,0 +1,141 @@
+import { isIP } from 'node:net';
+import { record } from './config';
+import type { Check, Runner } from './types';
+
+export interface TailnetComputer {
+  id: string;
+  name: string;
+  dns: string;
+  os: string;
+  ips: string[];
+  online: boolean;
+  self: boolean;
+}
+export interface Tailnet {
+  computers: TailnetComputer[];
+  excludedMobile: number;
+  checks: Check[];
+}
+
+export async function discoverTailnet(run: Runner): Promise<Tailnet> {
+  const result = await run(['tailscale', 'status', '--json'], {
+    timeoutMs: 15_000,
+  });
+  if (!result.ok)
+    return unavailable(
+      'Tailscale status is unavailable; check the CLI, daemon and account login.'
+    );
+  try {
+    return parseTailnet(JSON.parse(result.stdout) as unknown);
+  } catch {
+    return unavailable(
+      'Tailscale returned an unsupported status document. Update the CLI and check its daemon.'
+    );
+  }
+}
+
+export function parseTailnet(data: unknown): Tailnet {
+  if (!record(data) || data['BackendState'] !== 'Running')
+    return unavailable(
+      'Tailscale is not running and signed in. Review tailscale status locally.'
+    );
+  validateStatusFields(data);
+  const peers = Object.values(
+    (data['Peer'] as Record<string, unknown> | null) ?? {}
+  );
+  if (peers.length > 1024) throw new Error('Too many peers');
+  const all = [
+    parseComputer(data['Self'], true),
+    ...peers.map((peer) => parseComputer(peer, false)),
+  ];
+  if (new Set(all.map((peer) => peer.id)).size !== all.length)
+    throw new Error('Duplicate peer');
+  const mobile = (peer: TailnetComputer) =>
+    ['ios', 'android', 'tvos'].includes(peer.os.toLowerCase());
+  const computers = all
+    .filter((peer) => !mobile(peer))
+    .sort((a, b) => a.name.localeCompare(b.name, 'en'));
+  const healthy = healthyDiscovery(data, computers.length);
+  return {
+    computers,
+    excludedMobile: all.filter(mobile).length,
+    checks: [
+      {
+        id: 'tailscale:discovery',
+        status: healthy ? 'pass' : 'fail',
+        summary: healthy
+          ? 'Signed-in Tailscale daemon returned its visible computer inventory.'
+          : 'Tailscale reports a daemon/coordination health problem or no computers; discovery may be incomplete.',
+        ...(!healthy
+          ? {
+              remedy: {
+                action:
+                  'Review tailscale status and network connectivity. Resolve coordination/daemon health before claiming complete coverage.',
+              },
+            }
+          : {}),
+      },
+    ],
+  };
+}
+
+function healthyDiscovery(
+  data: Record<string, unknown>,
+  count: number
+): boolean {
+  const health = (data['Health'] as unknown[] | null | undefined) ?? [];
+  return health.length === 0 && count > 0;
+}
+
+function validateStatusFields(data: Record<string, unknown>): void {
+  const health = data['Health'] ?? [];
+  const peers = data['Peer'] ?? {};
+  if (!record(data['Self']) || !record(peers) || !Array.isArray(health))
+    throw new Error('Unsupported Tailscale status');
+}
+
+function parseComputer(value: unknown, self: boolean): TailnetComputer {
+  if (!record(value)) throw new Error('Invalid peer');
+  const {
+    ID: id,
+    HostName: name,
+    DNSName: dns,
+    OS: os,
+    TailscaleIPs: ips,
+    Online: online,
+  } = value;
+  if (
+    ![id, name, os].every(
+      (field) =>
+        typeof field === 'string' &&
+        field.length > 0 &&
+        field.length <= 256 &&
+        !/[\x00-\x1f\x7f]/.test(field)
+    ) ||
+    typeof dns !== 'string' ||
+    !/^[a-zA-Z0-9][a-zA-Z0-9.-]{0,252}\.?$/.test(dns) ||
+    !Array.isArray(ips) ||
+    ips.length < 1 ||
+    ips.length > 16 ||
+    !ips.every((ip) => typeof ip === 'string' && isIP(ip) !== 0) ||
+    typeof online !== 'boolean'
+  )
+    throw new Error('Invalid peer fields');
+  return {
+    id: id as string,
+    name: name as string,
+    dns: dns.replace(/\.$/, '').toLowerCase(),
+    os: os as string,
+    ips: ips as string[],
+    online,
+    self,
+  };
+}
+
+function unavailable(summary: string): Tailnet {
+  return {
+    computers: [],
+    excludedMobile: 0,
+    checks: [{ id: 'tailscale:discovery', status: 'fail', summary }],
+  };
+}