@crvouga/dev-fleet 0.0.0-stage → 0.2.0

package/src/t3/plan.ts

@@ -0,0 +1,82 @@
+import { FleetUsageError } from '../errors';
+import { canonical, hash, object, read } from './files';
+
+export type Change = {
+  id: string;
+  action: 'create' | 'update' | 'delete' | 'unchanged' | 'conflict';
+};
+export type T3Paths = { settings: string; state: string };
+
+export async function planT3(paths: T3Paths, desired: Record<string, unknown>) {
+  const raw = await read(paths.settings);
+  const stateRaw = await read(paths.state);
+  const settings = object(raw);
+  const current = object(JSON.stringify(settings['providerInstances'] ?? {}));
+  const owned = readOwnership(stateRaw, paths.settings);
+  const next = { ...current };
+  const changes: Change[] = [];
+  for (const [id, instance] of Object.entries(desired)) {
+    const exists = Object.hasOwn(current, id);
+    const action = actionFor(exists, owned.includes(id), current[id], instance);
+    changes.push({ id, action });
+    if (action !== 'conflict') next[id] = instance;
+  }
+  for (const id of owned.filter((id) => !Object.hasOwn(desired, id))) {
+    changes.push({
+      id,
+      action: Object.hasOwn(current, id) ? 'delete' : 'unchanged',
+    });
+    delete next[id];
+  }
+  const settingsChanged = canonical(next) !== canonical(current);
+  const state = ownership(paths.settings, Object.keys(desired));
+  return {
+    raw,
+    stateRaw,
+    settings: settingsChanged
+      ? { ...settings, providerInstances: next }
+      : settings,
+    state,
+    settingsChanged,
+    stateChanged: canonical(object(stateRaw)) !== canonical(state),
+    changes,
+    planHash: hash(
+      canonical({ target: paths.settings, raw, stateRaw, desired })
+    ),
+  };
+}
+
+export function ownership(target: string, ids: string[]) {
+  return { version: 1, target, instances: [...ids].sort() };
+}
+
+export function readOwnership(raw: string | null, target: string): string[] {
+  if (raw === null) return [];
+  const state = object(raw);
+  const ids = state['instances'];
+  if (
+    state['version'] !== 1 ||
+    state['target'] !== target ||
+    !Array.isArray(ids) ||
+    ids.some(
+      (id) =>
+        typeof id !== 'string' || !/^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/.test(id)
+    ) ||
+    new Set(ids).size !== ids.length
+  )
+    throw new FleetUsageError(
+      'Invalid T3 ownership state; restore its backup before reconciliation.'
+    );
+  return ids as string[];
+}
+
+function actionFor(
+  exists: boolean,
+  owned: boolean,
+  current: unknown,
+  desired: unknown
+): Change['action'] {
+  if (exists && !owned) return 'conflict';
+  if (!exists) return 'create';
+  return canonical(current) === canonical(desired) ? 'unchanged' : 'update';
+}

package/src/t3/providers.ts

@@ -0,0 +1,74 @@
+import type { Check } from '../types';
+import { desired, nodeId, paths } from './desired';
+import type { Roster } from './accounts';
+import { reconcileT3 } from './reconcile';
+import { stablePath, stableWorkspace } from '../platform/t3-policy';
+
+/** Every machine's T3 settings hold exactly the managed provider instances:
+ * the fixed ones plus one per account signed in on the CLI proxy. T3 watches
+ * its settings, so providers appear (and retired ones disappear) at once,
+ * without a restart. Workers additionally lock the same instances through T3
+ * administrator policy. The ownership-safe t3-apply engine preserves unrelated
+ * settings and unmanaged instances; an existing unowned instance of the same
+ * ID is reported, never overwritten.
+ */
+export async function machineProviders(
+  id: string,
+  workspace: string,
+  apply: boolean,
+  options: { t3Home?: string; roster?: Roster } = {}
+): Promise<Check> {
+  const remedy = {
+    action:
+      'Run devfleet t3-apply --dry-run on this machine and review any conflicting instance; adopt it with t3-apply --import ID only if it should be managed.',
+  };
+  try {
+    const target = paths(options, workspace);
+    // Providers launch from the stable path of the applied run checkout, not
+    // from this revision's own directory, which a later sync prunes.
+    const home = apply
+      ? await stableWorkspace(workspace)
+      : stablePath(workspace);
+    const instances = await desired(
+      target.definition,
+      home,
+      nodeId(id),
+      options.roster
+    );
+    const plan = await reconcileT3(target, instances, { apply: false });
+    const result =
+      apply && plan.ok && plan.changed
+        ? await reconcileT3(target, instances, {
+            apply: true,
+            planHash: plan.planHash,
+          })
+        : plan;
+    if (!result.ok)
+      return {
+        id: 't3:providers',
+        status: 'fail',
+        summary: 'T3_PROVIDER_CONFLICT',
+        remedy,
+      };
+    return result.changed && !apply
+      ? {
+          id: 't3:providers',
+          status: 'warn',
+          summary: 'T3_PROVIDERS_PENDING',
+          remedy,
+        }
+      : {
+          id: 't3:providers',
+          status: 'pass',
+          summary:
+            'T3 on this machine offers exactly the managed provider instances every machine gets.',
+        };
+  } catch {
+    return {
+      id: 't3:providers',
+      status: 'fail',
+      summary: 'T3_PROVIDERS_UNVERIFIED',
+      remedy,
+    };
+  }
+}

package/src/t3/reconcile.ts

@@ -0,0 +1,123 @@
+import { mkdir, rm } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { FleetUsageError } from '../errors';
+import { canonical, object, read, safePath, write } from './files';
+import { ownership, planT3, readOwnership, type T3Paths } from './plan';
+
+export async function reconcileT3(
+  paths: T3Paths,
+  desired: Record<string, unknown>,
+  options: { apply: boolean; planHash?: string }
+) {
+  if (!options.apply) return report(await planT3(paths, desired), false);
+  return locked(paths, async () => {
+    const plan = await planT3(paths, desired);
+    if (options.planHash && options.planHash !== plan.planHash)
+      throw new FleetUsageError(
+        'T3 plan is stale. Run devfleet t3-apply --dry-run again.'
+      );
+    if (plan.changes.some((change) => change.action === 'conflict'))
+      throw new FleetUsageError(
+        'T3 instance already exists without ownership. Review it and use devfleet t3-apply --import ID.'
+      );
+    if (!plan.settingsChanged && !plan.stateChanged) return report(plan, true);
+    await persist(paths, desired, plan);
+    const verified = await planT3(paths, desired);
+    if (verified.settingsChanged || verified.stateChanged)
+      throw new FleetUsageError(
+        'T3 changed after apply; reconciliation is incomplete. Replan and retry.'
+      );
+    return report(plan, true);
+  });
+}
+
+export async function importT3(paths: T3Paths, id: string) {
+  return locked(paths, async () => {
+    const raw = await read(paths.settings);
+    const instances = object(
+      JSON.stringify(object(raw)['providerInstances'] ?? {})
+    );
+    if (!Object.hasOwn(instances, id))
+      throw new FleetUsageError('T3 import requires an existing instance ID.');
+    const state = await read(paths.state);
+    const owned = readOwnership(state, paths.settings);
+    if (!owned.includes(id)) {
+      if (state !== null) await write(`${paths.state}.backup`, state);
+      if ((await read(paths.settings)) !== raw)
+        throw new FleetUsageError('T3 changed during import; retry.');
+      await write(
+        paths.state,
+        canonical(ownership(paths.settings, [...owned, id])) + '\n'
+      );
+    }
+    return { version: 1, ok: true, imported: id, changed: !owned.includes(id) };
+  });
+}
+
+async function locked<T>(paths: T3Paths, action: () => Promise<T>): Promise<T> {
+  await safePath(paths.state);
+  await mkdir(dirname(paths.state), { recursive: true, mode: 0o700 });
+  const lock = `${paths.state}.lock`;
+  try {
+    await mkdir(lock, { mode: 0o700 });
+  } catch {
+    throw new FleetUsageError(
+      'Another personal T3 operation holds the lock. Inspect interrupted operations before removing a stale lock.'
+    );
+  }
+  try {
+    return await action();
+  } finally {
+    await rm(lock, { recursive: true });
+  }
+}
+
+function report(plan: Awaited<ReturnType<typeof planT3>>, applied: boolean) {
+  return {
+    version: 1,
+    ok: !plan.changes.some((change) => change.action === 'conflict'),
+    applied,
+    changed: plan.settingsChanged || plan.stateChanged,
+    planHash: plan.planHash,
+    changes: plan.changes,
+    runtime:
+      'T3 watches settings.json; new sessions use the reconciled instances. Active sessions are not restarted.',
+  };
+}
+
+async function persist(
+  paths: T3Paths,
+  desired: Record<string, unknown>,
+  plan: Awaited<ReturnType<typeof planT3>>
+) {
+  // Keep a private recovery copy. Contents, environment and secret references never enter reports.
+  if (plan.raw !== null && plan.settingsChanged)
+    await write(`${paths.state}.settings-backup`, plan.raw);
+  if (plan.stateRaw !== null)
+    await write(`${paths.state}.backup`, plan.stateRaw);
+  if (
+    (await read(paths.settings)) !== plan.raw ||
+    (await read(paths.state)) !== plan.stateRaw
+  )
+    throw new FleetUsageError(
+      'T3 configuration changed during apply. Replan and retry.'
+    );
+  // Intent first: retry after interruption can finish creates/deletes without importing them.
+  const previous = readOwnership(plan.stateRaw, paths.settings);
+  await write(
+    paths.state,
+    canonical(
+      ownership(paths.settings, [
+        ...new Set([...previous, ...Object.keys(desired)]),
+      ])
+    ) + '\n'
+  );
+  if (plan.settingsChanged) {
+    if ((await read(paths.settings)) !== plan.raw)
+      throw new FleetUsageError(
+        'T3 changed during apply. Ownership intent was saved; replan and retry.'
+      );
+    await write(paths.settings, JSON.stringify(plan.settings, null, 2) + '\n');
+  }
+  await write(paths.state, canonical(plan.state) + '\n');
+}

package/src/t3-endpoint.ts

@@ -0,0 +1,104 @@
+import { discoverTailnet } from './tailscale';
+import type { Check, FleetNode, Runner } from './types';
+
+/** An origin no browser of the fleet uses: the private route must not let
+ * a page served from it read the worker's T3.
+ */
+export const FOREIGN_ORIGIN = 'https://origin-check.invalid';
+
+/** A fresh manager HTTPS request is transport evidence only. Never read
+ * pairing material or treat HTTP health as an authenticated tool session.
+ * A second request, from a foreign origin, must not be granted cross-origin
+ * access.
+ */
+export async function verifyT3Endpoint(
+  node: FleetNode,
+  run: Runner,
+  request: (url: string, options: RequestInit) => Promise<Response> = fetch
+): Promise<Check[]> {
+  const tailnet = await discoverTailnet(run);
+  const peer = tailnet.computers.find((entry) => entry.id === node.tailscaleId);
+  if (!peer || !/^[a-z0-9.-]+\.ts\.net$/.test(peer.dns))
+    return pending('T3_MANAGER_IDENTITY_UNVERIFIED');
+  try {
+    const response = await request('https://' + peer.dns, {
+      method: 'GET',
+      redirect: 'manual',
+      signal: AbortSignal.timeout(10_000),
+    });
+    await response.body?.cancel();
+    if (response.status !== 200) return pending('T3_MANAGER_ENDPOINT_REJECTED');
+  } catch {
+    return pending('T3_MANAGER_ENDPOINT_UNREACHABLE');
+  }
+  return [
+    {
+      id: 't3:manager-endpoint',
+      status: 'pass',
+      summary:
+        'A fresh HTTPS request from the manager reached the enrolled private endpoint with valid TLS and HTTP 200. Pairing, server compatibility and tool execution remain unverified.',
+    },
+    await originCheck('https://' + peer.dns, request),
+  ];
+}
+
+/** Grants cross-origin reads: it echoes the foreign origin, or allows any
+ * origin together with credentials.
+ */
+function permissive(headers: Headers): boolean {
+  const allowed = headers.get('access-control-allow-origin')?.trim();
+  return (
+    allowed === FOREIGN_ORIGIN ||
+    (allowed === '*' &&
+      headers.get('access-control-allow-credentials')?.trim() === 'true')
+  );
+}
+
+async function originCheck(
+  url: string,
+  request: (url: string, options: RequestInit) => Promise<Response>
+): Promise<Check> {
+  const check = (status: Check['status'], summary: string): Check => ({
+    id: 't3:manager-origin',
+    status,
+    summary,
+    ...(status === 'pass'
+      ? {}
+      : {
+          remedy: {
+            action:
+              'The private T3 route must not grant cross-origin access to pages from other sites. Restore the stock T3 server and its private Serve route from source; never add a proxy that rewrites CORS headers.',
+          },
+        }),
+  });
+  try {
+    const response = await request(url, {
+      method: 'GET',
+      redirect: 'manual',
+      headers: { Origin: FOREIGN_ORIGIN },
+      signal: AbortSignal.timeout(10_000),
+    });
+    await response.body?.cancel();
+    return permissive(response.headers)
+      ? check('fail', 'T3_MANAGER_ORIGIN_UNRESTRICTED')
+      : check(
+          'pass',
+          'A request from a foreign origin was not granted cross-origin access to the private T3 route.'
+        );
+  } catch {
+    return check('warn', 'T3_MANAGER_ORIGIN_UNVERIFIED');
+  }
+}
+function pending(code: string): Check[] {
+  return [
+    {
+      id: 't3:manager-endpoint',
+      status: 'fail',
+      summary: code,
+      remedy: {
+        action:
+          'Inspect the enrolled node’s private Serve route, T3 listener, Tailscale reachability and HTTPS capability. Restore the existing private route in source; never bypass TLS verification or interpret a local listener as manager access.',
+      },
+    },
+  ];
+}

package/src/t3-linked-check.ts

@@ -0,0 +1,123 @@
+import { sshArgs } from './ssh';
+import { record } from './config';
+import type { Check, FleetNode, Runner } from './types';
+
+/** Correlate a live SSH-verified server identity with the native manager's authenticated link. */
+export async function verifyT3Linked(
+  node: FleetNode,
+  run: Runner,
+  options: {
+    now?: () => number;
+    /** After this sync restarted the node's T3, give its link time to reconnect. */
+    reconnectMs?: number;
+    pause?: () => Promise<void>;
+  } = {}
+): Promise<Check> {
+  const now = options.now ?? Date.now;
+  const deadline = now() + (options.reconnectMs ?? 0);
+  let check = await linkedOnce(node, run, now());
+  while (check.status === 'fail' && now() < deadline) {
+    await (
+      options.pause ?? (() => new Promise((done) => setTimeout(done, 10_000)))
+    )();
+    check = await linkedOnce(node, run, now());
+  }
+  return check.status === 'fail' ? explained(node, run, check) : check;
+}
+
+/** A worker enrolled with one T3 environment that now runs another lost
+ * its identity (a reset data directory, a reinstall): the old link cannot
+ * reach it, and a new pairing is the remedy, not a reconnect.
+ */
+async function explained(
+  node: FleetNode,
+  run: Runner,
+  check: Check
+): Promise<Check> {
+  if (!node.t3EnvironmentId) return check;
+  const observed = await observedIdentity(node, run).catch(() => undefined);
+  if (!observed || observed === node.t3EnvironmentId) return check;
+  return {
+    ...check,
+    summary: 'T3_ENVIRONMENT_IDENTITY_CHANGED',
+    remedy: {
+      action:
+        'This worker now runs a different T3 environment than the one enrolled in the roster, so the manager’s link cannot reach it. Find out why its T3 data was replaced before anything else; then pair the new environment through T3 Connections and enroll it again. Fleet never copies or rewrites a T3 identity.',
+    },
+  };
+}
+
+async function linkedOnce(
+  node: FleetNode,
+  run: Runner,
+  now: number
+): Promise<Check> {
+  const unavailable: Check = {
+    id: 't3:paired-access',
+    status: 'fail',
+    summary: 'MANAGER_T3_LINK_UNVERIFIED',
+    remedy: {
+      action:
+        'Link this exact worker environment to the manager using T3 Connections once. Existing pairing identities are preserved; passwords and pairing tokens are never copied by Fleet.',
+    },
+  };
+  try {
+    const identity =
+      node.t3EnvironmentId ?? (await observedIdentity(node, run));
+    if (!identity) return unavailable;
+    const result = await run(['t3', 'environment', 'list', '--json'], {
+      timeoutMs: 30_000,
+    });
+    const links: unknown = JSON.parse(result.stdout);
+    if (!result.ok || !Array.isArray(links)) return unavailable;
+    return linkedEvidence(links, identity, unavailable, now);
+  } catch {
+    return unavailable;
+  }
+}
+
+function linkedEvidence(
+  links: unknown[],
+  environmentId: string,
+  unavailable: Check,
+  now: number
+): Check {
+  const link = links.find(
+    (value) => record(value) && value['environmentId'] === environmentId
+  );
+  if (
+    !record(link) ||
+    link['status'] !== 'reachable' ||
+    typeof link['expiresAt'] !== 'string'
+  )
+    return unavailable;
+  const expires = Date.parse(link['expiresAt']);
+  if (!Number.isFinite(expires) || expires <= now) return unavailable;
+  return {
+    ...unavailable,
+    status: expires - now < 7 * 86400000 ? 'warn' : 'pass',
+    summary:
+      expires - now < 7 * 86400000
+        ? 'Exact worker link is reachable but its manager authorization expires within seven days.'
+        : 'Native T3 authenticated link reaches the exact enrolled worker environment. This checks connected access, not a model task.',
+  };
+}
+
+async function observedIdentity(
+  node: FleetNode,
+  run: Runner
+): Promise<string | undefined> {
+  const server = await run(
+    [
+      ...sshArgs(node),
+      'curl --fail --silent --max-time 5 http://127.0.0.1:3773/.well-known/t3/environment',
+    ],
+    { timeoutMs: 15_000 }
+  );
+  const identity: unknown = JSON.parse(server.stdout);
+  return server.ok &&
+    record(identity) &&
+    typeof identity['environmentId'] === 'string'
+    ? identity['environmentId']
+    : undefined;
+}