@crvouga/dev-fleet 0.1.0 → 0.3.0

package/src/t3/profile.ts

@@ -0,0 +1,203 @@
+import { dirname, join } from 'node:path';
+import type { Check } from '../types';
+import { canonical, object, read, write } from './files';
+
+/** What happens to one declared setting. `release` stops managing a key the
+ * fleet no longer declares and leaves its value to the machine.
+ */
+export type ProfileAction =
+  'create' | 'update' | 'delete' | 'release' | 'unchanged' | 'conflict';
+export type ProfileChange = { key: string; action: ProfileAction };
+
+/** The values the fleet last wrote, kept beside T3's settings: the base of
+ * the three-way merge between what is declared, what T3 holds and what was
+ * applied before.
+ */
+export const profileRecord = (settings: string): string =>
+  join(dirname(settings), 'personal-profile.json');
+
+const same = (a: unknown, b: unknown): boolean => canonical(a) === canonical(b);
+
+/** Managed keys converge, keys nobody declared are left alone, and a key
+ * changed on the machine since the fleet last wrote it (or that existed
+ * before the fleet declared it) is a conflict, never overwritten.
+ */
+export function planProfile(
+  settings: Record<string, unknown>,
+  applied: Record<string, unknown>,
+  declared: Record<string, unknown>
+) {
+  const next = { ...settings };
+  const record = { ...applied };
+  const changes: ProfileChange[] = [];
+  const keys = [
+    ...new Set([...Object.keys(declared), ...Object.keys(applied)]),
+  ].sort();
+  for (const key of keys) {
+    const present = Object.hasOwn(settings, key);
+    const managed = Object.hasOwn(applied, key);
+    const action = actionFor(key, declared, {
+      present,
+      managed,
+      current: settings[key],
+      base: applied[key],
+    });
+    changes.push({ key, action });
+    if (action === 'create' || action === 'update') {
+      next[key] = declared[key];
+      record[key] = declared[key];
+    } else if (action === 'unchanged') record[key] = declared[key];
+    else if (action === 'delete') {
+      delete next[key];
+      delete record[key];
+    } else if (action === 'release') delete record[key];
+  }
+  return {
+    settings: next,
+    record,
+    changes,
+    settingsChanged: !same(next, settings),
+    recordChanged: !same(record, applied),
+  };
+}
+
+function actionFor(
+  key: string,
+  declared: Record<string, unknown>,
+  {
+    present,
+    managed,
+    current,
+    base,
+  }: { present: boolean; managed: boolean; current: unknown; base: unknown }
+): ProfileAction {
+  if (!Object.hasOwn(declared, key)) {
+    if (present && same(current, base)) return 'delete';
+    return 'release';
+  }
+  const wanted = declared[key];
+  if (present && same(current, wanted)) return 'unchanged';
+  if (!present) return managed ? 'conflict' : 'create';
+  return managed && same(current, base) ? 'update' : 'conflict';
+}
+
+export interface ProfileOptions {
+  apply: boolean;
+  /** Tries before a settings file T3 keeps rewriting is reported busy. */
+  attempts?: number;
+  pause?: () => Promise<void>;
+  /** Runs between the read and the write: where T3 may write too. */
+  beforeWrite?: () => Promise<void>;
+}
+
+/** Plans, and with `apply` writes, the declared profile. T3 writes the same
+ * file and offers no compare-and-swap, so a write happens only when the file
+ * is still what was read; otherwise the merge is planned again, a bounded
+ * number of times.
+ */
+export async function reconcileProfile(
+  settingsPath: string,
+  declared: Record<string, unknown>,
+  options: ProfileOptions
+) {
+  const recordPath = profileRecord(settingsPath);
+  const attempts = options.attempts ?? 3;
+  const pause =
+    options.pause ?? (() => new Promise((done) => setTimeout(done, 500)));
+  for (let attempt = 1; ; attempt++) {
+    const raw = await read(settingsPath);
+    const recordRaw = await read(recordPath);
+    const plan = planProfile(
+      object(raw),
+      readRecord(recordRaw, settingsPath),
+      declared
+    );
+    if (!options.apply || (!plan.settingsChanged && !plan.recordChanged))
+      return { ...plan, applied: false };
+    await options.beforeWrite?.();
+    if (!plan.settingsChanged || (await read(settingsPath)) === raw) {
+      if (plan.settingsChanged)
+        await write(
+          settingsPath,
+          JSON.stringify(plan.settings, null, 2) + '\n'
+        );
+      await write(
+        recordPath,
+        canonical({ version: 1, target: settingsPath, settings: plan.record }) +
+          '\n'
+      );
+      return { ...plan, applied: true };
+    }
+    if (attempt >= attempts) throw new Error('T3_SETTINGS_BUSY');
+    await pause();
+  }
+}
+
+function readRecord(
+  raw: string | null,
+  target: string
+): Record<string, unknown> {
+  if (raw === null) return {};
+  const record = object(raw);
+  if (record['version'] !== 1 || record['target'] !== target)
+    throw new Error('T3_PROFILE_RECORD_INVALID');
+  return object(JSON.stringify(record['settings'] ?? {}));
+}
+
+/** `t3:profile`: the declared settings on this machine. A report names the
+ * keys and what happens to them, never a value: a tool server's settings may
+ * carry a credential.
+ */
+export async function profileCheck(
+  settingsPath: string,
+  declared: Record<string, unknown>,
+  options: ProfileOptions
+): Promise<Check> {
+  const remedy = (action: string) => ({ remedy: { action } });
+  try {
+    const result = await reconcileProfile(settingsPath, declared, options);
+    const conflicts = result.changes.filter(
+      (change) => change.action === 'conflict'
+    );
+    if (conflicts.length)
+      return {
+        id: 't3:profile',
+        status: 'warn',
+        summary: 'T3_PROFILE_CONFLICT',
+        ...remedy(
+          `These T3 settings were changed on this machine since dev-fleet last wrote them, or existed before they were declared, and were left as they are: ${conflicts.map((change) => change.key).join(', ')}. Put the value every machine should hold under t3.profile in fleet.json, or remove the setting on this machine, then run fleet-sync.`
+        ),
+      };
+    const pending = result.changes.filter(
+      (change) => !['unchanged', 'conflict'].includes(change.action)
+    );
+    if (!options.apply && pending.length)
+      return {
+        id: 't3:profile',
+        status: 'warn',
+        summary: 'T3_PROFILE_PENDING',
+        ...remedy(
+          `fleet-sync would change these T3 settings: ${pending.map((change) => `${change.key} (${change.action})`).join(', ')}.`
+        ),
+      };
+    return {
+      id: 't3:profile',
+      status: 'pass',
+      summary: Object.keys(declared).length
+        ? 'T3 on this machine holds every setting of the declared profile; settings nobody declared are its own.'
+        : 'No T3 profile is declared; every setting stays local.',
+    };
+  } catch (error) {
+    const busy = error instanceof Error && error.message === 'T3_SETTINGS_BUSY';
+    return {
+      id: 't3:profile',
+      status: 'fail',
+      summary: busy ? 'T3_SETTINGS_BUSY' : 'T3_PROFILE_UNVERIFIED',
+      ...remedy(
+        busy
+          ? 'T3 kept rewriting its settings while dev-fleet merged the profile. Run fleet-sync again; nothing was overwritten.'
+          : 'Repair T3’s settings.json (or restore personal-profile.json from a backup) on this machine, then run fleet-sync.'
+      ),
+    };
+  }
+}

package/src/t3/version.ts

@@ -0,0 +1,119 @@
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { sshArgs } from '../ssh';
+import type { Check, FleetNode, Runner } from '../types';
+
+/** The T3 release every machine runs: `t3_version` in mise.toml. */
+export async function pinnedT3Version(workspace: string): Promise<string> {
+  const toml = await readFile(join(workspace, 'mise.toml'), 'utf8');
+  const version = /^t3_version = "([^"]+)"$/m.exec(toml)?.[1];
+  if (!version) throw new Error('mise.toml pins no T3 version.');
+  return version;
+}
+
+/** Where T3 notes the version it last started as, under its home. */
+export const STARTED_VERSION = 'userdata/desktop-last-started-version';
+
+const VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?$/;
+
+/** `t3:version`: the T3 a machine runs is the one the fleet pins, so the
+ * manager and the workers it drives never run different releases.
+ */
+export function versionCheck(
+  running: string | null,
+  pinned: string,
+  where: string,
+  install: string
+): Check {
+  const observed = running?.trim() ?? '';
+  if (!VERSION.test(observed))
+    return {
+      id: 't3:version',
+      status: 'warn',
+      summary: 'T3_VERSION_UNKNOWN',
+      remedy: {
+        action: `T3 on ${where} has not recorded the version it runs. Start T3 there once, then run fleet-sync.`,
+      },
+    };
+  if (observed !== pinned)
+    return {
+      id: 't3:version',
+      status: 'warn',
+      summary: 'T3_VERSION_SKEW',
+      remedy: {
+        action: `T3 on ${where} runs ${observed}; the fleet pins ${pinned}. ${install}`,
+      },
+    };
+  return {
+    id: 't3:version',
+    status: 'pass',
+    summary: `T3 on ${where} runs ${pinned}, the release every machine runs.`,
+  };
+}
+
+export async function managerVersion(
+  t3Home: string,
+  workspace: string
+): Promise<Check> {
+  const running = await readFile(join(t3Home, STARTED_VERSION), 'utf8').catch(
+    () => null
+  );
+  return versionCheck(
+    running,
+    await pinnedT3Version(workspace),
+    'the manager',
+    'Install that release of T3 Code on the manager. With the fleet policy staged (ansible-playbook -K ansible/macos/manager.yml) it no longer updates itself.'
+  );
+}
+
+export async function workerVersion(
+  node: FleetNode,
+  run: Runner,
+  workspace: string
+): Promise<Check> {
+  const result = await run(
+    [...sshArgs(node), `cat "$HOME/.t3/${STARTED_VERSION}"`],
+    { timeoutMs: 15_000 }
+  );
+  return versionCheck(
+    result.ok ? result.stdout : null,
+    await pinnedT3Version(workspace),
+    `worker ${node.id}`,
+    'Run fleet-sync for that worker: it installs the pinned desktop app and restarts T3 when its policy changed.'
+  );
+}
+
+/** Where T3 on macOS reads its administrator policy. */
+export const MACOS_POLICY = '/Library/Application Support/T3Code/policy.json';
+
+/** `t3:updates`: T3's own updater is off by the fleet's policy, so it changes
+ * only when the pin does.
+ */
+export async function updatesCheck(
+  policyPath: string,
+  stage: string
+): Promise<Check> {
+  let mode: unknown;
+  try {
+    const policy = JSON.parse(await readFile(policyPath, 'utf8')) as {
+      limits?: { updates?: { mode?: unknown } };
+    };
+    mode = policy.limits?.updates?.mode;
+  } catch {
+    mode = undefined;
+  }
+  return mode === 'off'
+    ? {
+        id: 't3:updates',
+        status: 'pass',
+        summary: 'T3’s own updater is off by the fleet’s policy.',
+      }
+    : {
+        id: 't3:updates',
+        status: 'warn',
+        summary: 'T3_UPDATES_UNMANAGED',
+        remedy: {
+          action: `T3 on this machine may update itself away from the pinned release. Stage the fleet's policy once, as an administrator: ${stage}`,
+        },
+      };
+}

package/src/t3-endpoint.ts

@@ -1,8 +1,15 @@
 import { discoverTailnet } from './tailscale';
 import type { Check, FleetNode, Runner } from './types';
 
+/** An origin no browser of the fleet uses: the private route must not let
+ * a page served from it read the worker's T3.
+ */
+export const FOREIGN_ORIGIN = 'https://origin-check.invalid';
+
 /** A fresh manager HTTPS request is transport evidence only. Never read
  * pairing material or treat HTTP health as an authenticated tool session.
+ * A second request, from a foreign origin, must not be granted cross-origin
+ * access.
  */
 export async function verifyT3Endpoint(
   node: FleetNode,
@@ -21,17 +28,66 @@ export async function verifyT3Endpoint(
     });
     await response.body?.cancel();
     if (response.status !== 200) return pending('T3_MANAGER_ENDPOINT_REJECTED');
-    return [
-      {
-        id: 't3:manager-endpoint',
-        status: 'pass',
-        summary:
-          'A fresh HTTPS request from the manager reached the enrolled private endpoint with valid TLS and HTTP 200. Pairing, server compatibility and tool execution remain unverified.',
-      },
-    ];
   } catch {
     return pending('T3_MANAGER_ENDPOINT_UNREACHABLE');
   }
+  return [
+    {
+      id: 't3:manager-endpoint',
+      status: 'pass',
+      summary:
+        'A fresh HTTPS request from the manager reached the enrolled private endpoint with valid TLS and HTTP 200. Pairing, server compatibility and tool execution remain unverified.',
+    },
+    await originCheck('https://' + peer.dns, request),
+  ];
+}
+
+/** Grants cross-origin reads: it echoes the foreign origin, or allows any
+ * origin together with credentials.
+ */
+function permissive(headers: Headers): boolean {
+  const allowed = headers.get('access-control-allow-origin')?.trim();
+  return (
+    allowed === FOREIGN_ORIGIN ||
+    (allowed === '*' &&
+      headers.get('access-control-allow-credentials')?.trim() === 'true')
+  );
+}
+
+async function originCheck(
+  url: string,
+  request: (url: string, options: RequestInit) => Promise<Response>
+): Promise<Check> {
+  const check = (status: Check['status'], summary: string): Check => ({
+    id: 't3:manager-origin',
+    status,
+    summary,
+    ...(status === 'pass'
+      ? {}
+      : {
+          remedy: {
+            action:
+              'The private T3 route must not grant cross-origin access to pages from other sites. Restore the stock T3 server and its private Serve route from source; never add a proxy that rewrites CORS headers.',
+          },
+        }),
+  });
+  try {
+    const response = await request(url, {
+      method: 'GET',
+      redirect: 'manual',
+      headers: { Origin: FOREIGN_ORIGIN },
+      signal: AbortSignal.timeout(10_000),
+    });
+    await response.body?.cancel();
+    return permissive(response.headers)
+      ? check('fail', 'T3_MANAGER_ORIGIN_UNRESTRICTED')
+      : check(
+          'pass',
+          'A request from a foreign origin was not granted cross-origin access to the private T3 route.'
+        );
+  } catch {
+    return check('warn', 'T3_MANAGER_ORIGIN_UNVERIFIED');
+  }
 }
 function pending(code: string): Check[] {
   return [

package/src/t3-linked-check.ts

@@ -22,7 +22,29 @@ export async function verifyT3Linked(
     )();
     check = await linkedOnce(node, run, now());
   }
-  return check;
+  return check.status === 'fail' ? explained(node, run, check) : check;
+}
+
+/** A worker enrolled with one T3 environment that now runs another lost
+ * its identity (a reset data directory, a reinstall): the old link cannot
+ * reach it, and a new pairing is the remedy, not a reconnect.
+ */
+async function explained(
+  node: FleetNode,
+  run: Runner,
+  check: Check
+): Promise<Check> {
+  if (!node.t3EnvironmentId) return check;
+  const observed = await observedIdentity(node, run).catch(() => undefined);
+  if (!observed || observed === node.t3EnvironmentId) return check;
+  return {
+    ...check,
+    summary: 'T3_ENVIRONMENT_IDENTITY_CHANGED',
+    remedy: {
+      action:
+        'This worker now runs a different T3 environment than the one enrolled in the roster, so the manager’s link cannot reach it. Find out why its T3 data was replaced before anything else; then pair the new environment through T3 Connections and enroll it again. Fleet never copies or rewrites a T3 identity.',
+    },
+  };
 }
 
 async function linkedOnce(
@@ -81,7 +103,7 @@ function linkedEvidence(
   };
 }
 
-async function observedIdentity(
+export async function observedIdentity(
   node: FleetNode,
   run: Runner
 ): Promise<string | undefined> {

package/src/toolchain-checks.ts

@@ -41,6 +41,29 @@ export async function toolchainChecks(
   }
 }
 
+/** Checks of the doctor that are not a tool version, with their own words. */
+const OWN: Record<string, { pass: string; fail: string; action: string }> = {
+  'global-mise-config': {
+    pass: 'The global mise configuration is the one fleet-sync publishes.',
+    fail: 'The global mise configuration is missing, was replaced by another project or tool, or is not the one this revision publishes.',
+    action:
+      'Run devfleet fleet-sync from the project that holds fleet.json. On a fleet machine the global mise configuration belongs to dev-fleet: a project pins its own tools in a project mise.toml and never publishes a global one.',
+  },
+};
+
+/** A doctor check with its own words, or null for a tool version. */
+function ownCheck(id: string, status: Check['status']): Check | null {
+  if (!Object.hasOwn(OWN, id)) return null;
+  const own = OWN[id]!;
+  const ok = status === 'pass';
+  return {
+    id: `tool:${id}`,
+    status,
+    summary: ok ? own.pass : own.fail,
+    ...(!ok ? { remedy: { action: own.action } } : {}),
+  };
+}
+
 function toolCheck(value: unknown): Check | null {
   if (
     !record(value) ||
@@ -49,6 +72,8 @@ function toolCheck(value: unknown): Check | null {
     !['pass', 'fail', 'warn', 'skip'].includes(String(value['status']))
   )
     return null;
+  const own = ownCheck(value['id'], value['status'] as Check['status']);
+  if (own) return own;
   const ok = value['status'] === 'pass';
   const version =
     typeof value['detail'] === 'string' &&

package/src/transport.ts

@@ -21,7 +21,7 @@ export async function bundleProbe(): Promise<string> {
 export interface RemoteOptions {
   source: string;
   timeoutMs: number;
-  action: 'probe' | 'sync' | 'repair' | 'configure';
+  action: 'probe' | 'sync' | 'repair' | 'configure' | 'accounts';
   configuration?: ConfigurationRequest;
   repairs?: AutomatedRepair[];
   expected?: Fingerprint;
@@ -41,7 +41,8 @@ export async function inspectRemote(
       return { node: node.id, checks: [sshFailure(preflight)] };
   }
   const target =
-    options.action === 'probe' && options.expected
+    (options.action === 'probe' || options.action === 'accounts') &&
+    options.expected
       ? await managedProbeTarget(node, options.expected.head, run)
       : { node, checks: [] };
   const command = remoteCommand(target.node, {

package/src/types.ts

@@ -80,6 +80,10 @@ export interface CommandOptions {
   input?: string;
   timeoutMs?: number;
   env?: NodeJS.ProcessEnv;
+  /** Run with `env` alone instead of on top of this process's environment:
+   * for a launch that must see only what a service would.
+   */
+  isolatedEnv?: boolean;
 }
 
 export interface CommandResult {

package/src/worker-access-checks.ts

@@ -1,5 +1,6 @@
 import type { Check, Runner } from './types';
-import { fleet, secretVia } from './fleet-config';
+import { fleet } from './fleet-config';
+import { secretsProvider } from './secrets';
 import { registry } from './agents/registry';
 import { resolveVaultConfig, usesVault } from './vault-config';
 import { record } from './config';
@@ -26,7 +27,29 @@ export async function workerAccessChecks(run: Runner): Promise<Check[]> {
     ...(usesVault() ? [await vaultRenewalCheck(run)] : []),
   ];
 }
+/** Vault refuses a read the token's policy does not allow with a 403. */
+const PERMISSION_DENIED = /permission denied|Code: 403/i;
+
 async function proxyAuthentication(run: Runner): Promise<Check> {
+  // Read through the declared provider, remembering only whether Vault said
+  // the policy forbids it: never what it printed.
+  let denied = false;
+  const secret = secretsProvider(fleet.secrets, {
+    run: async (argv) => {
+      const result = await run(argv, { timeoutMs: 20_000 });
+      if (!result.ok && PERMISSION_DENIED.test(result.stderr)) denied = true;
+      return result.ok ? result.stdout : null;
+    },
+  });
+  const forbidden: Check = {
+    id: 'worker:proxy-authentication',
+    status: 'fail',
+    summary: 'VAULT_POLICY_PERMISSION_REQUIRED',
+    remedy: {
+      action:
+        'This worker’s Vault identity was refused the proxy client key by its policy. Grant that identity read access to the declared location in the Vault policy you manage. fleet-sync seeds no secret, writes no policy and adds no secret store.',
+    },
+  };
   const unavailable: Check = {
     id: 'worker:proxy-authentication',
     status: 'fail',
@@ -37,7 +60,8 @@ async function proxyAuthentication(run: Runner): Promise<Check> {
     },
   };
   try {
-    const key = (await secretVia(run)('proxy-client-key')) ?? '';
+    const key = (await secret('proxy-client-key')) ?? '';
+    if (denied) return forbidden;
     if (!/^[\x21-\x7e]{16,4096}$/.test(key)) return unavailable;
     const result = await run(
       [
@@ -103,13 +127,42 @@ async function vaultRenewalCheck(run: Runner): Promise<Check> {
       token['ttl'] <= 1200
     )
       return unavailable;
+    if (!(await lastRenewalSucceeded(run)))
+      return {
+        ...unavailable,
+        summary: 'VAULT_RENEWAL_FAILING',
+        remedy: {
+          action:
+            'The identity is renewable, but the last run of the renewal job failed. Inspect it on this worker with: journalctl --user -u personal-worker-auth.service. Its token expires unless a renewal succeeds; no credentials are copied between machines.',
+        },
+      };
     return {
       ...unavailable,
       status: 'pass',
       summary:
-        'This worker has a renewable periodic Vault identity with sufficient lifetime for the renewal timer.',
+        'This worker has a renewable periodic Vault identity with sufficient lifetime for the renewal timer, and its last renewal succeeded.',
     };
   } catch {
     return unavailable;
   }
 }
+
+/** An active timer is not evidence that renewal works: the result of the
+ * job's last run is. A job that has not run yet has nothing to say.
+ */
+async function lastRenewalSucceeded(run: Runner): Promise<boolean> {
+  const shown = await run([
+    'systemctl',
+    '--user',
+    'show',
+    'personal-worker-auth.service',
+    '--property=Result,ExecMainStatus',
+  ]);
+  if (!shown.ok) return true;
+  const result = /^Result=(.*)$/m.exec(shown.stdout)?.[1];
+  const status = /^ExecMainStatus=(.*)$/m.exec(shown.stdout)?.[1];
+  return (
+    (result === undefined || result === 'success') &&
+    (status === undefined || status === '0')
+  );
+}