@crvouga/dev-fleet 0.2.0 → 0.3.0

package/src/platform/t3-service.ts

@@ -1,3 +1,4 @@
+import policy from '../../ansible/ubuntu/policy.json';
 import type { Runner, Check } from '../types';
 
 /** The user unit that runs the pinned desktop app (UI and embedded server). */
@@ -44,6 +45,109 @@ export async function convergeT3Service(
   ];
 }
 
+/** The recovery script, installed outside every revision (`tasks/recovery.yml`). */
+export const T3_HEALTH_SCRIPT = '/usr/local/lib/personal-fleet/t3-health.sh';
+
+/** The record of the package the desktop app was unpacked from
+ * (`t3-desktop.yml`): `T3-Code-<version>-<arch>.deb`. */
+export const T3_DESKTOP_RELEASE =
+  '/opt/T3 Code (Nightly)/.personal-fleet-version';
+
+/** Where a play records a T3 restart it deferred because agent turns were
+ * running (`tasks/t3-restart.yml`): the second it was first deferred.
+ * personal-worker-recovery clears it when it restarts the idle app. */
+export function t3RestartMarker(uid: number): string {
+  return `/run/user/${uid}/personal-t3-restart-pending`;
+}
+
+/** A restart deferred while agent turns run, and whether it has waited longer
+ * than the policy allows. */
+export async function restartDeferral(
+  run: Runner,
+  uid: number,
+  now: number = Date.now()
+): Promise<{ pending: boolean; overdue: boolean }> {
+  const marker = await run(['cat', t3RestartMarker(uid)]);
+  if (!marker.ok) return { pending: false, overdue: false };
+  const since = Number(marker.stdout.trim());
+  const limit = policy.t3RestartMaxDeferralHours * 3600;
+  return {
+    pending: true,
+    overdue: Number.isFinite(since) && now / 1000 - since > limit,
+  };
+}
+
+/** The release the running server reports, compared with the one installed.
+ * Asks the server itself, so it needs no `t3` on the search path. A new
+ * release is installed beside a running app, which restarts into it only
+ * while it runs no agent turn: until then this is T3_RESTART_PENDING, and a
+ * failure once the restart has waited longer than the policy allows. */
+export async function runningVersionCheck(
+  run: Runner,
+  uid: number,
+  now: number = Date.now()
+): Promise<Check> {
+  const server = await run(
+    [
+      'curl',
+      '--fail',
+      '--silent',
+      '--max-time',
+      '10',
+      'http://127.0.0.1:3773/.well-known/t3/environment',
+    ],
+    { timeoutMs: 15_000 }
+  );
+  const installed = await run(['cat', T3_DESKTOP_RELEASE]);
+  const running = serverVersion(server.ok ? server.stdout : '');
+  const release = /^T3-Code-(.+)-[a-z0-9]+\.deb$/.exec(
+    installed.ok ? installed.stdout.trim() : ''
+  )?.[1];
+  const id = 't3:running-version';
+  if (!running || !release)
+    return {
+      id,
+      status: 'fail',
+      summary: 'T3_RUNNING_VERSION_UNKNOWN',
+      remedy: {
+        action:
+          'The running server or the installed release could not be read. Check worker:t3-health, then rerun fleet-sync to reinstall the pinned desktop app.',
+      },
+    };
+  if (running === release)
+    return {
+      id,
+      status: 'pass',
+      summary: `The running T3 server is the installed release ${release}.`,
+    };
+  const deferral = await restartDeferral(run, uid, now);
+  return {
+    id,
+    status: deferral.overdue ? 'fail' : 'warn',
+    summary: 'T3_RESTART_PENDING',
+    remedy: {
+      action: deferral.overdue
+        ? `T3 still runs ${running} after ${policy.t3RestartMaxDeferralHours} hours because agent turns kept running. Finish or stop them, or restart ${T3_DESKTOP_UNIT} on the worker (this ends its running turns).`
+        : `T3 runs ${running}; ${release} is installed. personal-worker-recovery restarts it at the first check with no agent turn running.`,
+    },
+  };
+}
+
+function serverVersion(body: string): string | undefined {
+  try {
+    const value: unknown = JSON.parse(body);
+    const version =
+      typeof value === 'object' && value !== null
+        ? (value as Record<string, unknown>)['serverVersion']
+        : undefined;
+    return typeof version === 'string' && /^[\w.+-]+$/.test(version)
+      ? version
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 /** When the desktop app last started; changes exactly when it restarts. */
 export async function t3StartedAt(run: Runner): Promise<string> {
   const shown = await run([

package/src/platform/types.ts

@@ -110,10 +110,6 @@ export type Platform = {
 
   /** App-bundled Tailscale CLI binaries that exist (used when `tailscale` is not on PATH). */
   tailscaleAppBinaries(): readonly string[];
-  /** VS Code user `settings.json`, or `null` when unknown on this OS. */
-  vscodeSettingsPath(): string | null;
-  /** VS Code `code` CLI binaries shipped inside the installed app. */
-  vscodeCliBinaries(): readonly string[];
 
   /** Install (or replace) a periodic job. Never throws — returns a result. */
   installScheduledJob(job: ScheduledJob): NotifierTestResult;

package/src/platform/windows.ts

@@ -110,20 +110,6 @@ export class WindowsPlatform implements Platform {
     );
   }
 
-  vscodeSettingsPath(): string | null {
-    const base =
-      process.env['APPDATA']?.trim() || join(homedir(), 'AppData/Roaming');
-    return join(base, 'Code/User/settings.json');
-  }
-
-  vscodeCliBinaries(): readonly string[] {
-    const base =
-      process.env['LOCALAPPDATA']?.trim() || join(homedir(), 'AppData/Local');
-    return [join(base, 'Programs/Microsoft VS Code/bin/code.cmd')].filter((p) =>
-      existsSync(p)
-    );
-  }
-
   installScheduledJob(job: ScheduledJob): NotifierTestResult {
     return {
       ok: false,

package/src/remote-entry.ts

@@ -6,6 +6,8 @@ import { syncWorkspace } from './sync';
 import { repairWorkspace, validRepairs } from './repair-runner';
 import { validSnapshot } from './protocol';
 import type { Fingerprint } from './types';
+import { machineProviders } from './t3/providers';
+import { accountContext, nodeAccountChecks } from './accounts/context';
 
 async function main(): Promise<void> {
   const raw: unknown = JSON.parse(
@@ -36,6 +38,16 @@ async function main(): Promise<void> {
       raw['expected'] as Fingerprint
     );
     process.stdout.write(JSON.stringify({ version: 1, checks }));
+  } else if (raw['action'] === 'accounts') {
+    // The fast account path: only the account-derived T3 instances. T3
+    // watches its settings, so no restart and no running turn ends.
+    if (raw['apply'] !== true)
+      throw new Error('Account propagation requires explicit authorization.');
+    const checks = [
+      await machineProviders(node.id, node.workspace, true),
+      ...(await nodeAccountChecks(await accountContext())),
+    ];
+    process.stdout.write(JSON.stringify({ version: 1, checks }));
   } else if (raw['action'] === 'sync') {
     if (raw['apply'] !== true)
       throw new Error('Sync requires explicit authorization.');
@@ -84,6 +96,8 @@ async function configureRequest(
 function validRequest(raw: unknown): raw is Record<string, unknown> {
   return (
     record(raw) &&
-    ['probe', 'sync', 'repair', 'configure'].includes(String(raw['action']))
+    ['probe', 'sync', 'repair', 'configure', 'accounts'].includes(
+      String(raw['action'])
+    )
   );
 }

package/src/service-auth.ts

@@ -42,6 +42,18 @@ export async function codingAuth(
   ];
 }
 
+/** CLIs with no sign-in of their own to check, and why. Each says where
+ * its access is proven instead; none is a silent skip.
+ */
+export const PROXY_ROUTED: Record<string, string> = {
+  gemini:
+    'gemini signs in through the CLI proxy and has no T3 launcher; provider:gemini:inference proves its route from this node.',
+  copilot:
+    'copilot is not routed through the CLI proxy and dev-fleet does not manage its GitHub sign-in; no provider account is checked.',
+  opencode:
+    'opencode signs in through the CLI proxy; agent:muse:session and the per-account agent:opencode sessions prove it from this node.',
+};
+
 export async function serviceAuth(run: Runner): Promise<Check[]> {
   const services: Service[] = [
     {
@@ -89,14 +101,12 @@ export async function serviceAuth(run: Runner): Promise<Check[]> {
   ];
   const checks: Check[] = [];
   for (const service of services) checks.push(await serviceCheck(service, run));
-  for (const id of ['gemini', 'copilot', 'opencode'])
+  for (const [id, reason] of Object.entries(PROXY_ROUTED))
     checks.push({
       id: `auth:${id}`,
       status: 'skip',
-      summary: `${id} provider authorization has no verified noninteractive status adapter in this CLI. Installed binaries or saved keys are not proof of access.`,
-      remedy: {
-        action: `Verify ${id} provider access locally using its supported account diagnostics. No prompt or billable request was made.`,
-      },
+      required: false,
+      summary: `AUTH_NOT_SUPPORTED_ON_ROUTE: ${reason}`,
     });
   return checks;
 }

package/src/t3/accounts.ts

@@ -46,6 +46,12 @@ const TOOLS: Record<
   },
 };
 
+/** The T3 instance of one account, and the T3 driver that runs its tool. */
+export const accountInstanceId = (account: Account): string =>
+  `personal_${slug(account.prefix)}`.slice(0, 64);
+export const accountDriver = (tool: string): string | undefined =>
+  TOOLS[tool]?.driver;
+
 /** One `<Provider> <account email>` instance per signed-in account, identical
  * on every machine because every machine reads the same proxy.
  */
@@ -61,7 +67,7 @@ export function accountInstances(
       ...variables,
       account: slug(account.email),
     };
-    instances[`personal_${slug(account.prefix)}`.slice(0, 64)] = {
+    instances[accountInstanceId(account)] = {
       driver: tool.driver,
       displayName: `${tool.label} ${account.email}`,
       enabled: true,

package/src/t3/models.ts

@@ -0,0 +1,133 @@
+import { join } from 'node:path';
+import type { Check } from '../types';
+import { accountDriver, accountInstanceId, type Roster } from './accounts';
+import type { Account } from '../agents/accounts';
+import { object, read } from './files';
+
+/** The models T3 offers for one instance: the catalogue T3 keeps for its
+ * driver (userdata/model-manifest.json) and any custom models the instance
+ * adds. Null when T3 has no list for the driver: it asks the tool itself.
+ */
+function offered(
+  manifest: Record<string, unknown>,
+  instance: unknown,
+  driver: string
+): string[] | null {
+  const catalogue = object(
+    JSON.stringify(
+      object(JSON.stringify(manifest['manifest'] ?? {}))['currentModels'] ?? {}
+    )
+  )[driver];
+  const config =
+    instance && typeof instance === 'object'
+      ? (instance as { config?: { customModels?: unknown } }).config
+      : undefined;
+  const custom = Array.isArray(config?.customModels)
+    ? config.customModels.flatMap((model: unknown) =>
+        typeof model === 'string'
+          ? [model]
+          : model &&
+              typeof model === 'object' &&
+              typeof (model as { slug?: unknown }).slug === 'string'
+            ? [(model as { slug: string }).slug]
+            : []
+      )
+    : [];
+  if (!Array.isArray(catalogue) && !custom.length) return null;
+  return [
+    ...new Set([
+      ...(Array.isArray(catalogue)
+        ? catalogue.filter(
+            (model): model is string => typeof model === 'string'
+          )
+        : []),
+      ...custom,
+    ]),
+  ].sort();
+}
+
+/** `t3:models`: for every account instance the fleet manages, T3 offers only
+ * models the proxy serves for that account, and every required model is both
+ * served and offered. A report names tools and models, never an account.
+ */
+export async function modelsCheck(options: {
+  t3Home: string;
+  roster: Roster;
+  required?: Record<string, string[]>;
+}): Promise<Check> {
+  const remedy = (action: string) => ({ remedy: { action } });
+  let accounts;
+  let settings;
+  let manifest;
+  try {
+    accounts = await options.roster();
+    settings = object(
+      await read(join(options.t3Home, 'userdata/settings.json'))
+    );
+    manifest = object(
+      await read(join(options.t3Home, 'userdata/model-manifest.json'))
+    );
+  } catch {
+    return {
+      id: 't3:models',
+      status: 'warn',
+      summary: 'T3_MODELS_UNVERIFIED',
+      ...remedy(
+        'The proxy’s model list or T3’s settings could not be read on this machine. Check that the CLI proxy answers and T3 has started once, then run fleet-sync.'
+      ),
+    };
+  }
+  const problems = mismatches(
+    accounts,
+    object(JSON.stringify(settings['providerInstances'] ?? {})),
+    manifest,
+    options.required ?? {}
+  );
+  if (problems.size)
+    return {
+      id: 't3:models',
+      status: 'warn',
+      summary: 'T3_MODELS_MISMATCH',
+      ...remedy(
+        `${[...problems].sort().join('; ')}. Serve the model on the CLI proxy, or remove it from t3.requiredModels in fleet.json; a model the proxy no longer serves fails when it is chosen in T3.`
+      ),
+    };
+  return {
+    id: 't3:models',
+    status: 'pass',
+    summary:
+      'Every model T3 offers for a managed account is one the proxy serves for it, and every required model is offered.',
+  };
+}
+
+/** What is wrong with the models of each managed account instance. */
+function mismatches(
+  accounts: Account[],
+  instances: Record<string, unknown>,
+  manifest: Record<string, unknown>,
+  required: Record<string, string[]>
+): Set<string> {
+  const problems = new Set<string>();
+  for (const account of accounts) {
+    const driver = accountDriver(account.tool);
+    const id = accountInstanceId(account);
+    if (!driver || !Object.hasOwn(instances, id)) continue;
+    const offers = offered(manifest, instances[id], driver);
+    for (const model of offers ?? [])
+      if (!account.models.includes(model))
+        problems.add(
+          `${account.tool}: T3 offers ${model}, which the proxy does not serve`
+        );
+    for (const model of required[account.tool] ?? []) {
+      if (!account.models.includes(model))
+        problems.add(
+          `${account.tool}: the proxy does not serve the required ${model}`
+        );
+      else if (offers && !offers.includes(model))
+        problems.add(
+          `${account.tool}: T3 does not offer the required ${model}`
+        );
+    }
+  }
+  return problems;
+}

package/src/t3/pairing.ts

@@ -0,0 +1,31 @@
+import { parseConfig } from '../config';
+import { STATE_ROSTER, readRosterText, writeRosterText } from '../state/roster';
+import { mutateState, stateContext, type StateContext } from '../state/sync';
+
+/** Records the T3 environment a worker runs in the roster, the first time a
+ * verified link from the manager reaches it. From then on the link check
+ * compares against this identity, so a worker whose T3 data was replaced is
+ * told apart from one that is only offline. An identity already recorded is
+ * never rewritten.
+ */
+export async function rememberEnvironment(
+  path: string,
+  node: string,
+  environmentId: string,
+  context: () => StateContext = stateContext
+): Promise<boolean> {
+  const save = async (): Promise<boolean> => {
+    const text = await readRosterText(path);
+    if (text === null) return false;
+    const roster = parseConfig(JSON.parse(text) as unknown);
+    const entry = roster.nodes.find((candidate) => candidate.id === node);
+    if (!entry || entry.t3EnvironmentId) return false;
+    entry.t3EnvironmentId = environmentId;
+    // Parsing again refuses an identity that is not a T3 environment ID.
+    const next = parseConfig(roster);
+    await writeRosterText(path, `${JSON.stringify(next, null, 2)}\n`);
+    return true;
+  };
+  if (path !== STATE_ROSTER) return save();
+  return (await mutateState(context(), save)).value;
+}

package/src/t3/profile.ts

@@ -0,0 +1,203 @@
+import { dirname, join } from 'node:path';
+import type { Check } from '../types';
+import { canonical, object, read, write } from './files';
+
+/** What happens to one declared setting. `release` stops managing a key the
+ * fleet no longer declares and leaves its value to the machine.
+ */
+export type ProfileAction =
+  'create' | 'update' | 'delete' | 'release' | 'unchanged' | 'conflict';
+export type ProfileChange = { key: string; action: ProfileAction };
+
+/** The values the fleet last wrote, kept beside T3's settings: the base of
+ * the three-way merge between what is declared, what T3 holds and what was
+ * applied before.
+ */
+export const profileRecord = (settings: string): string =>
+  join(dirname(settings), 'personal-profile.json');
+
+const same = (a: unknown, b: unknown): boolean => canonical(a) === canonical(b);
+
+/** Managed keys converge, keys nobody declared are left alone, and a key
+ * changed on the machine since the fleet last wrote it (or that existed
+ * before the fleet declared it) is a conflict, never overwritten.
+ */
+export function planProfile(
+  settings: Record<string, unknown>,
+  applied: Record<string, unknown>,
+  declared: Record<string, unknown>
+) {
+  const next = { ...settings };
+  const record = { ...applied };
+  const changes: ProfileChange[] = [];
+  const keys = [
+    ...new Set([...Object.keys(declared), ...Object.keys(applied)]),
+  ].sort();
+  for (const key of keys) {
+    const present = Object.hasOwn(settings, key);
+    const managed = Object.hasOwn(applied, key);
+    const action = actionFor(key, declared, {
+      present,
+      managed,
+      current: settings[key],
+      base: applied[key],
+    });
+    changes.push({ key, action });
+    if (action === 'create' || action === 'update') {
+      next[key] = declared[key];
+      record[key] = declared[key];
+    } else if (action === 'unchanged') record[key] = declared[key];
+    else if (action === 'delete') {
+      delete next[key];
+      delete record[key];
+    } else if (action === 'release') delete record[key];
+  }
+  return {
+    settings: next,
+    record,
+    changes,
+    settingsChanged: !same(next, settings),
+    recordChanged: !same(record, applied),
+  };
+}
+
+function actionFor(
+  key: string,
+  declared: Record<string, unknown>,
+  {
+    present,
+    managed,
+    current,
+    base,
+  }: { present: boolean; managed: boolean; current: unknown; base: unknown }
+): ProfileAction {
+  if (!Object.hasOwn(declared, key)) {
+    if (present && same(current, base)) return 'delete';
+    return 'release';
+  }
+  const wanted = declared[key];
+  if (present && same(current, wanted)) return 'unchanged';
+  if (!present) return managed ? 'conflict' : 'create';
+  return managed && same(current, base) ? 'update' : 'conflict';
+}
+
+export interface ProfileOptions {
+  apply: boolean;
+  /** Tries before a settings file T3 keeps rewriting is reported busy. */
+  attempts?: number;
+  pause?: () => Promise<void>;
+  /** Runs between the read and the write: where T3 may write too. */
+  beforeWrite?: () => Promise<void>;
+}
+
+/** Plans, and with `apply` writes, the declared profile. T3 writes the same
+ * file and offers no compare-and-swap, so a write happens only when the file
+ * is still what was read; otherwise the merge is planned again, a bounded
+ * number of times.
+ */
+export async function reconcileProfile(
+  settingsPath: string,
+  declared: Record<string, unknown>,
+  options: ProfileOptions
+) {
+  const recordPath = profileRecord(settingsPath);
+  const attempts = options.attempts ?? 3;
+  const pause =
+    options.pause ?? (() => new Promise((done) => setTimeout(done, 500)));
+  for (let attempt = 1; ; attempt++) {
+    const raw = await read(settingsPath);
+    const recordRaw = await read(recordPath);
+    const plan = planProfile(
+      object(raw),
+      readRecord(recordRaw, settingsPath),
+      declared
+    );
+    if (!options.apply || (!plan.settingsChanged && !plan.recordChanged))
+      return { ...plan, applied: false };
+    await options.beforeWrite?.();
+    if (!plan.settingsChanged || (await read(settingsPath)) === raw) {
+      if (plan.settingsChanged)
+        await write(
+          settingsPath,
+          JSON.stringify(plan.settings, null, 2) + '\n'
+        );
+      await write(
+        recordPath,
+        canonical({ version: 1, target: settingsPath, settings: plan.record }) +
+          '\n'
+      );
+      return { ...plan, applied: true };
+    }
+    if (attempt >= attempts) throw new Error('T3_SETTINGS_BUSY');
+    await pause();
+  }
+}
+
+function readRecord(
+  raw: string | null,
+  target: string
+): Record<string, unknown> {
+  if (raw === null) return {};
+  const record = object(raw);
+  if (record['version'] !== 1 || record['target'] !== target)
+    throw new Error('T3_PROFILE_RECORD_INVALID');
+  return object(JSON.stringify(record['settings'] ?? {}));
+}
+
+/** `t3:profile`: the declared settings on this machine. A report names the
+ * keys and what happens to them, never a value: a tool server's settings may
+ * carry a credential.
+ */
+export async function profileCheck(
+  settingsPath: string,
+  declared: Record<string, unknown>,
+  options: ProfileOptions
+): Promise<Check> {
+  const remedy = (action: string) => ({ remedy: { action } });
+  try {
+    const result = await reconcileProfile(settingsPath, declared, options);
+    const conflicts = result.changes.filter(
+      (change) => change.action === 'conflict'
+    );
+    if (conflicts.length)
+      return {
+        id: 't3:profile',
+        status: 'warn',
+        summary: 'T3_PROFILE_CONFLICT',
+        ...remedy(
+          `These T3 settings were changed on this machine since dev-fleet last wrote them, or existed before they were declared, and were left as they are: ${conflicts.map((change) => change.key).join(', ')}. Put the value every machine should hold under t3.profile in fleet.json, or remove the setting on this machine, then run fleet-sync.`
+        ),
+      };
+    const pending = result.changes.filter(
+      (change) => !['unchanged', 'conflict'].includes(change.action)
+    );
+    if (!options.apply && pending.length)
+      return {
+        id: 't3:profile',
+        status: 'warn',
+        summary: 'T3_PROFILE_PENDING',
+        ...remedy(
+          `fleet-sync would change these T3 settings: ${pending.map((change) => `${change.key} (${change.action})`).join(', ')}.`
+        ),
+      };
+    return {
+      id: 't3:profile',
+      status: 'pass',
+      summary: Object.keys(declared).length
+        ? 'T3 on this machine holds every setting of the declared profile; settings nobody declared are its own.'
+        : 'No T3 profile is declared; every setting stays local.',
+    };
+  } catch (error) {
+    const busy = error instanceof Error && error.message === 'T3_SETTINGS_BUSY';
+    return {
+      id: 't3:profile',
+      status: 'fail',
+      summary: busy ? 'T3_SETTINGS_BUSY' : 'T3_PROFILE_UNVERIFIED',
+      ...remedy(
+        busy
+          ? 'T3 kept rewriting its settings while dev-fleet merged the profile. Run fleet-sync again; nothing was overwritten.'
+          : 'Repair T3’s settings.json (or restore personal-profile.json from a backup) on this machine, then run fleet-sync.'
+      ),
+    };
+  }
+}