@crvouga/dev-fleet 0.2.0 → 0.3.0

package/src/definition.ts

@@ -5,6 +5,12 @@ import { parseConfig, record } from './config';
 import type { FleetConfig } from './types';
 import { parseSecretsConfig, type SecretsConfig } from './secrets';
 import { parseLocation, type BucketLocation } from './state/bucket';
+import {
+  parseIdes,
+  parseT3,
+  type IdeId,
+  type T3Definition,
+} from './ide/definition';
 
 export interface MachineDefinition {
   id: string;
@@ -40,7 +46,7 @@ export interface FleetDefinition {
   workers: MachineDefinition[];
   services: {
     /** Public HTTPS origin of the CLI proxy every node uses. */
-    proxy: { endpoint: string };
+    proxy: { endpoint: string; accounts?: AccountPolicy };
     monitoring: {
       hubPort: 8443;
       /** Public dashboard origin. */
@@ -52,8 +58,28 @@ export interface FleetDefinition {
     };
   };
   worker?: WorkerPolicy;
+  /** The IDEs every machine runs; `["t3"]` when not named (idesOf). */
+  ides?: IdeId[];
+  /** What every machine's T3 holds beyond its provider instances. */
+  t3?: T3Definition;
   ubuntu: { release: '26.04'; architectures: ['amd64', 'arm64'] };
 }
+/** How provider accounts are watched and chosen (src/accounts/). */
+export interface AccountPolicy {
+  /** Fewer usable accounts of a provider than this is a warning; none fails. */
+  minimumUsable?: number;
+  /** Per provider (`claude`, `codex`, `antigravity`): which account an
+   * unpinned request goes to first. Accounts cooling down are skipped.
+   */
+  selection?: Record<string, AccountSelection>;
+}
+export interface AccountSelection {
+  strategy: 'fill-first' | 'round-robin' | 'least-recently-limited';
+  /** Account labels (sign-in emails) first, in this order; the others
+   * follow in the order of their routing prefix.
+   */
+  order?: string[];
+}
 const ID = /^[a-z][a-z0-9-]{0,62}$/;
 const HOST = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,252}$/;
 function fail(): never {
@@ -148,6 +174,8 @@ export function parseDefinition(value: unknown): FleetDefinition {
       'services',
       'ubuntu',
       'worker',
+      'ides',
+      't3',
     ])
   )
     return fail();
@@ -169,6 +197,9 @@ export function parseDefinition(value: unknown): FleetDefinition {
     ...(value['worker'] === undefined
       ? {}
       : { worker: parseWorkerPolicy(value['worker']) }),
+    // Kept only when named: an earlier revision must still read the copy.
+    ...(value['ides'] === undefined ? {} : { ides: parseIdes(value['ides']) }),
+    ...(value['t3'] === undefined ? {} : { t3: parseT3(value['t3']) }),
   };
 }
 /** Singleton services name no host: they always run on the manager. */
@@ -177,15 +208,66 @@ function services(value: unknown): FleetDefinition['services'] {
   const { proxy, monitoring } = value;
   if (
     !record(proxy) ||
-    !keys(proxy, ['endpoint']) ||
+    !keys(proxy, ['endpoint', 'accounts']) ||
     !origin(proxy['endpoint'])
   )
     return fail();
   return {
-    proxy: { endpoint: proxy['endpoint'] },
+    proxy: {
+      endpoint: proxy['endpoint'],
+      ...(proxy['accounts'] === undefined
+        ? {}
+        : { accounts: accountPolicy(proxy['accounts']) }),
+    },
     monitoring: monitoringService(monitoring),
   };
 }
+const STRATEGIES = ['fill-first', 'round-robin', 'least-recently-limited'];
+function accountPolicy(value: unknown): AccountPolicy {
+  if (!record(value) || !keys(value, ['minimumUsable', 'selection']))
+    return fail();
+  const { minimumUsable, selection } = value;
+  if (
+    minimumUsable !== undefined &&
+    !(Number.isInteger(minimumUsable) && Number(minimumUsable) >= 1)
+  )
+    return fail();
+  return {
+    ...(minimumUsable === undefined
+      ? {}
+      : { minimumUsable: minimumUsable as number }),
+    ...(selection === undefined
+      ? {}
+      : { selection: selectionPolicies(selection) }),
+  };
+}
+function selectionPolicies(value: unknown): Record<string, AccountSelection> {
+  if (!record(value)) return fail();
+  const providers: Record<string, AccountSelection> = {};
+  for (const [provider, entry] of Object.entries(value)) {
+    if (!/^[a-z][a-z0-9-]{0,31}$/.test(provider)) return fail();
+    providers[provider] = accountSelection(entry);
+  }
+  return providers;
+}
+function accountSelection(entry: unknown): AccountSelection {
+  if (
+    !record(entry) ||
+    !keys(entry, ['strategy', 'order']) ||
+    !STRATEGIES.includes(String(entry['strategy']))
+  )
+    return fail();
+  const { order } = entry;
+  if (
+    order !== undefined &&
+    !(Array.isArray(order) && order.every((label) => account(label)))
+  )
+    return fail();
+  return {
+    strategy: entry['strategy'] as AccountSelection['strategy'],
+    ...(order ? { order: order as string[] } : {}),
+  };
+}
 function monitoringService(
   value: unknown
 ): FleetDefinition['services']['monitoring'] {

package/src/ide/definition.ts

@@ -0,0 +1,86 @@
+import { FleetUsageError } from '../errors';
+import { record } from '../config';
+
+/** The IDEs dev-fleet supports. The support matrix in the README lists the
+ * same; enabling one on the fleet is `ides` in fleet.json, not new code.
+ */
+export const SUPPORTED_IDES = ['t3'] as const;
+export type IdeId = (typeof SUPPORTED_IDES)[number];
+
+/** What `t3` in fleet.json declares for every machine's T3. */
+export interface T3Definition {
+  /** Top-level T3 settings every machine holds the same: tool servers,
+   * projects, selected preferences. A key not named here stays local.
+   */
+  profile?: Record<string, unknown>;
+  /** Models T3 must offer for every account of a tool, by tool. */
+  requiredModels?: Record<string, string[]>;
+}
+
+const KEY = /^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/;
+/** Settings another part of dev-fleet owns. */
+const RESERVED = ['providerInstances'];
+
+function invalid(detail: string): never {
+  throw new FleetUsageError(`Invalid fleet.json: ${detail}`);
+}
+
+/** The IDEs a fleet runs: those it names, or T3 alone. */
+export const idesOf = (definition: { ides?: readonly IdeId[] }): IdeId[] => [
+  ...(definition.ides ?? ['t3']),
+];
+
+export function parseIdes(value: unknown): IdeId[] {
+  if (
+    !Array.isArray(value) ||
+    value.length === 0 ||
+    new Set(value).size !== value.length ||
+    value.some((id) => !SUPPORTED_IDES.includes(id as IdeId))
+  )
+    invalid(`ides lists each of ${SUPPORTED_IDES.join(', ')} at most once.`);
+  return value as IdeId[];
+}
+
+export function parseT3(value: unknown): T3Definition {
+  if (
+    !record(value) ||
+    Object.keys(value).some(
+      (key) => !['profile', 'requiredModels'].includes(key)
+    )
+  )
+    invalid('t3 holds only profile and requiredModels.');
+  const result: T3Definition = {};
+  if (value['profile'] !== undefined) {
+    const profile = value['profile'];
+    if (
+      !record(profile) ||
+      Object.keys(profile).some(
+        (key) => !KEY.test(key) || RESERVED.includes(key)
+      ) ||
+      JSON.stringify(profile).length > 65_536
+    )
+      invalid(
+        't3.profile names top-level T3 settings (not providerInstances) and stays under 64 KiB.'
+      );
+    result.profile = profile;
+  }
+  if (value['requiredModels'] !== undefined) {
+    const required = value['requiredModels'];
+    if (
+      !record(required) ||
+      Object.entries(required).some(
+        ([tool, models]) =>
+          !KEY.test(tool) ||
+          !Array.isArray(models) ||
+          models.some(
+            (model) =>
+              typeof model !== 'string' ||
+              !/^[A-Za-z0-9._:-]{1,128}$/.test(model)
+          )
+      )
+    )
+      invalid('t3.requiredModels lists model names by tool.');
+    result.requiredModels = required as Record<string, string[]>;
+  }
+  return result;
+}

package/src/ide/ide.ts

@@ -0,0 +1,43 @@
+import type { Check, FleetNode, Runner } from '../types';
+
+/** What every supported IDE is asked for. An implementation owns its own
+ * install, service and policy (an Ansible role per IDE, with the common
+ * variable contract: version, checksum, policy, instances, profile) and is
+ * reached only through these calls, so the orchestration never names one.
+ */
+export interface Ide {
+  /** The name the fleet configuration enables it by, under `ides`. */
+  readonly id: string;
+  /** The IDE's settings on this machine: provider instances from the account
+   * roster, the declared profile and the models it offers. Run on the manager
+   * and on every worker by the machine itself.
+   */
+  settings(context: MachineContext): Promise<Check[]>;
+  /** The manager's own IDE: its settings, its pinned version and its policy. */
+  manager(context: MachineContext): Promise<Check[]>;
+  /** What the manager measures of a worker's IDE: remote access, pairing and
+   * the version the worker runs.
+   */
+  worker(node: FleetNode, context: WorkerContext): Promise<Check[]>;
+}
+
+export interface MachineContext {
+  /** The node ID this machine is in the fleet. */
+  id: string;
+  /** The checkout this run applies. */
+  workspace: string;
+  apply: boolean;
+  run: Runner;
+}
+
+export interface WorkerContext {
+  run: Runner;
+  /** The manager's checkout of the revision this sync applies. */
+  workspace: string;
+  /** The checks this sync already reported for the worker. */
+  earlier: Check[];
+  /** Records what identifies the worker's IDE in the fleet state, the first
+   * time it is seen through a verified link.
+   */
+  remember?: (node: string, identity: string) => Promise<void>;
+}

package/src/ide/index.ts

@@ -0,0 +1,5 @@
+import type { Ide } from './ide';
+import { t3Ide } from './t3';
+
+/** Every IDE dev-fleet implements, by the ID fleet.json enables it with. */
+export const IDES: Readonly<Record<string, Ide>> = { t3: t3Ide() };

package/src/ide/orchestrate.ts

@@ -0,0 +1,50 @@
+import type { Check, FleetNode } from '../types';
+import type { Ide, MachineContext, WorkerContext } from './ide';
+
+/** The implementations the fleet enables, in the order it names them. An ID
+ * nobody implements is refused before anything runs.
+ */
+export function enabled(
+  ids: readonly string[],
+  registry: Readonly<Record<string, Ide>>
+): Ide[] {
+  return ids.map((id) => {
+    const ide = registry[id];
+    if (!ide) throw new Error(`Unsupported IDE: ${id}`);
+    return ide;
+  });
+}
+
+export async function settingsChecks(
+  ides: Ide[],
+  context: MachineContext
+): Promise<Check[]> {
+  const checks: Check[] = [];
+  for (const ide of ides) checks.push(...(await ide.settings(context)));
+  return checks;
+}
+
+export async function managerChecks(
+  ides: Ide[],
+  context: MachineContext
+): Promise<Check[]> {
+  const checks: Check[] = [];
+  for (const ide of ides) checks.push(...(await ide.manager(context)));
+  return checks;
+}
+
+export async function workerChecks(
+  ides: Ide[],
+  node: FleetNode,
+  context: WorkerContext
+): Promise<Check[]> {
+  const checks: Check[] = [];
+  for (const ide of ides)
+    checks.push(
+      ...(await ide.worker(node, {
+        ...context,
+        earlier: [...context.earlier, ...checks],
+      }))
+    );
+  return checks;
+}

package/src/ide/t3.ts

@@ -0,0 +1,111 @@
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import type { Check, FleetNode } from '../types';
+import type { Ide, MachineContext, WorkerContext } from './ide';
+import { fleet } from '../fleet-config';
+import { machineProviders } from '../t3/providers';
+import { providerNames } from '../t3/names-check';
+import { profileCheck } from '../t3/profile';
+import { modelsCheck } from '../t3/models';
+import { proxyAccounts, type Roster } from '../t3/accounts';
+import {
+  MACOS_POLICY,
+  managerVersion,
+  updatesCheck,
+  workerVersion,
+} from '../t3/version';
+import { verifyT3Endpoint } from '../t3-endpoint';
+import { observedIdentity, verifyT3Linked } from '../t3-linked-check';
+import { T3_RESTARTED_SUMMARY } from '../platform/t3-policy';
+import { ansibleCommand } from '../platform/ansible';
+
+/** Where T3 keeps its data on this machine. */
+const t3Home = (): string =>
+  resolve(process.env['T3CODE_HOME'] ?? join(homedir(), '.t3'));
+
+/** What the T3 checks read besides the machine: injected by scenarios. */
+export interface T3Inputs {
+  t3Home?: string;
+  roster?: Roster;
+  profile?: Record<string, unknown>;
+  requiredModels?: Record<string, string[]>;
+  policyPath?: string;
+}
+
+/** T3 Code, the first supported IDE. Its install, service and policy on
+ * workers are the `ansible/ubuntu/t3*` playbooks and on the manager
+ * `ansible/macos/t3-policy.yml`; everything else is reached from here.
+ */
+export function t3Ide(inputs: T3Inputs = {}): Ide {
+  const home = () => inputs.t3Home ?? t3Home();
+  const settings = async (context: MachineContext): Promise<Check[]> => {
+    const required = inputs.requiredModels ?? fleet.t3?.requiredModels;
+    const options = {
+      t3Home: home(),
+      ...(inputs.roster ? { roster: inputs.roster } : {}),
+    };
+    return [
+      await machineProviders(
+        context.id,
+        context.workspace,
+        context.apply,
+        options
+      ),
+      await providerNames(
+        context.id,
+        context.workspace,
+        context.apply,
+        options
+      ),
+      await profileCheck(
+        join(home(), 'userdata/settings.json'),
+        inputs.profile ?? fleet.t3?.profile ?? {},
+        { apply: context.apply }
+      ),
+      await modelsCheck({
+        t3Home: home(),
+        roster: inputs.roster ?? proxyAccounts,
+        ...(required ? { required } : {}),
+      }),
+    ];
+  };
+  return {
+    id: 't3',
+    settings,
+    async manager(context) {
+      return [
+        ...(await settings(context)),
+        await managerVersion(home(), context.workspace),
+        await updatesCheck(
+          inputs.policyPath ?? MACOS_POLICY,
+          `cd ${context.workspace} && ${ansibleCommand().join(' ')} -K ansible/macos/manager.yml`
+        ),
+      ];
+    },
+    async worker(node: FleetNode, context: WorkerContext) {
+      const checks = [...(await verifyT3Endpoint(node, context.run))];
+      // A worker T3 restarted by this sync (idle policy heal) needs a minute
+      // or two before the manager's link reconnects.
+      const restarted = context.earlier.some(
+        (check) => check.summary === T3_RESTARTED_SUMMARY
+      );
+      const linked = await verifyT3Linked(node, context.run, {
+        reconnectMs: restarted ? 180_000 : 0,
+      });
+      checks.push(linked);
+      if (
+        linked.status !== 'fail' &&
+        !node.t3EnvironmentId &&
+        context.remember
+      ) {
+        const identity = await observedIdentity(node, context.run).catch(
+          () => undefined
+        );
+        if (identity)
+          await context.remember(node.id, identity).catch(() => undefined);
+      }
+      checks.push(await workerVersion(node, context.run, context.workspace));
+      return checks;
+    },
+  };
+}

package/src/manager-sync.ts

@@ -385,21 +385,7 @@ function mayDeploy(result: ManagerNode, nodes: FleetNode[]): boolean {
  */
 export function blocksPromotion(check: Check): boolean {
   if (!blocking(check)) return false;
-  if (check.id === 't3:service' && T3_MAINTENANCE.has(check.summary))
-    return false;
-  return !/^(worker:|beszel:|provider:|agent:|proxy:|t3:end-to-end$|t3:paired-access$|t3:policy-loaded$|ubuntu:boot-policy$)/.test(
+  return !/^(worker:|beszel:|provider:|agent:|proxy:|t3:end-to-end$|t3:paired-access$|t3:policy-loaded$|t3:running-version$|ubuntu:boot-policy$)/.test(
     check.id
   );
 }
-
-/** Native T3 outcomes that need an operator maintenance window (an active
- * listener or running release must never be killed by routine sync). They are
- * reported as failures but do not block independent OS policy on other nodes.
- */
-const T3_MAINTENANCE = new Set([
-  'T3_LISTENER_CONFLICT',
-  'T3_ACTIVE_LISTENER_MIGRATION_REQUIRED',
-  'T3_MAINTENANCE_REQUIRED',
-  'T3_POLICY_RESTART_PENDING',
-  'T3_DESKTOP_MIGRATION_REQUIRED',
-]);

package/src/node-converge-linux.ts

@@ -6,12 +6,19 @@ import { verifyUbuntuBoot } from './platform/ubuntu-verify';
 import { verifyResourceSafety } from './platform/resource-verify';
 import { convergeT3Route, T3_ROUTE } from './platform/t3-route';
 import { convergeProxy } from './proxy-converge';
-import { providerNames } from './t3/names-check';
-import { machineProviders } from './t3/providers';
+import { IDES } from './ide';
+import { enabled, settingsChecks } from './ide/orchestrate';
+import { idesOf } from './ide/definition';
+import { fleet } from './fleet-config';
 import { quotaLimited, verifyFleetProviders } from './agents/fleet-verify';
 import { verifyNativeAgents } from './agents/native-verify';
+import { accountContext, nodeAccountChecks } from './accounts/context';
 import { managedT3, managedT3Desktop } from '../bootstrap/native-t3';
-import { convergeT3Service, t3StartedAt } from './platform/t3-service';
+import {
+  convergeT3Service,
+  runningVersionCheck,
+  t3StartedAt,
+} from './platform/t3-service';
 import { userInfo } from 'node:os';
 import { resolve } from 'node:path';
 import { auditUbuntu } from './platform/ubuntu-audit';
@@ -74,12 +81,19 @@ export async function convergeLinux(
     verifyFleetProviders(node)
   );
   checks.push(...providers);
+  const accounts = await accountContext();
+  checks.push(
+    ...(await attempt('accounts:verification', () =>
+      nodeAccountChecks(accounts)
+    ))
+  );
   checks.push(
     ...(await attempt('agent:verification', () =>
       verifyNativeAgents(node, root, run, {
         outOfQuota: quotaLimited(providers),
         // Where T3 starts them: the environment of its user service.
         t3Service: true,
+        ...(accounts ? { accounts } : {}),
       })
     ))
   );
@@ -236,13 +250,20 @@ async function convergeT3(
         restarted: startedAt !== (await t3StartedAt(run)),
       })
     );
-  // Settings are watched by T3: providers arrive and stale ones leave at once.
-  checks.push(await machineProviders(node, root, true));
-  // Names live in the same settings, so they conform without a restart.
-  checks.push(await providerNames(node, root, true));
+  // Settings are watched by T3: providers, names and the profile apply
+  // without a restart.
+  checks.push(
+    ...(await settingsChecks(enabled(idesOf(fleet), IDES), {
+      id: node,
+      workspace: root,
+      apply: true,
+      run,
+    }))
+  );
   const serviceReady = checks.some(
     (check) => check.id === 't3:service' && check.status === 'pass'
   );
+  if (serviceReady) checks.push(await runningVersionCheck(run, userInfo().uid));
   if (serviceReady)
     checks.push(
       ...(await convergeT3Route(run, () => publishT3Route(root, run)))

package/src/platform/darwin.ts

@@ -176,19 +176,6 @@ export class DarwinPlatform implements Platform {
     );
   }
 
-  vscodeSettingsPath(): string | null {
-    return join(
-      homedir(),
-      'Library/Application Support/Code/User/settings.json'
-    );
-  }
-
-  vscodeCliBinaries(): readonly string[] {
-    return [
-      '/Applications/Visual Studio Code.app/Contents/Resources/app/bin/code',
-    ].filter((p) => existsSync(p));
-  }
-
   installScheduledJob(job: ScheduledJob): NotifierTestResult {
     return installLaunchdJob(job);
   }

package/src/platform/fallback.ts

@@ -78,14 +78,6 @@ export class FallbackPlatform implements Platform {
     return [];
   }
 
-  vscodeSettingsPath(): string | null {
-    return null;
-  }
-
-  vscodeCliBinaries(): readonly string[] {
-    return [];
-  }
-
   installScheduledJob(job: ScheduledJob): NotifierTestResult {
     return {
       ok: false,

package/src/platform/linux.ts

@@ -115,16 +115,6 @@ export class LinuxPlatform implements Platform {
     );
   }
 
-  vscodeSettingsPath(): string | null {
-    const base =
-      process.env['XDG_CONFIG_HOME']?.trim() || join(homedir(), '.config');
-    return join(base, 'Code/User/settings.json');
-  }
-
-  vscodeCliBinaries(): readonly string[] {
-    return ['/usr/share/code/bin/code'].filter((p) => existsSync(p));
-  }
-
   installScheduledJob(job: ScheduledJob): NotifierTestResult {
     return installSystemdJob(job);
   }

package/src/platform/t3-policy.ts

@@ -1,6 +1,8 @@
 import { runAnsible } from './ansible';
 import { desired } from '../t3/desired';
 import policy from '../../ansible/ubuntu/t3/policy.json';
+import ubuntu from '../../ansible/ubuntu/policy.json';
+import { restartDeferral } from './t3-service';
 import { basename, dirname, join, resolve } from 'node:path';
 import { rename, rm, symlink } from 'node:fs/promises';
 import { userInfo } from 'node:os';
@@ -34,6 +36,7 @@ export async function stageT3Policy(
       t3_policy_document: document,
       development_user: user.username,
       development_uid: user.uid,
+      development_home: user.homedir,
     }),
     timeoutMs: 120_000,
   });
@@ -84,14 +87,21 @@ export const T3_RESTARTED_SUMMARY =
   'T3 was restarted to load the new managed policy; the running server confirms it.';
 
 /** `t3-policy.yml` restarts a running T3 that has not loaded the staged policy
- * (ending its running turns), so every node offers the same providers after
- * every sync. This measures the outcome; `restarted` says whether the app
- * restarted during this sync and so needs time to reconnect.
+ * so every node offers the same providers, but only while it runs no agent
+ * turn: a busy T3 keeps its turns and the play records the restart as
+ * pending, for personal-worker-recovery to apply at the first idle check.
+ * This measures the outcome; `restarted` says whether the app restarted
+ * during this sync and so needs time to reconnect.
  */
 export async function loadedT3Policy(
   executable: string,
   run: Runner,
-  options: { restarted: boolean; pause?: () => Promise<void> }
+  options: {
+    restarted: boolean;
+    pause?: () => Promise<void>;
+    uid?: number;
+    now?: number;
+  }
 ): Promise<Check> {
   // A restarted desktop app takes a few seconds to come back with its server.
   const check = await verifyT3Policy(
@@ -103,8 +113,25 @@ export async function loadedT3Policy(
           setTimeout(done, options.restarted ? 5000 : 1000)
         ))
   );
-  if (check.status !== 'pass')
-    return { ...check, summary: 'T3_POLICY_RESTART_FAILED' };
+  if (check.status !== 'pass') {
+    const deferral = await restartDeferral(
+      run,
+      options.uid ?? userInfo().uid,
+      options.now
+    );
+    if (!deferral.pending)
+      return { ...check, summary: 'T3_POLICY_RESTART_FAILED' };
+    return {
+      ...check,
+      status: deferral.overdue ? 'fail' : 'warn',
+      summary: 'T3_POLICY_RESTART_PENDING',
+      remedy: {
+        action: deferral.overdue
+          ? `Agent turns kept T3 from loading the staged policy for ${ubuntu.t3RestartMaxDeferralHours} hours. Finish or stop them, or restart t3code-desktop.service on the worker (this ends its running turns).`
+          : 'Agent turns are running, so T3 keeps them and loads the staged policy at the first check of personal-worker-recovery with no turn running.',
+      },
+    };
+  }
   return options.restarted
     ? { ...check, summary: T3_RESTARTED_SUMMARY }
     : check;