@crvouga/dev-fleet 0.2.0 → 0.3.0

package/schema/fleet.schema.json

@@ -126,6 +126,41 @@
               "type": "string",
               "pattern": "^https://[A-Za-z0-9.-]+$",
               "description": "Public HTTPS origin of the CLI proxy every node uses."
+            },
+            "accounts": {
+              "type": "object",
+              "additionalProperties": false,
+              "description": "How provider accounts signed in on the proxy are watched and chosen. See docs/accounts.md.",
+              "properties": {
+                "minimumUsable": {
+                  "type": "integer",
+                  "minimum": 1,
+                  "description": "proxy:account:<provider> warns when a provider has fewer usable accounts than this (default 2) and fails when it has none."
+                },
+                "selection": {
+                  "type": "object",
+                  "description": "Per provider (claude, codex, antigravity): which account an unpinned request goes to first. Accounts cooling down are always skipped.",
+                  "additionalProperties": {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "required": ["strategy"],
+                    "properties": {
+                      "strategy": {
+                        "enum": [
+                          "fill-first",
+                          "round-robin",
+                          "least-recently-limited"
+                        ]
+                      },
+                      "order": {
+                        "type": "array",
+                        "items": { "type": "string" },
+                        "description": "Account labels (sign-in emails) first, in this order; the others follow in the order of their routing prefix."
+                      }
+                    }
+                  }
+                }
+              }
             }
           }
         },
@@ -244,6 +279,37 @@
         }
       }
     },
+    "ides": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": { "enum": ["t3"] },
+      "description": "The IDEs every machine runs. Defaults to [\"t3\"]; the README's support matrix lists the supported ones."
+    },
+    "t3": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "What every machine's T3 Code holds beyond the provider instances generated from the proxy's accounts.",
+      "properties": {
+        "profile": {
+          "type": "object",
+          "description": "Top-level T3 settings every machine holds the same (tool servers, projects, selected preferences). Keys not named here stay local; a key edited on a machine since dev-fleet last wrote it is reported as a conflict, never overwritten.",
+          "propertyNames": {
+            "pattern": "^[a-zA-Z][a-zA-Z0-9_-]{0,63}$",
+            "not": { "const": "providerInstances" }
+          }
+        },
+        "requiredModels": {
+          "type": "object",
+          "description": "Models T3 must offer, and the proxy must serve, for every account of a tool, by tool (claude, codex, opencode).",
+          "propertyNames": { "pattern": "^[a-zA-Z][a-zA-Z0-9_-]{0,63}$" },
+          "additionalProperties": {
+            "type": "array",
+            "items": { "type": "string", "pattern": "^[A-Za-z0-9._:-]{1,128}$" }
+          }
+        }
+      }
+    },
     "state": {
       "description": "The bucket that holds the fleet state (the roster, the approved revision and the configuration last applied). Any S3-compatible service, or a directory. Never credentials: a devfleet command reads those from DEVFLEET_BUCKET_ACCESS_KEY_ID and DEVFLEET_BUCKET_SECRET_ACCESS_KEY.",
       "oneOf": [

package/src/accounts/checks.ts

@@ -0,0 +1,220 @@
+import { existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import type { Check } from '../types';
+import type { ProxyRoster, RosterAccount } from './roster';
+
+/** Fewer usable accounts than this warns unless the fleet says otherwise. */
+export const MINIMUM_USABLE = 2;
+
+/** `proxy:account:<provider>`: fails when a provider has no usable account,
+ * warns below the fleet's minimum. Accounts are named by their opaque index
+ * and counts only, never by an identifier.
+ */
+export function providerAccountChecks(
+  roster: ProxyRoster,
+  minimumUsable = MINIMUM_USABLE
+): Check[] {
+  const providers = new Map<string, RosterAccount[]>();
+  for (const account of roster.accounts)
+    if (account.state !== 'removed')
+      providers.set(account.provider, [
+        ...(providers.get(account.provider) ?? []),
+        account,
+      ]);
+  return [...providers.entries()]
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([provider, accounts]) =>
+      providerCheck(provider, accounts, minimumUsable)
+    );
+}
+
+/** "account 2 needs-login, account 3 cooling-down until …" */
+const describe = (accounts: RosterAccount[]): string[] =>
+  accounts
+    .filter((account) => account.state !== 'usable')
+    .map(
+      (account) =>
+        `account ${account.index} ${account.state}${account.until ? ` until ${account.until}` : ''}`
+    );
+
+function reloginRemedy(provider: string, accounts: RosterAccount[]) {
+  const login = accounts.find((account) => account.state === 'needs-login');
+  return login
+    ? {
+        action: `Sign ${provider} account ${login.index} in again. Signing in is the only manual step.`,
+        command: `devfleet fleet-accounts --relogin ${provider} ${login.index}`,
+      }
+    : undefined;
+}
+
+function providerCheck(
+  provider: string,
+  accounts: RosterAccount[],
+  minimumUsable: number
+): Check {
+  const usable = accounts.filter((account) => account.state === 'usable');
+  const id = `proxy:account:${provider}`;
+  const detail = describe(accounts);
+  const counted = `${usable.length} of ${accounts.length} ${provider} accounts usable${detail.length ? ` (${detail.join(', ')})` : ''}`;
+  const remedy = reloginRemedy(provider, accounts);
+  if (usable.length === 0) return noneUsable(provider, accounts);
+  if (usable.length < minimumUsable)
+    return {
+      id,
+      status: 'warn',
+      summary: `${counted}; one limit or expired login away from none.`,
+      remedy: remedy ?? {
+        action: `Add a ${provider} account: devfleet fleet-accounts --add ${provider}.`,
+      },
+    };
+  return {
+    id,
+    status: 'pass',
+    summary: `${counted}.`,
+    ...(remedy ? { remedy } : {}),
+  };
+}
+
+/** No usable account: say why, and when the first cooldown clears. */
+function noneUsable(provider: string, accounts: RosterAccount[]): Check {
+  const clears = accounts
+    .map((account) => account.until)
+    .filter((until): until is string => until !== null)
+    .sort()[0];
+  const cooling = accounts.every((account) => account.state === 'cooling-down');
+  const wait = `Wait until ${clears ?? 'the provider lifts the limit'}, or add an account: devfleet fleet-accounts --add ${provider}.`;
+  return {
+    id: `proxy:account:${provider}`,
+    status: 'fail',
+    summary: cooling
+      ? `No usable ${provider} account: every one is cooling down${clears ? `; the first clears at ${clears}` : ''}.`
+      : `No usable ${provider} account (${describe(accounts).join(', ')}).`,
+    remedy: reloginRemedy(provider, accounts) ?? {
+      action: cooling
+        ? wait
+        : `Inspect the ${provider} accounts with devfleet fleet-accounts.`,
+    },
+  };
+}
+
+/** `accounts:roster-current`: this node offers exactly the accounts the
+ * proxy holds. `proxy` is the proxy's roster version (null: unreadable),
+ * `node` the version of the account instances the node exposes.
+ */
+export function rosterCurrentCheck(proxy: string | null, node: string): Check {
+  if (proxy === null)
+    return {
+      id: 'accounts:roster-current',
+      status: 'warn',
+      summary: `The proxy's account roster could not be read; this node offers roster version ${node}.`,
+      remedy: {
+        action: 'Check proxy:reachable-from, then run fleet-status again.',
+      },
+    };
+  return proxy === node
+    ? {
+        id: 'accounts:roster-current',
+        status: 'pass',
+        summary: `This node offers the proxy's account roster (version ${node}).`,
+      }
+    : {
+        id: 'accounts:roster-current',
+        status: 'fail',
+        summary: `This node offers account roster version ${node}; the proxy holds ${proxy}.`,
+        remedy: {
+          action: 'Propagate the accounts to every node without a full sync.',
+          command: 'devfleet fleet-sync --only accounts',
+        },
+      };
+}
+
+/** Where agent CLIs keep a provider login of their own, relative to home. */
+export const NATIVE_LOGINS: Record<string, string[]> = {
+  claude: ['.claude/.credentials.json'],
+  codex: ['.codex/auth.json'],
+  gemini: ['.gemini/oauth_creds.json'],
+  opencode: ['.local/share/opencode/auth.json'],
+  grok: ['.grok/auth.json'],
+};
+
+/** The provider logins found on this node outside the proxy path. */
+export function nativeLogins(
+  home = homedir(),
+  exists: (path: string) => boolean = existsSync
+): { cli: string; path: string }[] {
+  return Object.entries(NATIVE_LOGINS).flatMap(([cli, paths]) =>
+    paths
+      .map((path) => join(home, path))
+      .filter((path) => exists(path))
+      .map((path) => ({ cli, path }))
+  );
+}
+
+/** `accounts:native-login`: a node holds no provider login of its own, so it
+ * can use no account the proxy does not know. Names the CLIs only.
+ */
+export function nativeLoginCheck(found: { cli: string }[]): Check {
+  const clis = [...new Set(found.map((login) => login.cli))].sort();
+  return clis.length === 0
+    ? {
+        id: 'accounts:native-login',
+        status: 'pass',
+        summary:
+          'No agent CLI on this node holds a provider login of its own; every account comes from the proxy.',
+      }
+    : {
+        id: 'accounts:native-login',
+        status: 'warn',
+        summary: `Provider logins outside the proxy path: ${clis.join(', ')}.`,
+        remedy: {
+          action:
+            'Provider accounts belong on the proxy. Remove these logins (opt-in), and sign the account in on the proxy if it is needed.',
+          command: 'devfleet fleet-accounts --remove-native-logins --yes',
+        },
+      };
+}
+
+/** `proxy:reachable-from`: this node reaches the proxy, and how fast. */
+export async function proxyReachableCheck(
+  endpoint: string,
+  send: typeof fetch = fetch,
+  clock: () => number = () => performance.now()
+): Promise<Check> {
+  const started = clock();
+  try {
+    // Unauthenticated: the proxy refuses it, which proves it answered.
+    const response = await send(endpoint + '/v1/models', {
+      redirect: 'error',
+      signal: AbortSignal.timeout(10_000),
+    });
+    await response.body?.cancel().catch(() => undefined);
+    const latency = Math.round(clock() - started);
+    const answered = response.status < 500;
+    return {
+      id: 'proxy:reachable-from',
+      status: !answered ? 'fail' : latency > 2000 ? 'warn' : 'pass',
+      summary: answered
+        ? `The proxy answered in ${latency} ms.`
+        : 'PROXY_UNREACHABLE: the route to the proxy answered, the proxy did not.',
+      ...(answered
+        ? {}
+        : {
+            remedy: {
+              action:
+                'Agents retry for about a minute and then report "proxy unreachable". Check the proxy on the manager: devfleet cliproxy status.',
+            },
+          }),
+    };
+  } catch {
+    return {
+      id: 'proxy:reachable-from',
+      status: 'fail',
+      summary: 'PROXY_UNREACHABLE: no answer from the proxy.',
+      remedy: {
+        action:
+          'Agents retry for about a minute and then report "proxy unreachable". Check this node’s tailnet and the proxy on the manager: devfleet cliproxy status.',
+      },
+    };
+  }
+}

package/src/accounts/context.ts

@@ -0,0 +1,76 @@
+import { createHash } from 'node:crypto';
+import { homedir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { accountPrefix, discoverAccounts } from '../agents/accounts';
+import { proxyKey } from '../agents/secrets';
+import { registry } from '../agents/registry';
+import type { AccountContext } from '../agents/native-verify';
+import type { Check } from '../types';
+import {
+  nativeLoginCheck,
+  nativeLogins,
+  proxyReachableCheck,
+  rosterCurrentCheck,
+} from './checks';
+import { liveVersion, rosterVersion } from './roster';
+import { exposedInstances, type Instance } from './sessions';
+
+/** This machine's T3 settings, where its account instances are written. */
+export const t3Settings = (): string =>
+  join(
+    resolve(process.env['T3CODE_HOME'] ?? join(homedir(), '.t3')),
+    'userdata/settings.json'
+  );
+
+/** A short digest of the client key: its version, never its value. */
+export const keyVersion = (key: string): string =>
+  createHash('sha256').update(key).digest('hex').slice(0, 12);
+
+/** The roster version of the instances a node offers. */
+export function exposedVersion(instances: Instance[]): string {
+  return rosterVersion(
+    instances.flatMap((instance) => {
+      try {
+        return [accountPrefix(instance.tool, instance.account)];
+      } catch {
+        return [];
+      }
+    })
+  );
+}
+
+/** The proxy's accounts and what this node offers, read with this node's
+ * own key; undefined when the proxy or the key is unavailable (the provider
+ * checks report that).
+ */
+export async function accountContext(
+  settings = t3Settings()
+): Promise<AccountContext | undefined> {
+  try {
+    const key = await proxyKey();
+    const live = await discoverAccounts(key);
+    return {
+      live,
+      exposed: await exposedInstances(settings),
+      rosterVersion: liveVersion(live),
+      keyVersion: keyVersion(key),
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+/** What a node measures about accounts: it reaches the proxy, it offers
+ * the proxy's roster, and it holds no provider login of its own.
+ */
+export async function nodeAccountChecks(
+  context: AccountContext | undefined,
+  settings = t3Settings()
+): Promise<Check[]> {
+  const exposed = context?.exposed ?? (await exposedInstances(settings));
+  return [
+    await proxyReachableCheck(registry.proxy.endpoint),
+    rosterCurrentCheck(context?.rosterVersion ?? null, exposedVersion(exposed)),
+    nativeLoginCheck(nativeLogins()),
+  ];
+}

package/src/accounts/propagate.ts

@@ -0,0 +1,62 @@
+import { blocking, type Check, type FleetNode, type Runner } from '../types';
+import { inspectRemote } from '../transport';
+import type { CommandOutcome } from '../cli/contracts';
+
+export interface Propagation {
+  /** The enrolled workers. */
+  nodes: FleetNode[];
+  /** The approved revision every worker runs. */
+  revision: string;
+  /** The bundled remote entry sent to each worker. */
+  source: string;
+  /** The manager's own account instances, applied locally. */
+  manager: () => Promise<Check[]>;
+  run: Runner;
+}
+
+/** `fleet-sync --only accounts`: the account-derived configuration and
+ * nothing else, on the manager and on every worker at once. Each node
+ * re-reads the proxy's roster and rewrites only its per-account T3
+ * instances; T3 watches its settings, so nothing restarts and no running
+ * turn ends. Instance ids derive from the account's routing prefix, so they
+ * stay the same and existing threads keep working. A node that cannot be
+ * reached is reported; it catches up on the next propagation or sync, and
+ * reports the roster version it has (`accounts:roster-current`) until then.
+ */
+export async function propagateAccounts(
+  input: Propagation
+): Promise<CommandOutcome> {
+  const started = Date.now();
+  const [manager, ...workers] = await Promise.all([
+    input.manager().then((checks) => ({ node: 'manager', checks })),
+    ...input.nodes.map((node) =>
+      inspectRemote(
+        node,
+        {
+          source: input.source,
+          timeoutMs: 180_000,
+          action: 'accounts',
+          expected: { head: input.revision, files: {} },
+        },
+        input.run
+      )
+    ),
+  ]);
+  const nodes = [manager!, ...workers].map((report) => ({
+    node: report.node,
+    checks: report.checks,
+  }));
+  const ok = nodes.every((node) => !node.checks.some(blocking));
+  return {
+    exitCode: ok ? 0 : 1,
+    data: {
+      version: 1,
+      command: 'fleet-sync',
+      only: 'accounts',
+      ok,
+      revision: input.revision,
+      durationMs: Date.now() - started,
+      nodes,
+    },
+  };
+}

package/src/accounts/roster.ts

@@ -0,0 +1,85 @@
+import { createHash } from 'node:crypto';
+import type { Account } from '../agents/accounts';
+
+/** Provider accounts exist only as the CLI proxy's credential files. Every
+ * node and IDE gets a view derived from them; nothing on a node holds a
+ * provider login. See docs/accounts.md.
+ */
+export type AccountState =
+  'usable' | 'cooling-down' | 'needs-login' | 'refresh-failing' | 'removed';
+
+const STATES: AccountState[] = [
+  'usable',
+  'cooling-down',
+  'needs-login',
+  'refresh-failing',
+  'removed',
+];
+
+/** One account as the manager reports it. `index` is opaque: the position of
+ * the account among its provider's, the only way checks name it.
+ */
+export interface RosterAccount {
+  provider: string;
+  index: number;
+  label: string;
+  /** The routing prefix every node offers it under; null if not routed. */
+  prefix: string | null;
+  state: AccountState;
+  /** While cooling down: when it clears (ISO 8601), when known. */
+  until: string | null;
+}
+
+/** What `devfleet cliproxy accounts --json` prints on the manager. */
+export interface ProxyRoster {
+  version: 1;
+  rosterVersion: string;
+  /** Whether the proxy's management API answered: without it, cooldowns and
+   * refresh failures are not known.
+   */
+  managementApi: boolean;
+  accounts: RosterAccount[];
+}
+
+export function parseRoster(text: string): ProxyRoster {
+  const value = JSON.parse(text) as Partial<ProxyRoster>;
+  if (
+    value.version !== 1 ||
+    typeof value.rosterVersion !== 'string' ||
+    !Array.isArray(value.accounts) ||
+    !value.accounts.every(
+      (account) =>
+        typeof account.provider === 'string' &&
+        Number.isInteger(account.index) &&
+        typeof account.label === 'string' &&
+        STATES.includes(account.state)
+    )
+  )
+    throw new Error('The proxy reported an unreadable account roster.');
+  return {
+    version: 1,
+    rosterVersion: value.rosterVersion,
+    managementApi: value.managementApi === true,
+    accounts: value.accounts.map((account) => ({
+      ...account,
+      prefix: account.prefix ?? null,
+      until: account.until ?? null,
+    })),
+  };
+}
+
+/** The roster version: a digest of the routing prefixes of the accounts
+ * every node offers. The manager computes it from the credential files
+ * (cliproxy/scripts/accounts.py) and every node from the proxy's model list;
+ * the two agree exactly when the node offers what the proxy holds.
+ */
+export function rosterVersion(prefixes: Iterable<string>): string {
+  return createHash('sha256')
+    .update([...new Set(prefixes)].sort().join('\n'))
+    .digest('hex')
+    .slice(0, 16);
+}
+
+/** The version of the accounts a node reads from the proxy's model list. */
+export const liveVersion = (accounts: Account[]): string =>
+  rosterVersion(accounts.map((account) => account.prefix));

package/src/accounts/selection.ts

@@ -0,0 +1,67 @@
+import type { AccountSelection } from '../definition';
+
+/** Which account an unpinned request goes to first, per the policy the fleet
+ * declares for the provider. The order is deterministic: the accounts the
+ * policy names, in its order, then the others in the order of their routing
+ * prefix. Accounts cooling down are skipped while any other is left.
+ */
+export class AccountSelector {
+  private readonly cooling = new Map<string, number>();
+  private readonly limitedAt = new Map<string, number>();
+  private turn = 0;
+  readonly order: string[];
+
+  constructor(
+    prefixes: string[],
+    private readonly policy: AccountSelection,
+    private readonly now: () => number = Date.now
+  ) {
+    const label = (prefix: string) => prefix.slice(prefix.indexOf('-') + 1);
+    const named = policy.order ?? [];
+    const rank = (prefix: string) => {
+      const position = named.indexOf(label(prefix));
+      return position < 0 ? named.length : position;
+    };
+    this.order = [...new Set(prefixes)].sort(
+      (a, b) => rank(a) - rank(b) || a.localeCompare(b)
+    );
+  }
+
+  /** The accounts to try for the next request, in order. */
+  candidates(): string[] {
+    const now = this.now();
+    let order = [...this.order];
+    if (this.policy.strategy === 'round-robin' && order.length) {
+      const start = this.turn++ % order.length;
+      order = [...order.slice(start), ...order.slice(0, start)];
+    }
+    if (this.policy.strategy === 'least-recently-limited')
+      order.sort(
+        (a, b) => (this.limitedAt.get(a) ?? 0) - (this.limitedAt.get(b) ?? 0)
+      );
+    return order.filter((prefix) => (this.cooling.get(prefix) ?? 0) <= now);
+  }
+
+  /** The provider limited this account; `seconds` from its Retry-After. */
+  limited(prefix: string, seconds = 60): void {
+    const now = this.now();
+    this.limitedAt.set(prefix, now);
+    this.cooling.set(prefix, now + seconds * 1000);
+  }
+
+  /** When this account's cooldown clears, if it is cooling down. */
+  coolingUntil(prefix: string): number | null {
+    const until = this.cooling.get(prefix) ?? 0;
+    return until > this.now() ? until : null;
+  }
+}
+
+/** A quota or authentication answer: the next account may still serve. */
+export const accountRefused = (status: number): boolean =>
+  status === 429 || status === 401 || status === 403;
+
+/** Seconds a refused account cools down for: its Retry-After, or a minute. */
+export function retryAfter(headers: Headers): number {
+  const value = Number(headers.get('retry-after'));
+  return Number.isFinite(value) && value > 0 ? Math.min(value, 3600) : 60;
+}