@crvouga/dev-fleet 0.2.0 → 0.4.0

package/src/agents/account-relay.ts

@@ -1,19 +1,38 @@
+import {
+  AccountSelector,
+  accountRefused,
+  retryAfter,
+} from '../accounts/selection';
+
 /** Pins one agent process to one provider account. The proxy routes a model
  * named `<prefix>/<model>` only to the credential that owns the prefix, but
  * agents send whichever model the user picked. This loopback relay lives only
  * as long as the agent it was started for: it adds the prefix to each request
  * and forwards everything else (credentials, streaming) unchanged.
+ *
+ * With an empty prefix it pins nothing: every launch goes through a relay, so
+ * every launch rides out a proxy restart the same way. `selector` then picks
+ * the account of each request by the fleet's selection policy.
  */
 export function startAccountRelay(
   endpoint: string,
-  prefix: string
+  prefix: string,
+  options: Pick<RelayOptions, 'selector' | 'alternatives'> = {}
 ): { url: string; stop: () => void } {
+  const pinned =
+    prefix && !options.selector
+      ? new AccountSelector([prefix], { strategy: 'fill-first' })
+      : options.selector;
   const server = Bun.serve({
     hostname: '127.0.0.1',
     port: 0,
     // Model streams are idle for as long as a model thinks.
     idleTimeout: 0,
-    fetch: (request) => forward(request, endpoint, prefix),
+    fetch: (request) =>
+      forward(request, endpoint, prefix, {
+        ...options,
+        ...(pinned ? { selector: pinned } : {}),
+      }),
   });
   return {
     url: `http://127.0.0.1:${server.port}`,
@@ -24,6 +43,11 @@ export function startAccountRelay(
 type RelayOptions = {
   send?: typeof fetch;
   pause?: (seconds: number) => Promise<void>;
+  /** Unpinned: the accounts to try in turn. Pinned: remembers cooldowns. */
+  selector?: AccountSelector;
+  /** Pinned: the other instances of the provider, named in its error. */
+  alternatives?: string[];
+  now?: () => number;
 };
 const sleep = (seconds: number) =>
   new Promise<void>((done) => setTimeout(done, seconds * 1000));
@@ -32,33 +56,54 @@ export async function forward(
   request: Request,
   endpoint: string,
   prefix: string,
-  { send = fetch, pause = sleep }: RelayOptions = {}
+  options: RelayOptions = {}
 ): Promise<Response> {
   const url = new URL(request.url);
   const headers = new Headers(request.headers);
   // fetch sets these for the upstream request and decodes its response.
   for (const name of ['host', 'content-length', 'accept-encoding'])
     headers.delete(name);
-  let body: string | undefined;
-  if (request.body) {
-    // An unreadable body could not be pinned; never forward it to the pool.
-    if (headers.has('content-encoding'))
-      return new Response('The account relay needs an uncompressed request.', {
-        status: 415,
-      });
-    body = pinModel(await request.text(), prefix);
-  }
-  const attempt = () =>
-    send(endpoint + url.pathname + url.search, {
-      method: request.method,
-      headers,
-      ...(body === undefined ? {} : { body }),
-      redirect: 'error',
-      signal: request.signal,
+  // An unreadable body could not be pinned; never forward it to the pool.
+  if (request.body && headers.has('content-encoding'))
+    return new Response('The account relay needs an uncompressed request.', {
+      status: 415,
     });
-  const upstream = await reachProxy(attempt, request.signal, pause);
-  if (!upstream)
-    return new Response('The CLI proxy is unreachable.', { status: 502 });
+  const raw = request.body ? await request.text() : undefined;
+  const reach = reacher(
+    request,
+    headers,
+    endpoint + url.pathname + url.search,
+    options
+  );
+  const { selector } = options;
+  const reached =
+    !prefix && selector && raw !== undefined
+      ? await failOver(selector, raw, reach)
+      : await pinnedReach(prefix, raw, reach, options);
+  if (reached instanceof Response) return reached;
+  return answer(reached);
+}
+
+/** Sends one body to the proxy, retrying while the route is down. */
+function reacher(
+  request: Request,
+  headers: Headers,
+  target: string,
+  { send = fetch, pause = sleep, now = Date.now }: RelayOptions
+): (body: string | undefined) => Promise<Reached> {
+  return (body) =>
+    reachProxy(
+      () => send(target, upstreamInit(request, headers, body)),
+      request.signal,
+      pause,
+      now
+    );
+}
+
+/** The proxy's answer, decoded, or the distinct "unreachable" error. */
+function answer(reached: Reached): Response {
+  const upstream = reached.response;
+  if (!upstream) return unreachable(reached.since);
   const answered = new Headers(upstream.headers);
   answered.delete('content-encoding');
   answered.delete('content-length');
@@ -68,6 +113,109 @@ export async function forward(
   });
 }
 
+/** A pinned account (or no pinning at all). A pinned account the provider
+ * limited answers at once, until its cooldown clears, with the time and the
+ * other instances of its provider.
+ */
+async function pinnedReach(
+  prefix: string,
+  raw: string | undefined,
+  reach: (body: string | undefined) => Promise<Reached>,
+  { selector, alternatives = [] }: RelayOptions
+): Promise<Reached | Response> {
+  const cooling = prefix && selector ? selector.coolingUntil(prefix) : null;
+  if (cooling !== null) return coolingDown(cooling, alternatives);
+  const reached = await reach(
+    raw !== undefined && prefix ? pinModel(raw, prefix) : raw
+  );
+  if (!prefix || !selector || reached.response?.status !== 429) return reached;
+  selector.limited(prefix, retryAfter(reached.response.headers));
+  await reached.response.body?.cancel().catch(() => undefined);
+  return coolingDown(selector.coolingUntil(prefix)!, alternatives);
+}
+
+type Reached = { response: Response | null; since: number };
+
+function upstreamInit(
+  request: Request,
+  headers: Headers,
+  body: string | undefined
+): RequestInit {
+  return {
+    method: request.method,
+    headers,
+    ...(body === undefined ? {} : { body }),
+    redirect: 'error',
+    signal: request.signal,
+  };
+}
+
+/** An unpinned request goes to the first usable account by the fleet's
+ * policy; a quota or authentication refusal sends it on to the next. With
+ * every account cooling down, the proxy's own pool serves it.
+ */
+async function failOver(
+  selector: AccountSelector,
+  raw: string,
+  reach: (body: string | undefined) => Promise<Reached>
+): Promise<Reached> {
+  const candidates = selector.candidates();
+  if (candidates.length === 0) return reach(raw);
+  let reached!: Reached;
+  for (const [position, candidate] of candidates.entries()) {
+    reached = await reach(pinModel(raw, candidate));
+    const status = reached.response?.status;
+    if (status === undefined || !accountRefused(status)) return reached;
+    selector.limited(candidate, retryAfter(reached.response!.headers));
+    if (position < candidates.length - 1)
+      await reached.response!.body?.cancel().catch(() => undefined);
+  }
+  return reached;
+}
+
+/** One error both Anthropic and OpenAI clients show as a message. */
+function agentError(
+  status: number,
+  type: string,
+  message: string,
+  retry: number
+) {
+  return new Response(
+    JSON.stringify({ type: 'error', error: { type, message } }),
+    {
+      status,
+      headers: {
+        'content-type': 'application/json',
+        'retry-after': String(Math.max(1, Math.ceil(retry))),
+      },
+    }
+  );
+}
+
+function coolingDown(until: number, alternatives: string[]): Response {
+  const others = alternatives.length
+    ? ` Pick another instance of this provider in T3: ${alternatives.join(', ')}.`
+    : '';
+  return agentError(
+    429,
+    'rate_limit_error',
+    `This account is cooling down until ${new Date(until).toISOString()}.${others}`,
+    (until - Date.now()) / 1000
+  );
+}
+
+/** The proxy did not answer within the retry window: a distinct, retryable
+ * error that says since when, never a provider's error.
+ */
+function unreachable(since: number): Response {
+  return agentError(
+    503,
+    'proxy_unreachable',
+    `The CLI proxy has been unreachable since ${new Date(since).toISOString()}. This is retryable; nothing is wrong with the account.`,
+    30
+  );
+}
+
 /** Seconds to wait before each further attempt (about a minute in total). */
 export const RETRY_DELAYS = [1, 2, 4, 8, 15, 30];
 /** The route to the proxy answered instead of the proxy: its connector or the
@@ -83,13 +231,15 @@ const routeDown = (status: number) =>
 async function reachProxy(
   attempt: () => Promise<Response>,
   signal: AbortSignal,
-  pause: (seconds: number) => Promise<void>
-): Promise<Response | null> {
+  pause: (seconds: number) => Promise<void>,
+  now: () => number
+): Promise<Reached> {
+  const since = now();
   for (let tries = 0; ; tries++) {
     const delay = RETRY_DELAYS[tries];
     const response = await attempt().catch(() => null);
-    if (response && !routeDown(response.status)) return response;
-    if (delay === undefined || signal.aborted) return response;
+    if (response && !routeDown(response.status)) return { response, since };
+    if (delay === undefined || signal.aborted) return { response, since };
     await response?.body?.cancel().catch(() => undefined);
     await pause(delay);
   }

package/src/agents/launch.ts

@@ -6,6 +6,9 @@ import { accountPrefix, accountSlug, discoverAccounts } from './accounts';
 import { startAccountRelay } from './account-relay';
 import { proxyKey } from './secrets';
 import { pinnedBinary, pinnedPath } from './toolchain';
+import { AccountSelector } from '../accounts/selection';
+import { fleet } from '../fleet-config';
+import type { AccountSelection } from '../definition';
 
 /** `account` (its email) pins a proxy launch to one signed-in proxy account. */
 export async function launchProfile(
@@ -43,9 +46,11 @@ export async function launchProfile(
   }
   toolEnvironment(env, selection, directory);
   const pinned =
-    account && selection.profile === 'proxy'
-      ? await pinAccount(env, selection, { account, directory })
-      : null;
+    selection.profile !== 'proxy'
+      ? null
+      : account
+        ? await pinAccount(env, selection, { account, directory })
+        : await routeThroughRelay(env, selection);
   try {
     return await spawnAgent(selection, directory, env, [
       pinned?.args ?? [],
@@ -96,9 +101,14 @@ export async function pinAccount(
 ): Promise<{ args: string[]; stop: () => void }> {
   const prefix = accountPrefix(selection.tool, account);
   // What the account serves right now, preferred default first.
-  const served = (await discover(env['CLIPROXY_API_KEY'] ?? '')).find(
+  const roster = await discover(env['CLIPROXY_API_KEY'] ?? '');
+  const served = roster.find(
     (entry) => entry.tool === selection.tool && entry.email === account
   );
+  // Named in this account's "cooling down" error, so the person can switch.
+  const alternatives = roster
+    .filter((entry) => entry.tool === selection.tool && entry.email !== account)
+    .map((entry) => entry.email);
   if (!served)
     throw new AgentConfigurationError(
       'This account is not signed in on the CLI proxy.'
@@ -108,7 +118,9 @@ export async function pinAccount(
     env['OPENCODE_CONFIG_CONTENT'] = JSON.stringify(accountOpenCode(served));
     return { args: [], stop: () => {} };
   }
-  const { url, stop } = relay(registry.proxy.endpoint, prefix);
+  const { url, stop } = relay(registry.proxy.endpoint, prefix, {
+    alternatives,
+  });
   if (selection.tool === 'claude') {
     env['ANTHROPIC_BASE_URL'] = url;
     // Aliases and the default must name models a Claude account serves.
@@ -146,6 +158,73 @@ export async function pinAccount(
   };
 }
 
+/** Which tools reach the proxy through an address this wrapper sets, and so
+ * can be sent through a relay. OpenCode and Grok read theirs from rendered
+ * configuration and still go to the proxy directly.
+ */
+const RELAYED = new Set(['claude', 'codex', 'gemini', 'antigravity']);
+
+/** An unpinned launch goes through a relay as well: it retries a proxy that
+ * is restarting exactly as a pinned one does, reports "proxy unreachable"
+ * when it stays down, and picks each request's account by the fleet's
+ * selection policy when it declares one for the provider.
+ */
+export async function routeThroughRelay(
+  env: NodeJS.ProcessEnv,
+  selection: Selection,
+  {
+    relay = startAccountRelay,
+    discover = discoverAccounts,
+    policy = selectionPolicy,
+  } = {}
+): Promise<{ args: string[]; stop: () => void } | null> {
+  if (!RELAYED.has(selection.tool)) return null;
+  const selector = await policySelector(env, selection, discover, policy);
+  const { url, stop } = relay(
+    registry.proxy.endpoint,
+    '',
+    selector ? { selector } : {}
+  );
+  if (selection.tool === 'claude') env['ANTHROPIC_BASE_URL'] = url;
+  if (selection.tool === 'gemini' || selection.tool === 'antigravity')
+    env['GOOGLE_GEMINI_BASE_URL'] = url;
+  const args =
+    selection.tool === 'codex'
+      ? [
+          '-c',
+          `model_providers.personal_proxy.base_url=${JSON.stringify(`${url}/v1`)}`,
+        ]
+      : [];
+  return { args, stop };
+}
+
+/** The fleet's selection policy for the tool's provider, over the accounts
+ * the proxy lists. The list is read only when a policy needs it; an
+ * unreadable list leaves the choice to the proxy's own pool.
+ */
+async function policySelector(
+  env: NodeJS.ProcessEnv,
+  selection: Selection,
+  discover: typeof discoverAccounts,
+  policy: (provider: string) => AccountSelection | undefined
+): Promise<AccountSelector | undefined> {
+  const route = (
+    registry.proxy.accounts as Record<string, { provider: string }>
+  )[selection.tool];
+  const declared = route ? policy(route.provider) : undefined;
+  if (!declared) return undefined;
+  const accounts = await discover(env['CLIPROXY_API_KEY'] ?? '').catch(
+    () => []
+  );
+  const prefixes = accounts
+    .filter((account) => account.tool === selection.tool)
+    .map((account) => account.prefix);
+  return prefixes.length ? new AccountSelector(prefixes, declared) : undefined;
+}
+
+const selectionPolicy = (provider: string): AccountSelection | undefined =>
+  fleet.services.proxy.accounts?.selection?.[provider];
+
 /** OpenCode configuration that offers only one account's models. */
 function accountOpenCode(route: { prefix: string; models: string[] }) {
   const models = route.models.map((model) => `${route.prefix}/${model}`);

package/src/agents/native-verify.ts

@@ -4,6 +4,13 @@ import { join } from 'node:path';
 import type { Check, Runner } from '../types';
 import { pinnedPath } from './toolchain';
 import { registry } from './registry';
+import type { Account } from './accounts';
+import { accountSlug } from './accounts';
+import {
+  accountSessionChecks,
+  sessionResult,
+  type Instance,
+} from '../accounts/sessions';
 import {
   cacheKey,
   cachedCheck,
@@ -98,6 +105,11 @@ export async function verifyNativeAgents(
     outOfQuota?: Set<string>;
     cache?: VerifyCache;
     t3Service?: boolean;
+    /** The proxy's accounts and what this node offers. Every cached result
+     * is keyed on the roster version and the client key's version, so any
+     * account change or key rotation re-verifies at once.
+     */
+    accounts?: AccountContext;
   } = {}
 ): Promise<Check[]> {
   const { outOfQuota = new Set<string>(), cache = {} } = options;
@@ -122,7 +134,9 @@ export async function verifyNativeAgents(
                   agent,
                   pins[agent.pin] ?? '',
                   registry,
-                  await sourceKey(launcherSources(workspace, agent))
+                  await sourceKey(launcherSources(workspace, agent)),
+                  options.accounts?.rosterVersion ?? '',
+                  options.accounts?.keyVersion ?? ''
                 ),
                 () =>
                   sessionCheck(agent, { node, workspace, cwd, launch }, run),
@@ -130,12 +144,124 @@ export async function verifyNativeAgents(
               ),
       ])
     );
-    return [...environment.checks, ...results.flat()];
+    const accounts =
+      options.accounts && launch.kind !== 'incomplete'
+        ? await accountSessionChecks(
+            options.accounts.live,
+            options.accounts.exposed,
+            (instance) =>
+              accountSession(instance, options.accounts!, {
+                node,
+                workspace,
+                cwd,
+                launch,
+                cache,
+                run,
+              })
+          )
+        : [];
+    return [...environment.checks, ...results.flat(), ...accounts];
   } finally {
     await rm(cwd, { recursive: true, force: true });
   }
 }
 
+export interface AccountContext {
+  live: Account[];
+  exposed: Instance[];
+  rosterVersion: string;
+  keyVersion: string;
+}
+
+/** How a per-account instance is launched: its tool's launcher, pinned to
+ * the account as T3 pins it.
+ */
+const ACCOUNT_LAUNCH: Record<
+  string,
+  { wrapper: string; launcher: string; args: string[] }
+> = {
+  claude: {
+    wrapper: 'claude-proxy.ts',
+    launcher: 'claude-proxy',
+    args: ['-p', PROMPT],
+  },
+  codex: {
+    wrapper: 'codex-proxy.ts',
+    launcher: 'codex-proxy.ts',
+    args: ['exec', '--skip-git-repo-check', PROMPT],
+  },
+  opencode: {
+    wrapper: 'opencode-proxy.ts',
+    launcher: 'opencode-proxy.ts',
+    args: ['run', PROMPT],
+  },
+};
+
+async function accountSession(
+  instance: Instance,
+  accounts: AccountContext,
+  context: {
+    node: string;
+    workspace: string;
+    cwd: string;
+    launch: Launch;
+    cache: VerifyCache;
+    run: Runner;
+  }
+): Promise<'ok' | 'cooling-down' | 'failed'> {
+  const spec = ACCOUNT_LAUNCH[instance.tool];
+  if (!spec) return 'failed';
+  const check = await cachedCheck(
+    `account-${context.node}-${instance.tool}-${accountSlug(instance.account)}`,
+    cacheKey(
+      context.node,
+      instance,
+      spec,
+      accounts.rosterVersion,
+      accounts.keyVersion
+    ),
+    async () => {
+      const agents = join(context.workspace, 'agents');
+      const service = context.launch.kind === 'service';
+      const result = await context.run(
+        service
+          ? [join(agents, spec.launcher), ...spec.args]
+          : ['bun', join(agents, spec.wrapper), ...spec.args],
+        {
+          cwd: context.cwd,
+          env: {
+            ...(context.launch.kind === 'service'
+              ? context.launch.env
+              : { PATH: pinnedPath() }),
+            WS_AGENT_NODE: context.node,
+            WS_AGENT_ACCOUNT: instance.account,
+          },
+          ...(service ? { isolatedEnv: true } : {}),
+          input: '',
+          timeoutMs: 150_000,
+        }
+      );
+      const outcome = sessionResult(result.ok, result.stdout + result.stderr);
+      return {
+        id: 'account-session',
+        status:
+          outcome === 'ok'
+            ? 'pass'
+            : outcome === 'cooling-down'
+              ? 'warn'
+              : 'fail',
+        summary: outcome,
+      };
+    },
+    context.cache
+  );
+  return check.status === 'pass'
+    ? 'ok'
+    : check.status === 'warn'
+      ? 'cooling-down'
+      : 'failed';
+}
+
 /** Reads `tool = "x"` and `"tool" = { version = "x", ... }` pins. */
 export function parsePins(source: string): Record<string, string> {
   const pins: Record<string, string> = {};

package/src/auth-checks.ts

@@ -5,6 +5,7 @@ import { record } from './config';
 import { codingAuth, serviceAuth } from './service-auth';
 import { workstationChecks } from './workstation-checks';
 import { toolchainChecks } from './toolchain-checks';
+import { workerSelection } from '../bootstrap/worker-tools';
 import type { Capability, Check, Runner, CommandResult } from './types';
 
 export async function authCheck(
@@ -24,6 +25,8 @@ export async function authCheck(
       root,
       run,
       definition.workers.some((node) => node.id === nodeId)
+        ? workerSelection(definition, nodeId)
+        : undefined
     );
   return basicAuth(capability, run, root);
 }

package/src/cli/agent-guide.ts

@@ -63,7 +63,7 @@ export function agentGuide(release: Release) {
       'devfleet --interactive (explicit opt-in; TTY required, disabled in CI)',
     groups: {
       'fleet-*':
-        'Developer computers: fleet-status, fleet-discover, fleet-sync, fleet-state, fleet-node-sync, fleet-repair, fleet-nodes, fleet-node-add',
+        'Developer computers: fleet-status, fleet-discover, fleet-sync, fleet-state, fleet-report, fleet-node-sync, fleet-repair, fleet-nodes, fleet-node-add',
       't3-*': 'Declared T3 provider instances: t3-status, t3-apply, t3-cleanup',
       cliproxy: 'The CLI proxy service on the manager; arguments are forwarded',
     },
@@ -89,9 +89,11 @@ export function agentGuide(release: Release) {
         '130': 'Cancelled',
       },
       errors:
-        '{version:1,ok:false,code,error}; raw subprocess output and exception details are withheld',
+        'schema/output/error.schema.json: {version:1,ok:false,code,error}; raw subprocess output and exception details are withheld',
       reports:
-        '{version:1,command,ok,generatedAt,nodes:[{node,checks:[{id,status,summary,remedy?}]}],summary,coverage}',
+        'Each output has a JSON schema in this package; parse by it and by its version. Diagnostic reports (fleet-status, fleet-state, fleet-repair): schema/output/report.schema.json, version 1. fleet-sync: schema/output/fleet-sync.schema.json, version 3 (a --dry-run adds plan). fleet-report: schema/output/fleet-report.schema.json, version 1.',
+      checks:
+        'schema/check-ids.json lists every check ID with what a passing check means; * stands for a name. Each failing check carries its own remedy.',
       statuses: ['pass', 'warn', 'fail', 'skip'],
       coverage:
         'Default fleet-status tests the manager and all configured nodes. --local tests only this manager; --tailnet and --workers cover Tailscale computers. A missing roster fails explicitly. T3/VS Code end-to-end checks remain manual.',