@crvouga/dev-fleet 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/README.md +265 -37
  2. package/agents/setup.md +42 -12
  3. package/ansible/macos/manager.yml +10 -7
  4. package/ansible/macos/policy.json +2 -10
  5. package/ansible/macos/t3-policy.yml +49 -0
  6. package/ansible/monitoring/README.md +86 -25
  7. package/ansible/monitoring/agent.yml +40 -2
  8. package/ansible/monitoring/hub-route-darwin.yml +51 -0
  9. package/ansible/monitoring/lock.json +5 -0
  10. package/ansible/pyproject.toml +12 -0
  11. package/ansible/ubuntu/baseline.yml +1 -1
  12. package/ansible/ubuntu/disk-gc.py +336 -0
  13. package/ansible/ubuntu/policy.json +13 -3
  14. package/ansible/ubuntu/t3-desktop.yml +19 -0
  15. package/ansible/ubuntu/t3-health.sh +93 -22
  16. package/ansible/ubuntu/t3-idle.py +42 -0
  17. package/ansible/ubuntu/t3-policy.yml +4 -11
  18. package/ansible/ubuntu/tailscale.yml +8 -5
  19. package/ansible/ubuntu/tasks/recovery.yml +60 -1
  20. package/ansible/ubuntu/tasks/resources.yml +61 -1
  21. package/ansible/ubuntu/tasks/t3-restart.yml +37 -0
  22. package/ansible/ubuntu/wip-guard.py +121 -0
  23. package/ansible/uv.lock +190 -0
  24. package/bootstrap/bootstrap.sh +25 -18
  25. package/bootstrap/doctor.ts +46 -1
  26. package/bootstrap/fleet-manager-run.sh +42 -15
  27. package/bootstrap/fleet-remote-bootstrap.sh +85 -12
  28. package/bootstrap/global-config.ts +64 -6
  29. package/bootstrap/install-launcher.sh +3 -0
  30. package/bootstrap/native-t3.ts +22 -11
  31. package/bootstrap/npm.ts +10 -2
  32. package/bootstrap/release-version.ts +157 -0
  33. package/cliproxy/README.md +3 -3
  34. package/cliproxy/scripts/accounts.py +216 -0
  35. package/cliproxy/scripts/checkpoint.py +29 -1
  36. package/cliproxy/scripts/cli.py +1 -1
  37. package/cliproxy/scripts/clientkeys.py +99 -0
  38. package/cliproxy/scripts/lease.py +80 -0
  39. package/cliproxy/scripts/ops +137 -25
  40. package/cliproxy/scripts/restore-verify +15 -0
  41. package/docs/accounts.md +154 -0
  42. package/docs/consumer-setup.md +408 -0
  43. package/docs/declared.md +91 -0
  44. package/docs/durability.md +26 -0
  45. package/docs/machine-setup.md +400 -0
  46. package/docs/operations.md +276 -0
  47. package/docs/releasing.md +103 -0
  48. package/docs/state.md +254 -0
  49. package/docs/topology.md +69 -0
  50. package/docs/ubuntu-sync.md +357 -0
  51. package/docs/worker-contract.md +67 -0
  52. package/examples/fleet.json +5 -0
  53. package/llms.txt +36 -0
  54. package/mise.lock +29 -0
  55. package/mise.toml +15 -2
  56. package/package.json +11 -2
  57. package/schema/fleet.schema.json +137 -15
  58. package/src/accounts/checks.ts +220 -0
  59. package/src/accounts/context.ts +76 -0
  60. package/src/accounts/propagate.ts +62 -0
  61. package/src/accounts/roster.ts +85 -0
  62. package/src/accounts/selection.ts +67 -0
  63. package/src/accounts/sessions.ts +120 -0
  64. package/src/agents/account-relay.ts +176 -26
  65. package/src/agents/fleet-verify.ts +9 -0
  66. package/src/agents/launch.ts +84 -5
  67. package/src/agents/native-verify.ts +293 -22
  68. package/src/agents/provider-errors.ts +9 -2
  69. package/src/agents/registry.ts +7 -1
  70. package/src/cli/agent-guide.ts +39 -5
  71. package/src/cli/app.ts +11 -4
  72. package/src/cli/cli.ts +7 -0
  73. package/src/cli/command-index.ts +7 -0
  74. package/src/cli/fleet-accounts.ts +253 -0
  75. package/src/cli/fleet-state.ts +499 -0
  76. package/src/cli/fleet-sync-accounts.ts +58 -0
  77. package/src/cli/fleet-sync-checkout.ts +56 -17
  78. package/src/cli/fleet-sync-enrollment.ts +19 -23
  79. package/src/cli/fleet-sync-forward.ts +1 -1
  80. package/src/cli/fleet-sync-manager.ts +84 -12
  81. package/src/cli/fleet-sync-monitoring.ts +5 -5
  82. package/src/cli/fleet-sync-output.ts +7 -3
  83. package/src/cli/fleet-sync-remote.ts +31 -18
  84. package/src/cli/fleet-sync-state.ts +86 -0
  85. package/src/cli/fleet-sync.ts +159 -57
  86. package/src/cli/fleet.ts +4 -2
  87. package/src/cli/managed-health.ts +4 -4
  88. package/src/cli/node-sync.ts +7 -0
  89. package/src/cli/program.ts +20 -4
  90. package/src/cli/project.ts +51 -0
  91. package/src/cli/register-fleet.ts +86 -1
  92. package/src/cli/registry.ts +41 -1
  93. package/src/cli/roster.ts +46 -30
  94. package/src/cli/settings.ts +15 -12
  95. package/src/cli/source-operations.ts +27 -3
  96. package/src/cli/tailnet.ts +2 -2
  97. package/src/definition.ts +116 -4
  98. package/src/diagnostics.ts +1 -1
  99. package/src/fleet-config.ts +105 -19
  100. package/src/ide/definition.ts +86 -0
  101. package/src/ide/ide.ts +43 -0
  102. package/src/ide/index.ts +5 -0
  103. package/src/ide/orchestrate.ts +50 -0
  104. package/src/ide/t3.ts +111 -0
  105. package/src/machine/source.ts +3 -3
  106. package/src/machine/state.ts +15 -27
  107. package/src/manager-sync.ts +9 -18
  108. package/src/monitoring/agent-evidence.ts +146 -0
  109. package/src/monitoring/apply.ts +15 -3
  110. package/src/monitoring/beszel.ts +11 -4
  111. package/src/monitoring/enroll.ts +2 -1
  112. package/src/monitoring/hub-access.ts +4 -3
  113. package/src/monitoring/hub-backup.ts +121 -0
  114. package/src/monitoring/hub-viewer.ts +8 -8
  115. package/src/monitoring/hub.ts +3 -3
  116. package/src/monitoring/manager-hub.ts +30 -14
  117. package/src/node-converge-linux.ts +30 -7
  118. package/src/package-info.ts +45 -0
  119. package/src/platform/ansible.ts +19 -15
  120. package/src/platform/darwin.ts +0 -13
  121. package/src/platform/fallback.ts +0 -8
  122. package/src/platform/linux.ts +0 -10
  123. package/src/platform/resource-verify.ts +8 -4
  124. package/src/platform/t3-policy.ts +33 -6
  125. package/src/platform/t3-service.ts +104 -0
  126. package/src/platform/types.ts +0 -4
  127. package/src/platform/ubuntu-audit.ts +5 -2
  128. package/src/platform/windows.ts +0 -14
  129. package/src/remote-bootstrap.ts +17 -19
  130. package/src/remote-entry.ts +15 -1
  131. package/src/repair-catalog.ts +1 -1
  132. package/src/runner.ts +1 -1
  133. package/src/service-auth.ts +15 -5
  134. package/src/state/bucket.ts +208 -0
  135. package/src/state/check.ts +74 -0
  136. package/src/state/connection.ts +92 -0
  137. package/src/state/enrollment.ts +45 -0
  138. package/src/state/legacy.ts +89 -0
  139. package/src/state/litestream.ts +143 -0
  140. package/src/state/operator.ts +20 -0
  141. package/src/state/paths.ts +40 -0
  142. package/src/state/roster.ts +62 -0
  143. package/src/state/store.ts +169 -0
  144. package/src/state/sync.ts +352 -0
  145. package/src/t3/accounts.ts +7 -1
  146. package/src/t3/models.ts +133 -0
  147. package/src/t3/pairing.ts +31 -0
  148. package/src/t3/profile.ts +203 -0
  149. package/src/t3/version.ts +119 -0
  150. package/src/t3-endpoint.ts +64 -8
  151. package/src/t3-linked-check.ts +24 -2
  152. package/src/toolchain-checks.ts +25 -0
  153. package/src/transport.ts +3 -2
  154. package/src/types.ts +4 -0
  155. package/src/worker-access-checks.ts +56 -3
  156. package/src/worker-host-checks.ts +41 -13
  157. package/src/workstation-checks.ts +1 -1
  158. package/src/index.ts +0 -6
  159. package/src/test-fixtures.ts +0 -36
@@ -0,0 +1,85 @@
1
+ import { createHash } from 'node:crypto';
2
+ import type { Account } from '../agents/accounts';
3
+
4
+ /** Provider accounts exist only as the CLI proxy's credential files. Every
5
+ * node and IDE gets a view derived from them; nothing on a node holds a
6
+ * provider login. See docs/accounts.md.
7
+ */
8
+ export type AccountState =
9
+ 'usable' | 'cooling-down' | 'needs-login' | 'refresh-failing' | 'removed';
10
+
11
+ const STATES: AccountState[] = [
12
+ 'usable',
13
+ 'cooling-down',
14
+ 'needs-login',
15
+ 'refresh-failing',
16
+ 'removed',
17
+ ];
18
+
19
+ /** One account as the manager reports it. `index` is opaque: the position of
20
+ * the account among its provider's, the only way checks name it.
21
+ */
22
+ export interface RosterAccount {
23
+ provider: string;
24
+ index: number;
25
+ label: string;
26
+ /** The routing prefix every node offers it under; null if not routed. */
27
+ prefix: string | null;
28
+ state: AccountState;
29
+ /** While cooling down: when it clears (ISO 8601), when known. */
30
+ until: string | null;
31
+ }
32
+
33
+ /** What `devfleet cliproxy accounts --json` prints on the manager. */
34
+ export interface ProxyRoster {
35
+ version: 1;
36
+ rosterVersion: string;
37
+ /** Whether the proxy's management API answered: without it, cooldowns and
38
+ * refresh failures are not known.
39
+ */
40
+ managementApi: boolean;
41
+ accounts: RosterAccount[];
42
+ }
43
+
44
+ export function parseRoster(text: string): ProxyRoster {
45
+ const value = JSON.parse(text) as Partial<ProxyRoster>;
46
+ if (
47
+ value.version !== 1 ||
48
+ typeof value.rosterVersion !== 'string' ||
49
+ !Array.isArray(value.accounts) ||
50
+ !value.accounts.every(
51
+ (account) =>
52
+ typeof account.provider === 'string' &&
53
+ Number.isInteger(account.index) &&
54
+ typeof account.label === 'string' &&
55
+ STATES.includes(account.state)
56
+ )
57
+ )
58
+ throw new Error('The proxy reported an unreadable account roster.');
59
+ return {
60
+ version: 1,
61
+ rosterVersion: value.rosterVersion,
62
+ managementApi: value.managementApi === true,
63
+ accounts: value.accounts.map((account) => ({
64
+ ...account,
65
+ prefix: account.prefix ?? null,
66
+ until: account.until ?? null,
67
+ })),
68
+ };
69
+ }
70
+
71
+ /** The roster version: a digest of the routing prefixes of the accounts
72
+ * every node offers. The manager computes it from the credential files
73
+ * (cliproxy/scripts/accounts.py) and every node from the proxy's model list;
74
+ * the two agree exactly when the node offers what the proxy holds.
75
+ */
76
+ export function rosterVersion(prefixes: Iterable<string>): string {
77
+ return createHash('sha256')
78
+ .update([...new Set(prefixes)].sort().join('\n'))
79
+ .digest('hex')
80
+ .slice(0, 16);
81
+ }
82
+
83
+ /** The version of the accounts a node reads from the proxy's model list. */
84
+ export const liveVersion = (accounts: Account[]): string =>
85
+ rosterVersion(accounts.map((account) => account.prefix));
@@ -0,0 +1,67 @@
1
+ import type { AccountSelection } from '../definition';
2
+
3
+ /** Which account an unpinned request goes to first, per the policy the fleet
4
+ * declares for the provider. The order is deterministic: the accounts the
5
+ * policy names, in its order, then the others in the order of their routing
6
+ * prefix. Accounts cooling down are skipped while any other is left.
7
+ */
8
+ export class AccountSelector {
9
+ private readonly cooling = new Map<string, number>();
10
+ private readonly limitedAt = new Map<string, number>();
11
+ private turn = 0;
12
+ readonly order: string[];
13
+
14
+ constructor(
15
+ prefixes: string[],
16
+ private readonly policy: AccountSelection,
17
+ private readonly now: () => number = Date.now
18
+ ) {
19
+ const label = (prefix: string) => prefix.slice(prefix.indexOf('-') + 1);
20
+ const named = policy.order ?? [];
21
+ const rank = (prefix: string) => {
22
+ const position = named.indexOf(label(prefix));
23
+ return position < 0 ? named.length : position;
24
+ };
25
+ this.order = [...new Set(prefixes)].sort(
26
+ (a, b) => rank(a) - rank(b) || a.localeCompare(b)
27
+ );
28
+ }
29
+
30
+ /** The accounts to try for the next request, in order. */
31
+ candidates(): string[] {
32
+ const now = this.now();
33
+ let order = [...this.order];
34
+ if (this.policy.strategy === 'round-robin' && order.length) {
35
+ const start = this.turn++ % order.length;
36
+ order = [...order.slice(start), ...order.slice(0, start)];
37
+ }
38
+ if (this.policy.strategy === 'least-recently-limited')
39
+ order.sort(
40
+ (a, b) => (this.limitedAt.get(a) ?? 0) - (this.limitedAt.get(b) ?? 0)
41
+ );
42
+ return order.filter((prefix) => (this.cooling.get(prefix) ?? 0) <= now);
43
+ }
44
+
45
+ /** The provider limited this account; `seconds` from its Retry-After. */
46
+ limited(prefix: string, seconds = 60): void {
47
+ const now = this.now();
48
+ this.limitedAt.set(prefix, now);
49
+ this.cooling.set(prefix, now + seconds * 1000);
50
+ }
51
+
52
+ /** When this account's cooldown clears, if it is cooling down. */
53
+ coolingUntil(prefix: string): number | null {
54
+ const until = this.cooling.get(prefix) ?? 0;
55
+ return until > this.now() ? until : null;
56
+ }
57
+ }
58
+
59
+ /** A quota or authentication answer: the next account may still serve. */
60
+ export const accountRefused = (status: number): boolean =>
61
+ status === 429 || status === 401 || status === 403;
62
+
63
+ /** Seconds a refused account cools down for: its Retry-After, or a minute. */
64
+ export function retryAfter(headers: Headers): number {
65
+ const value = Number(headers.get('retry-after'));
66
+ return Number.isFinite(value) && value > 0 ? Math.min(value, 3600) : 60;
67
+ }
@@ -0,0 +1,120 @@
1
+ import { readFile } from 'node:fs/promises';
2
+ import type { Account } from '../agents/accounts';
3
+ import type { Check } from '../types';
4
+
5
+ /** The tools T3 offers once per proxy account, by T3 driver. */
6
+ const DRIVERS: Record<string, string> = {
7
+ claudeAgent: 'claude',
8
+ codex: 'codex',
9
+ opencode: 'opencode',
10
+ };
11
+
12
+ /** One per-account instance a node offers: its tool and account. */
13
+ export interface Instance {
14
+ tool: string;
15
+ account: string;
16
+ }
17
+
18
+ /** The per-account instances this node's T3 settings offer, read from the
19
+ * account each one pins (WS_AGENT_ACCOUNT). An unreadable file offers none.
20
+ */
21
+ export async function exposedInstances(settings: string): Promise<Instance[]> {
22
+ try {
23
+ const parsed = JSON.parse(await readFile(settings, 'utf8')) as {
24
+ providerInstances?: Record<
25
+ string,
26
+ {
27
+ driver?: unknown;
28
+ environment?: { name?: unknown; value?: unknown }[];
29
+ }
30
+ >;
31
+ };
32
+ return Object.values(parsed.providerInstances ?? {}).flatMap((instance) => {
33
+ const tool = DRIVERS[String(instance.driver)];
34
+ const account = (instance.environment ?? []).find(
35
+ (variable) => variable.name === 'WS_AGENT_ACCOUNT'
36
+ )?.value;
37
+ return tool && typeof account === 'string' ? [{ tool, account }] : [];
38
+ });
39
+ } catch {
40
+ return [];
41
+ }
42
+ }
43
+
44
+ export type SessionResult = 'ok' | 'cooling-down' | 'failed';
45
+
46
+ /** Runs one real, cheap session through the instance's launcher. */
47
+ export type Session = (instance: Instance) => Promise<SessionResult>;
48
+
49
+ /** `agent:<tool>:account-<index>:session`, one per account instance: the
50
+ * accounts the proxy holds and any instance the node still offers. An
51
+ * instance whose account the proxy no longer holds fails without being run;
52
+ * a rate-limited account is `cooling-down`, a warning and never a pass. The
53
+ * index is the account's position among its provider's, never its name.
54
+ */
55
+ export async function accountSessionChecks(
56
+ live: Account[],
57
+ exposed: Instance[],
58
+ session: Session
59
+ ): Promise<Check[]> {
60
+ const key = (instance: Instance) => `${instance.tool}\0${instance.account}`;
61
+ const held = new Set(
62
+ live.map((account) => key({ tool: account.tool, account: account.email }))
63
+ );
64
+ const index = (instance: Instance) =>
65
+ live
66
+ .filter((account) => account.tool === instance.tool)
67
+ .findIndex((account) => account.email === instance.account) + 1;
68
+ const removed = exposed.filter((instance) => !held.has(key(instance)));
69
+ const checks = await Promise.all(
70
+ live.map(async (account): Promise<Check> => {
71
+ const instance = { tool: account.tool, account: account.email };
72
+ const id = `agent:${account.tool}:account-${index(instance)}:session`;
73
+ const result = await session(instance);
74
+ if (result === 'ok')
75
+ return {
76
+ id,
77
+ status: 'pass',
78
+ summary: `${account.tool} account ${index(instance)} runs a session through its T3 launcher.`,
79
+ };
80
+ if (result === 'cooling-down')
81
+ return {
82
+ id,
83
+ status: 'warn',
84
+ summary: `AGENT_ACCOUNT_COOLING_DOWN: ${account.tool} account ${index(instance)} is rate limited; checked again on the next status.`,
85
+ };
86
+ return {
87
+ id,
88
+ status: 'fail',
89
+ summary: `AGENT_ACCOUNT_SESSION_FAILED: ${account.tool} account ${index(instance)} did not run a session.`,
90
+ remedy: {
91
+ action: 'Check this account on the proxy.',
92
+ command: 'devfleet fleet-accounts',
93
+ },
94
+ };
95
+ })
96
+ );
97
+ const byTool = new Map<string, number>();
98
+ for (const instance of removed) {
99
+ const count = (byTool.get(instance.tool) ?? 0) + 1;
100
+ byTool.set(instance.tool, count);
101
+ checks.push({
102
+ id: `agent:${instance.tool}:removed-${count}:session`,
103
+ status: 'fail',
104
+ summary: `AGENT_ACCOUNT_REMOVED: this node still offers a ${instance.tool} instance for an account the proxy no longer holds.`,
105
+ remedy: {
106
+ action: 'Propagate the accounts to every node.',
107
+ command: 'devfleet fleet-sync --only accounts',
108
+ },
109
+ });
110
+ }
111
+ return checks;
112
+ }
113
+
114
+ /** What a session's output says about its account. */
115
+ export function sessionResult(ok: boolean, output: string): SessionResult {
116
+ if (ok && /\bOK\b/.test(output)) return 'ok';
117
+ if (/cooling down until|rate[ _-]?limit|\b429\b|quota/i.test(output))
118
+ return 'cooling-down';
119
+ return 'failed';
120
+ }
@@ -1,19 +1,38 @@
1
+ import {
2
+ AccountSelector,
3
+ accountRefused,
4
+ retryAfter,
5
+ } from '../accounts/selection';
6
+
1
7
  /** Pins one agent process to one provider account. The proxy routes a model
2
8
  * named `<prefix>/<model>` only to the credential that owns the prefix, but
3
9
  * agents send whichever model the user picked. This loopback relay lives only
4
10
  * as long as the agent it was started for: it adds the prefix to each request
5
11
  * and forwards everything else (credentials, streaming) unchanged.
12
+ *
13
+ * With an empty prefix it pins nothing: every launch goes through a relay, so
14
+ * every launch rides out a proxy restart the same way. `selector` then picks
15
+ * the account of each request by the fleet's selection policy.
6
16
  */
7
17
  export function startAccountRelay(
8
18
  endpoint: string,
9
- prefix: string
19
+ prefix: string,
20
+ options: Pick<RelayOptions, 'selector' | 'alternatives'> = {}
10
21
  ): { url: string; stop: () => void } {
22
+ const pinned =
23
+ prefix && !options.selector
24
+ ? new AccountSelector([prefix], { strategy: 'fill-first' })
25
+ : options.selector;
11
26
  const server = Bun.serve({
12
27
  hostname: '127.0.0.1',
13
28
  port: 0,
14
29
  // Model streams are idle for as long as a model thinks.
15
30
  idleTimeout: 0,
16
- fetch: (request) => forward(request, endpoint, prefix),
31
+ fetch: (request) =>
32
+ forward(request, endpoint, prefix, {
33
+ ...options,
34
+ ...(pinned ? { selector: pinned } : {}),
35
+ }),
17
36
  });
18
37
  return {
19
38
  url: `http://127.0.0.1:${server.port}`,
@@ -24,6 +43,11 @@ export function startAccountRelay(
24
43
  type RelayOptions = {
25
44
  send?: typeof fetch;
26
45
  pause?: (seconds: number) => Promise<void>;
46
+ /** Unpinned: the accounts to try in turn. Pinned: remembers cooldowns. */
47
+ selector?: AccountSelector;
48
+ /** Pinned: the other instances of the provider, named in its error. */
49
+ alternatives?: string[];
50
+ now?: () => number;
27
51
  };
28
52
  const sleep = (seconds: number) =>
29
53
  new Promise<void>((done) => setTimeout(done, seconds * 1000));
@@ -32,33 +56,54 @@ export async function forward(
32
56
  request: Request,
33
57
  endpoint: string,
34
58
  prefix: string,
35
- { send = fetch, pause = sleep }: RelayOptions = {}
59
+ options: RelayOptions = {}
36
60
  ): Promise<Response> {
37
61
  const url = new URL(request.url);
38
62
  const headers = new Headers(request.headers);
39
63
  // fetch sets these for the upstream request and decodes its response.
40
64
  for (const name of ['host', 'content-length', 'accept-encoding'])
41
65
  headers.delete(name);
42
- let body: string | undefined;
43
- if (request.body) {
44
- // An unreadable body could not be pinned; never forward it to the pool.
45
- if (headers.has('content-encoding'))
46
- return new Response('The account relay needs an uncompressed request.', {
47
- status: 415,
48
- });
49
- body = pinModel(await request.text(), prefix);
50
- }
51
- const attempt = () =>
52
- send(endpoint + url.pathname + url.search, {
53
- method: request.method,
54
- headers,
55
- ...(body === undefined ? {} : { body }),
56
- redirect: 'error',
57
- signal: request.signal,
66
+ // An unreadable body could not be pinned; never forward it to the pool.
67
+ if (request.body && headers.has('content-encoding'))
68
+ return new Response('The account relay needs an uncompressed request.', {
69
+ status: 415,
58
70
  });
59
- const upstream = await reachProxy(attempt, request.signal, pause);
60
- if (!upstream)
61
- return new Response('The CLI proxy is unreachable.', { status: 502 });
71
+ const raw = request.body ? await request.text() : undefined;
72
+ const reach = reacher(
73
+ request,
74
+ headers,
75
+ endpoint + url.pathname + url.search,
76
+ options
77
+ );
78
+ const { selector } = options;
79
+ const reached =
80
+ !prefix && selector && raw !== undefined
81
+ ? await failOver(selector, raw, reach)
82
+ : await pinnedReach(prefix, raw, reach, options);
83
+ if (reached instanceof Response) return reached;
84
+ return answer(reached);
85
+ }
86
+
87
+ /** Sends one body to the proxy, retrying while the route is down. */
88
+ function reacher(
89
+ request: Request,
90
+ headers: Headers,
91
+ target: string,
92
+ { send = fetch, pause = sleep, now = Date.now }: RelayOptions
93
+ ): (body: string | undefined) => Promise<Reached> {
94
+ return (body) =>
95
+ reachProxy(
96
+ () => send(target, upstreamInit(request, headers, body)),
97
+ request.signal,
98
+ pause,
99
+ now
100
+ );
101
+ }
102
+
103
+ /** The proxy's answer, decoded, or the distinct "unreachable" error. */
104
+ function answer(reached: Reached): Response {
105
+ const upstream = reached.response;
106
+ if (!upstream) return unreachable(reached.since);
62
107
  const answered = new Headers(upstream.headers);
63
108
  answered.delete('content-encoding');
64
109
  answered.delete('content-length');
@@ -68,6 +113,109 @@ export async function forward(
68
113
  });
69
114
  }
70
115
 
116
+ /** A pinned account (or no pinning at all). A pinned account the provider
117
+ * limited answers at once, until its cooldown clears, with the time and the
118
+ * other instances of its provider.
119
+ */
120
+ async function pinnedReach(
121
+ prefix: string,
122
+ raw: string | undefined,
123
+ reach: (body: string | undefined) => Promise<Reached>,
124
+ { selector, alternatives = [] }: RelayOptions
125
+ ): Promise<Reached | Response> {
126
+ const cooling = prefix && selector ? selector.coolingUntil(prefix) : null;
127
+ if (cooling !== null) return coolingDown(cooling, alternatives);
128
+ const reached = await reach(
129
+ raw !== undefined && prefix ? pinModel(raw, prefix) : raw
130
+ );
131
+ if (!prefix || !selector || reached.response?.status !== 429) return reached;
132
+ selector.limited(prefix, retryAfter(reached.response.headers));
133
+ await reached.response.body?.cancel().catch(() => undefined);
134
+ return coolingDown(selector.coolingUntil(prefix)!, alternatives);
135
+ }
136
+
137
+ type Reached = { response: Response | null; since: number };
138
+
139
+ function upstreamInit(
140
+ request: Request,
141
+ headers: Headers,
142
+ body: string | undefined
143
+ ): RequestInit {
144
+ return {
145
+ method: request.method,
146
+ headers,
147
+ ...(body === undefined ? {} : { body }),
148
+ redirect: 'error',
149
+ signal: request.signal,
150
+ };
151
+ }
152
+
153
+ /** An unpinned request goes to the first usable account by the fleet's
154
+ * policy; a quota or authentication refusal sends it on to the next. With
155
+ * every account cooling down, the proxy's own pool serves it.
156
+ */
157
+ async function failOver(
158
+ selector: AccountSelector,
159
+ raw: string,
160
+ reach: (body: string | undefined) => Promise<Reached>
161
+ ): Promise<Reached> {
162
+ const candidates = selector.candidates();
163
+ if (candidates.length === 0) return reach(raw);
164
+ let reached!: Reached;
165
+ for (const [position, candidate] of candidates.entries()) {
166
+ reached = await reach(pinModel(raw, candidate));
167
+ const status = reached.response?.status;
168
+ if (status === undefined || !accountRefused(status)) return reached;
169
+ selector.limited(candidate, retryAfter(reached.response!.headers));
170
+ if (position < candidates.length - 1)
171
+ await reached.response!.body?.cancel().catch(() => undefined);
172
+ }
173
+ return reached;
174
+ }
175
+
176
+ /** One error both Anthropic and OpenAI clients show as a message. */
177
+ function agentError(
178
+ status: number,
179
+ type: string,
180
+ message: string,
181
+ retry: number
182
+ ) {
183
+ return new Response(
184
+ JSON.stringify({ type: 'error', error: { type, message } }),
185
+ {
186
+ status,
187
+ headers: {
188
+ 'content-type': 'application/json',
189
+ 'retry-after': String(Math.max(1, Math.ceil(retry))),
190
+ },
191
+ }
192
+ );
193
+ }
194
+
195
+ function coolingDown(until: number, alternatives: string[]): Response {
196
+ const others = alternatives.length
197
+ ? ` Pick another instance of this provider in T3: ${alternatives.join(', ')}.`
198
+ : '';
199
+ return agentError(
200
+ 429,
201
+ 'rate_limit_error',
202
+ `This account is cooling down until ${new Date(until).toISOString()}.${others}`,
203
+ (until - Date.now()) / 1000
204
+ );
205
+ }
206
+
207
+ /** The proxy did not answer within the retry window: a distinct, retryable
208
+ * error that says since when, never a provider's error.
209
+ */
210
+ function unreachable(since: number): Response {
211
+ return agentError(
212
+ 503,
213
+ 'proxy_unreachable',
214
+ `The CLI proxy has been unreachable since ${new Date(since).toISOString()}. This is retryable; nothing is wrong with the account.`,
215
+ 30
216
+ );
217
+ }
218
+
71
219
  /** Seconds to wait before each further attempt (about a minute in total). */
72
220
  export const RETRY_DELAYS = [1, 2, 4, 8, 15, 30];
73
221
  /** The route to the proxy answered instead of the proxy: its connector or the
@@ -83,13 +231,15 @@ const routeDown = (status: number) =>
83
231
  async function reachProxy(
84
232
  attempt: () => Promise<Response>,
85
233
  signal: AbortSignal,
86
- pause: (seconds: number) => Promise<void>
87
- ): Promise<Response | null> {
234
+ pause: (seconds: number) => Promise<void>,
235
+ now: () => number
236
+ ): Promise<Reached> {
237
+ const since = now();
88
238
  for (let tries = 0; ; tries++) {
89
239
  const delay = RETRY_DELAYS[tries];
90
240
  const response = await attempt().catch(() => null);
91
- if (response && !routeDown(response.status)) return response;
92
- if (delay === undefined || signal.aborted) return response;
241
+ if (response && !routeDown(response.status)) return { response, since };
242
+ if (delay === undefined || signal.aborted) return { response, since };
93
243
  await response?.body?.cancel().catch(() => undefined);
94
244
  await pause(delay);
95
245
  }
@@ -135,6 +135,14 @@ const UPSTREAM: Record<string, string> = {
135
135
  'The provider did not answer within the bounded verification deadline. Retry later.',
136
136
  };
137
137
 
138
+ /** The declared route cannot work as it is: it fails, and stays failed,
139
+ * until the route is changed. Retrying or signing in again changes nothing.
140
+ */
141
+ const INCOMPATIBLE: Record<string, string> = {
142
+ PROVIDER_PROTOCOL_INCOMPATIBLE:
143
+ 'The gateway does not serve the protocol this route uses (no such endpoint, method or media type). Nothing on this machine can fix it: change the route in agents/registry.json to one listed under "Supported alternatives" in agents/setup.md, then run fleet-sync.',
144
+ };
145
+
138
146
  export function inferenceFailure(provider: string, code: string): Check {
139
147
  if (QUOTA.has(code))
140
148
  return {
@@ -150,6 +158,7 @@ export function inferenceFailure(provider: string, code: string): Check {
150
158
  remedy: {
151
159
  action:
152
160
  upstream ??
161
+ INCOMPATIBLE[code] ??
153
162
  `Verify unattended Vault access and the declared ${provider} model on the singleton gateway. Preserve OAuth state; do not log native auth on proxy clients.`,
154
163
  },
155
164
  };
@@ -6,6 +6,9 @@ import { accountPrefix, accountSlug, discoverAccounts } from './accounts';
6
6
  import { startAccountRelay } from './account-relay';
7
7
  import { proxyKey } from './secrets';
8
8
  import { pinnedBinary, pinnedPath } from './toolchain';
9
+ import { AccountSelector } from '../accounts/selection';
10
+ import { fleet } from '../fleet-config';
11
+ import type { AccountSelection } from '../definition';
9
12
 
10
13
  /** `account` (its email) pins a proxy launch to one signed-in proxy account. */
11
14
  export async function launchProfile(
@@ -43,9 +46,11 @@ export async function launchProfile(
43
46
  }
44
47
  toolEnvironment(env, selection, directory);
45
48
  const pinned =
46
- account && selection.profile === 'proxy'
47
- ? await pinAccount(env, selection, { account, directory })
48
- : null;
49
+ selection.profile !== 'proxy'
50
+ ? null
51
+ : account
52
+ ? await pinAccount(env, selection, { account, directory })
53
+ : await routeThroughRelay(env, selection);
49
54
  try {
50
55
  return await spawnAgent(selection, directory, env, [
51
56
  pinned?.args ?? [],
@@ -96,9 +101,14 @@ export async function pinAccount(
96
101
  ): Promise<{ args: string[]; stop: () => void }> {
97
102
  const prefix = accountPrefix(selection.tool, account);
98
103
  // What the account serves right now, preferred default first.
99
- const served = (await discover(env['CLIPROXY_API_KEY'] ?? '')).find(
104
+ const roster = await discover(env['CLIPROXY_API_KEY'] ?? '');
105
+ const served = roster.find(
100
106
  (entry) => entry.tool === selection.tool && entry.email === account
101
107
  );
108
+ // Named in this account's "cooling down" error, so the person can switch.
109
+ const alternatives = roster
110
+ .filter((entry) => entry.tool === selection.tool && entry.email !== account)
111
+ .map((entry) => entry.email);
102
112
  if (!served)
103
113
  throw new AgentConfigurationError(
104
114
  'This account is not signed in on the CLI proxy.'
@@ -108,7 +118,9 @@ export async function pinAccount(
108
118
  env['OPENCODE_CONFIG_CONTENT'] = JSON.stringify(accountOpenCode(served));
109
119
  return { args: [], stop: () => {} };
110
120
  }
111
- const { url, stop } = relay(registry.proxy.endpoint, prefix);
121
+ const { url, stop } = relay(registry.proxy.endpoint, prefix, {
122
+ alternatives,
123
+ });
112
124
  if (selection.tool === 'claude') {
113
125
  env['ANTHROPIC_BASE_URL'] = url;
114
126
  // Aliases and the default must name models a Claude account serves.
@@ -146,6 +158,73 @@ export async function pinAccount(
146
158
  };
147
159
  }
148
160
 
161
+ /** Which tools reach the proxy through an address this wrapper sets, and so
162
+ * can be sent through a relay. OpenCode and Grok read theirs from rendered
163
+ * configuration and still go to the proxy directly.
164
+ */
165
+ const RELAYED = new Set(['claude', 'codex', 'gemini', 'antigravity']);
166
+
167
+ /** An unpinned launch goes through a relay as well: it retries a proxy that
168
+ * is restarting exactly as a pinned one does, reports "proxy unreachable"
169
+ * when it stays down, and picks each request's account by the fleet's
170
+ * selection policy when it declares one for the provider.
171
+ */
172
+ export async function routeThroughRelay(
173
+ env: NodeJS.ProcessEnv,
174
+ selection: Selection,
175
+ {
176
+ relay = startAccountRelay,
177
+ discover = discoverAccounts,
178
+ policy = selectionPolicy,
179
+ } = {}
180
+ ): Promise<{ args: string[]; stop: () => void } | null> {
181
+ if (!RELAYED.has(selection.tool)) return null;
182
+ const selector = await policySelector(env, selection, discover, policy);
183
+ const { url, stop } = relay(
184
+ registry.proxy.endpoint,
185
+ '',
186
+ selector ? { selector } : {}
187
+ );
188
+ if (selection.tool === 'claude') env['ANTHROPIC_BASE_URL'] = url;
189
+ if (selection.tool === 'gemini' || selection.tool === 'antigravity')
190
+ env['GOOGLE_GEMINI_BASE_URL'] = url;
191
+ const args =
192
+ selection.tool === 'codex'
193
+ ? [
194
+ '-c',
195
+ `model_providers.personal_proxy.base_url=${JSON.stringify(`${url}/v1`)}`,
196
+ ]
197
+ : [];
198
+ return { args, stop };
199
+ }
200
+
201
+ /** The fleet's selection policy for the tool's provider, over the accounts
202
+ * the proxy lists. The list is read only when a policy needs it; an
203
+ * unreadable list leaves the choice to the proxy's own pool.
204
+ */
205
+ async function policySelector(
206
+ env: NodeJS.ProcessEnv,
207
+ selection: Selection,
208
+ discover: typeof discoverAccounts,
209
+ policy: (provider: string) => AccountSelection | undefined
210
+ ): Promise<AccountSelector | undefined> {
211
+ const route = (
212
+ registry.proxy.accounts as Record<string, { provider: string }>
213
+ )[selection.tool];
214
+ const declared = route ? policy(route.provider) : undefined;
215
+ if (!declared) return undefined;
216
+ const accounts = await discover(env['CLIPROXY_API_KEY'] ?? '').catch(
217
+ () => []
218
+ );
219
+ const prefixes = accounts
220
+ .filter((account) => account.tool === selection.tool)
221
+ .map((account) => account.prefix);
222
+ return prefixes.length ? new AccountSelector(prefixes, declared) : undefined;
223
+ }
224
+
225
+ const selectionPolicy = (provider: string): AccountSelection | undefined =>
226
+ fleet.services.proxy.accounts?.selection?.[provider];
227
+
149
228
  /** OpenCode configuration that offers only one account's models. */
150
229
  function accountOpenCode(route: { prefix: string; models: string[] }) {
151
230
  const models = route.models.map((model) => `${route.prefix}/${model}`);