@crvouga/dev-fleet 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +265 -37
- package/agents/setup.md +42 -12
- package/ansible/macos/manager.yml +10 -7
- package/ansible/macos/policy.json +2 -10
- package/ansible/macos/t3-policy.yml +49 -0
- package/ansible/monitoring/README.md +86 -25
- package/ansible/monitoring/agent.yml +40 -2
- package/ansible/monitoring/hub-route-darwin.yml +51 -0
- package/ansible/monitoring/lock.json +5 -0
- package/ansible/pyproject.toml +12 -0
- package/ansible/ubuntu/baseline.yml +1 -1
- package/ansible/ubuntu/disk-gc.py +336 -0
- package/ansible/ubuntu/policy.json +13 -3
- package/ansible/ubuntu/t3-desktop.yml +19 -0
- package/ansible/ubuntu/t3-health.sh +93 -22
- package/ansible/ubuntu/t3-idle.py +42 -0
- package/ansible/ubuntu/t3-policy.yml +4 -11
- package/ansible/ubuntu/tailscale.yml +8 -5
- package/ansible/ubuntu/tasks/recovery.yml +60 -1
- package/ansible/ubuntu/tasks/resources.yml +61 -1
- package/ansible/ubuntu/tasks/t3-restart.yml +37 -0
- package/ansible/ubuntu/wip-guard.py +121 -0
- package/ansible/uv.lock +190 -0
- package/bootstrap/bootstrap.sh +25 -18
- package/bootstrap/doctor.ts +46 -1
- package/bootstrap/fleet-manager-run.sh +42 -15
- package/bootstrap/fleet-remote-bootstrap.sh +85 -12
- package/bootstrap/global-config.ts +64 -6
- package/bootstrap/install-launcher.sh +3 -0
- package/bootstrap/native-t3.ts +22 -11
- package/bootstrap/npm.ts +10 -2
- package/bootstrap/release-version.ts +157 -0
- package/cliproxy/README.md +3 -3
- package/cliproxy/scripts/accounts.py +216 -0
- package/cliproxy/scripts/checkpoint.py +29 -1
- package/cliproxy/scripts/cli.py +1 -1
- package/cliproxy/scripts/clientkeys.py +99 -0
- package/cliproxy/scripts/lease.py +80 -0
- package/cliproxy/scripts/ops +137 -25
- package/cliproxy/scripts/restore-verify +15 -0
- package/docs/accounts.md +154 -0
- package/docs/consumer-setup.md +408 -0
- package/docs/declared.md +91 -0
- package/docs/durability.md +26 -0
- package/docs/machine-setup.md +400 -0
- package/docs/operations.md +276 -0
- package/docs/releasing.md +103 -0
- package/docs/state.md +254 -0
- package/docs/topology.md +69 -0
- package/docs/ubuntu-sync.md +357 -0
- package/docs/worker-contract.md +67 -0
- package/examples/fleet.json +5 -0
- package/llms.txt +36 -0
- package/mise.lock +29 -0
- package/mise.toml +15 -2
- package/package.json +11 -2
- package/schema/fleet.schema.json +137 -15
- package/src/accounts/checks.ts +220 -0
- package/src/accounts/context.ts +76 -0
- package/src/accounts/propagate.ts +62 -0
- package/src/accounts/roster.ts +85 -0
- package/src/accounts/selection.ts +67 -0
- package/src/accounts/sessions.ts +120 -0
- package/src/agents/account-relay.ts +176 -26
- package/src/agents/fleet-verify.ts +9 -0
- package/src/agents/launch.ts +84 -5
- package/src/agents/native-verify.ts +293 -22
- package/src/agents/provider-errors.ts +9 -2
- package/src/agents/registry.ts +7 -1
- package/src/cli/agent-guide.ts +39 -5
- package/src/cli/app.ts +11 -4
- package/src/cli/cli.ts +7 -0
- package/src/cli/command-index.ts +7 -0
- package/src/cli/fleet-accounts.ts +253 -0
- package/src/cli/fleet-state.ts +499 -0
- package/src/cli/fleet-sync-accounts.ts +58 -0
- package/src/cli/fleet-sync-checkout.ts +56 -17
- package/src/cli/fleet-sync-enrollment.ts +19 -23
- package/src/cli/fleet-sync-forward.ts +1 -1
- package/src/cli/fleet-sync-manager.ts +84 -12
- package/src/cli/fleet-sync-monitoring.ts +5 -5
- package/src/cli/fleet-sync-output.ts +7 -3
- package/src/cli/fleet-sync-remote.ts +31 -18
- package/src/cli/fleet-sync-state.ts +86 -0
- package/src/cli/fleet-sync.ts +159 -57
- package/src/cli/fleet.ts +4 -2
- package/src/cli/managed-health.ts +4 -4
- package/src/cli/node-sync.ts +7 -0
- package/src/cli/program.ts +20 -4
- package/src/cli/project.ts +51 -0
- package/src/cli/register-fleet.ts +86 -1
- package/src/cli/registry.ts +41 -1
- package/src/cli/roster.ts +46 -30
- package/src/cli/settings.ts +15 -12
- package/src/cli/source-operations.ts +27 -3
- package/src/cli/tailnet.ts +2 -2
- package/src/definition.ts +116 -4
- package/src/diagnostics.ts +1 -1
- package/src/fleet-config.ts +105 -19
- package/src/ide/definition.ts +86 -0
- package/src/ide/ide.ts +43 -0
- package/src/ide/index.ts +5 -0
- package/src/ide/orchestrate.ts +50 -0
- package/src/ide/t3.ts +111 -0
- package/src/machine/source.ts +3 -3
- package/src/machine/state.ts +15 -27
- package/src/manager-sync.ts +9 -18
- package/src/monitoring/agent-evidence.ts +146 -0
- package/src/monitoring/apply.ts +15 -3
- package/src/monitoring/beszel.ts +11 -4
- package/src/monitoring/enroll.ts +2 -1
- package/src/monitoring/hub-access.ts +4 -3
- package/src/monitoring/hub-backup.ts +121 -0
- package/src/monitoring/hub-viewer.ts +8 -8
- package/src/monitoring/hub.ts +3 -3
- package/src/monitoring/manager-hub.ts +30 -14
- package/src/node-converge-linux.ts +30 -7
- package/src/package-info.ts +45 -0
- package/src/platform/ansible.ts +19 -15
- package/src/platform/darwin.ts +0 -13
- package/src/platform/fallback.ts +0 -8
- package/src/platform/linux.ts +0 -10
- package/src/platform/resource-verify.ts +8 -4
- package/src/platform/t3-policy.ts +33 -6
- package/src/platform/t3-service.ts +104 -0
- package/src/platform/types.ts +0 -4
- package/src/platform/ubuntu-audit.ts +5 -2
- package/src/platform/windows.ts +0 -14
- package/src/remote-bootstrap.ts +17 -19
- package/src/remote-entry.ts +15 -1
- package/src/repair-catalog.ts +1 -1
- package/src/runner.ts +1 -1
- package/src/service-auth.ts +15 -5
- package/src/state/bucket.ts +208 -0
- package/src/state/check.ts +74 -0
- package/src/state/connection.ts +92 -0
- package/src/state/enrollment.ts +45 -0
- package/src/state/legacy.ts +89 -0
- package/src/state/litestream.ts +143 -0
- package/src/state/operator.ts +20 -0
- package/src/state/paths.ts +40 -0
- package/src/state/roster.ts +62 -0
- package/src/state/store.ts +169 -0
- package/src/state/sync.ts +352 -0
- package/src/t3/accounts.ts +7 -1
- package/src/t3/models.ts +133 -0
- package/src/t3/pairing.ts +31 -0
- package/src/t3/profile.ts +203 -0
- package/src/t3/version.ts +119 -0
- package/src/t3-endpoint.ts +64 -8
- package/src/t3-linked-check.ts +24 -2
- package/src/toolchain-checks.ts +25 -0
- package/src/transport.ts +3 -2
- package/src/types.ts +4 -0
- package/src/worker-access-checks.ts +56 -3
- package/src/worker-host-checks.ts +41 -13
- package/src/workstation-checks.ts +1 -1
- package/src/index.ts +0 -6
- package/src/test-fixtures.ts +0 -36
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import type { Account } from '../agents/accounts';
|
|
3
|
+
|
|
4
|
+
/** Provider accounts exist only as the CLI proxy's credential files. Every
|
|
5
|
+
* node and IDE gets a view derived from them; nothing on a node holds a
|
|
6
|
+
* provider login. See docs/accounts.md.
|
|
7
|
+
*/
|
|
8
|
+
export type AccountState =
|
|
9
|
+
'usable' | 'cooling-down' | 'needs-login' | 'refresh-failing' | 'removed';
|
|
10
|
+
|
|
11
|
+
const STATES: AccountState[] = [
|
|
12
|
+
'usable',
|
|
13
|
+
'cooling-down',
|
|
14
|
+
'needs-login',
|
|
15
|
+
'refresh-failing',
|
|
16
|
+
'removed',
|
|
17
|
+
];
|
|
18
|
+
|
|
19
|
+
/** One account as the manager reports it. `index` is opaque: the position of
|
|
20
|
+
* the account among its provider's, the only way checks name it.
|
|
21
|
+
*/
|
|
22
|
+
export interface RosterAccount {
|
|
23
|
+
provider: string;
|
|
24
|
+
index: number;
|
|
25
|
+
label: string;
|
|
26
|
+
/** The routing prefix every node offers it under; null if not routed. */
|
|
27
|
+
prefix: string | null;
|
|
28
|
+
state: AccountState;
|
|
29
|
+
/** While cooling down: when it clears (ISO 8601), when known. */
|
|
30
|
+
until: string | null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** What `devfleet cliproxy accounts --json` prints on the manager. */
|
|
34
|
+
export interface ProxyRoster {
|
|
35
|
+
version: 1;
|
|
36
|
+
rosterVersion: string;
|
|
37
|
+
/** Whether the proxy's management API answered: without it, cooldowns and
|
|
38
|
+
* refresh failures are not known.
|
|
39
|
+
*/
|
|
40
|
+
managementApi: boolean;
|
|
41
|
+
accounts: RosterAccount[];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function parseRoster(text: string): ProxyRoster {
|
|
45
|
+
const value = JSON.parse(text) as Partial<ProxyRoster>;
|
|
46
|
+
if (
|
|
47
|
+
value.version !== 1 ||
|
|
48
|
+
typeof value.rosterVersion !== 'string' ||
|
|
49
|
+
!Array.isArray(value.accounts) ||
|
|
50
|
+
!value.accounts.every(
|
|
51
|
+
(account) =>
|
|
52
|
+
typeof account.provider === 'string' &&
|
|
53
|
+
Number.isInteger(account.index) &&
|
|
54
|
+
typeof account.label === 'string' &&
|
|
55
|
+
STATES.includes(account.state)
|
|
56
|
+
)
|
|
57
|
+
)
|
|
58
|
+
throw new Error('The proxy reported an unreadable account roster.');
|
|
59
|
+
return {
|
|
60
|
+
version: 1,
|
|
61
|
+
rosterVersion: value.rosterVersion,
|
|
62
|
+
managementApi: value.managementApi === true,
|
|
63
|
+
accounts: value.accounts.map((account) => ({
|
|
64
|
+
...account,
|
|
65
|
+
prefix: account.prefix ?? null,
|
|
66
|
+
until: account.until ?? null,
|
|
67
|
+
})),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** The roster version: a digest of the routing prefixes of the accounts
|
|
72
|
+
* every node offers. The manager computes it from the credential files
|
|
73
|
+
* (cliproxy/scripts/accounts.py) and every node from the proxy's model list;
|
|
74
|
+
* the two agree exactly when the node offers what the proxy holds.
|
|
75
|
+
*/
|
|
76
|
+
export function rosterVersion(prefixes: Iterable<string>): string {
|
|
77
|
+
return createHash('sha256')
|
|
78
|
+
.update([...new Set(prefixes)].sort().join('\n'))
|
|
79
|
+
.digest('hex')
|
|
80
|
+
.slice(0, 16);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The version of the accounts a node reads from the proxy's model list. */
|
|
84
|
+
export const liveVersion = (accounts: Account[]): string =>
|
|
85
|
+
rosterVersion(accounts.map((account) => account.prefix));
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { AccountSelection } from '../definition';
|
|
2
|
+
|
|
3
|
+
/** Which account an unpinned request goes to first, per the policy the fleet
|
|
4
|
+
* declares for the provider. The order is deterministic: the accounts the
|
|
5
|
+
* policy names, in its order, then the others in the order of their routing
|
|
6
|
+
* prefix. Accounts cooling down are skipped while any other is left.
|
|
7
|
+
*/
|
|
8
|
+
export class AccountSelector {
|
|
9
|
+
private readonly cooling = new Map<string, number>();
|
|
10
|
+
private readonly limitedAt = new Map<string, number>();
|
|
11
|
+
private turn = 0;
|
|
12
|
+
readonly order: string[];
|
|
13
|
+
|
|
14
|
+
constructor(
|
|
15
|
+
prefixes: string[],
|
|
16
|
+
private readonly policy: AccountSelection,
|
|
17
|
+
private readonly now: () => number = Date.now
|
|
18
|
+
) {
|
|
19
|
+
const label = (prefix: string) => prefix.slice(prefix.indexOf('-') + 1);
|
|
20
|
+
const named = policy.order ?? [];
|
|
21
|
+
const rank = (prefix: string) => {
|
|
22
|
+
const position = named.indexOf(label(prefix));
|
|
23
|
+
return position < 0 ? named.length : position;
|
|
24
|
+
};
|
|
25
|
+
this.order = [...new Set(prefixes)].sort(
|
|
26
|
+
(a, b) => rank(a) - rank(b) || a.localeCompare(b)
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** The accounts to try for the next request, in order. */
|
|
31
|
+
candidates(): string[] {
|
|
32
|
+
const now = this.now();
|
|
33
|
+
let order = [...this.order];
|
|
34
|
+
if (this.policy.strategy === 'round-robin' && order.length) {
|
|
35
|
+
const start = this.turn++ % order.length;
|
|
36
|
+
order = [...order.slice(start), ...order.slice(0, start)];
|
|
37
|
+
}
|
|
38
|
+
if (this.policy.strategy === 'least-recently-limited')
|
|
39
|
+
order.sort(
|
|
40
|
+
(a, b) => (this.limitedAt.get(a) ?? 0) - (this.limitedAt.get(b) ?? 0)
|
|
41
|
+
);
|
|
42
|
+
return order.filter((prefix) => (this.cooling.get(prefix) ?? 0) <= now);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** The provider limited this account; `seconds` from its Retry-After. */
|
|
46
|
+
limited(prefix: string, seconds = 60): void {
|
|
47
|
+
const now = this.now();
|
|
48
|
+
this.limitedAt.set(prefix, now);
|
|
49
|
+
this.cooling.set(prefix, now + seconds * 1000);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** When this account's cooldown clears, if it is cooling down. */
|
|
53
|
+
coolingUntil(prefix: string): number | null {
|
|
54
|
+
const until = this.cooling.get(prefix) ?? 0;
|
|
55
|
+
return until > this.now() ? until : null;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** A quota or authentication answer: the next account may still serve. */
|
|
60
|
+
export const accountRefused = (status: number): boolean =>
|
|
61
|
+
status === 429 || status === 401 || status === 403;
|
|
62
|
+
|
|
63
|
+
/** Seconds a refused account cools down for: its Retry-After, or a minute. */
|
|
64
|
+
export function retryAfter(headers: Headers): number {
|
|
65
|
+
const value = Number(headers.get('retry-after'));
|
|
66
|
+
return Number.isFinite(value) && value > 0 ? Math.min(value, 3600) : 60;
|
|
67
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import type { Account } from '../agents/accounts';
|
|
3
|
+
import type { Check } from '../types';
|
|
4
|
+
|
|
5
|
+
/** The tools T3 offers once per proxy account, by T3 driver. */
|
|
6
|
+
const DRIVERS: Record<string, string> = {
|
|
7
|
+
claudeAgent: 'claude',
|
|
8
|
+
codex: 'codex',
|
|
9
|
+
opencode: 'opencode',
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
/** One per-account instance a node offers: its tool and account. */
|
|
13
|
+
export interface Instance {
|
|
14
|
+
tool: string;
|
|
15
|
+
account: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** The per-account instances this node's T3 settings offer, read from the
|
|
19
|
+
* account each one pins (WS_AGENT_ACCOUNT). An unreadable file offers none.
|
|
20
|
+
*/
|
|
21
|
+
export async function exposedInstances(settings: string): Promise<Instance[]> {
|
|
22
|
+
try {
|
|
23
|
+
const parsed = JSON.parse(await readFile(settings, 'utf8')) as {
|
|
24
|
+
providerInstances?: Record<
|
|
25
|
+
string,
|
|
26
|
+
{
|
|
27
|
+
driver?: unknown;
|
|
28
|
+
environment?: { name?: unknown; value?: unknown }[];
|
|
29
|
+
}
|
|
30
|
+
>;
|
|
31
|
+
};
|
|
32
|
+
return Object.values(parsed.providerInstances ?? {}).flatMap((instance) => {
|
|
33
|
+
const tool = DRIVERS[String(instance.driver)];
|
|
34
|
+
const account = (instance.environment ?? []).find(
|
|
35
|
+
(variable) => variable.name === 'WS_AGENT_ACCOUNT'
|
|
36
|
+
)?.value;
|
|
37
|
+
return tool && typeof account === 'string' ? [{ tool, account }] : [];
|
|
38
|
+
});
|
|
39
|
+
} catch {
|
|
40
|
+
return [];
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export type SessionResult = 'ok' | 'cooling-down' | 'failed';
|
|
45
|
+
|
|
46
|
+
/** Runs one real, cheap session through the instance's launcher. */
|
|
47
|
+
export type Session = (instance: Instance) => Promise<SessionResult>;
|
|
48
|
+
|
|
49
|
+
/** `agent:<tool>:account-<index>:session`, one per account instance: the
|
|
50
|
+
* accounts the proxy holds and any instance the node still offers. An
|
|
51
|
+
* instance whose account the proxy no longer holds fails without being run;
|
|
52
|
+
* a rate-limited account is `cooling-down`, a warning and never a pass. The
|
|
53
|
+
* index is the account's position among its provider's, never its name.
|
|
54
|
+
*/
|
|
55
|
+
export async function accountSessionChecks(
|
|
56
|
+
live: Account[],
|
|
57
|
+
exposed: Instance[],
|
|
58
|
+
session: Session
|
|
59
|
+
): Promise<Check[]> {
|
|
60
|
+
const key = (instance: Instance) => `${instance.tool}\0${instance.account}`;
|
|
61
|
+
const held = new Set(
|
|
62
|
+
live.map((account) => key({ tool: account.tool, account: account.email }))
|
|
63
|
+
);
|
|
64
|
+
const index = (instance: Instance) =>
|
|
65
|
+
live
|
|
66
|
+
.filter((account) => account.tool === instance.tool)
|
|
67
|
+
.findIndex((account) => account.email === instance.account) + 1;
|
|
68
|
+
const removed = exposed.filter((instance) => !held.has(key(instance)));
|
|
69
|
+
const checks = await Promise.all(
|
|
70
|
+
live.map(async (account): Promise<Check> => {
|
|
71
|
+
const instance = { tool: account.tool, account: account.email };
|
|
72
|
+
const id = `agent:${account.tool}:account-${index(instance)}:session`;
|
|
73
|
+
const result = await session(instance);
|
|
74
|
+
if (result === 'ok')
|
|
75
|
+
return {
|
|
76
|
+
id,
|
|
77
|
+
status: 'pass',
|
|
78
|
+
summary: `${account.tool} account ${index(instance)} runs a session through its T3 launcher.`,
|
|
79
|
+
};
|
|
80
|
+
if (result === 'cooling-down')
|
|
81
|
+
return {
|
|
82
|
+
id,
|
|
83
|
+
status: 'warn',
|
|
84
|
+
summary: `AGENT_ACCOUNT_COOLING_DOWN: ${account.tool} account ${index(instance)} is rate limited; checked again on the next status.`,
|
|
85
|
+
};
|
|
86
|
+
return {
|
|
87
|
+
id,
|
|
88
|
+
status: 'fail',
|
|
89
|
+
summary: `AGENT_ACCOUNT_SESSION_FAILED: ${account.tool} account ${index(instance)} did not run a session.`,
|
|
90
|
+
remedy: {
|
|
91
|
+
action: 'Check this account on the proxy.',
|
|
92
|
+
command: 'devfleet fleet-accounts',
|
|
93
|
+
},
|
|
94
|
+
};
|
|
95
|
+
})
|
|
96
|
+
);
|
|
97
|
+
const byTool = new Map<string, number>();
|
|
98
|
+
for (const instance of removed) {
|
|
99
|
+
const count = (byTool.get(instance.tool) ?? 0) + 1;
|
|
100
|
+
byTool.set(instance.tool, count);
|
|
101
|
+
checks.push({
|
|
102
|
+
id: `agent:${instance.tool}:removed-${count}:session`,
|
|
103
|
+
status: 'fail',
|
|
104
|
+
summary: `AGENT_ACCOUNT_REMOVED: this node still offers a ${instance.tool} instance for an account the proxy no longer holds.`,
|
|
105
|
+
remedy: {
|
|
106
|
+
action: 'Propagate the accounts to every node.',
|
|
107
|
+
command: 'devfleet fleet-sync --only accounts',
|
|
108
|
+
},
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
return checks;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** What a session's output says about its account. */
|
|
115
|
+
export function sessionResult(ok: boolean, output: string): SessionResult {
|
|
116
|
+
if (ok && /\bOK\b/.test(output)) return 'ok';
|
|
117
|
+
if (/cooling down until|rate[ _-]?limit|\b429\b|quota/i.test(output))
|
|
118
|
+
return 'cooling-down';
|
|
119
|
+
return 'failed';
|
|
120
|
+
}
|
|
@@ -1,19 +1,38 @@
|
|
|
1
|
+
import {
|
|
2
|
+
AccountSelector,
|
|
3
|
+
accountRefused,
|
|
4
|
+
retryAfter,
|
|
5
|
+
} from '../accounts/selection';
|
|
6
|
+
|
|
1
7
|
/** Pins one agent process to one provider account. The proxy routes a model
|
|
2
8
|
* named `<prefix>/<model>` only to the credential that owns the prefix, but
|
|
3
9
|
* agents send whichever model the user picked. This loopback relay lives only
|
|
4
10
|
* as long as the agent it was started for: it adds the prefix to each request
|
|
5
11
|
* and forwards everything else (credentials, streaming) unchanged.
|
|
12
|
+
*
|
|
13
|
+
* With an empty prefix it pins nothing: every launch goes through a relay, so
|
|
14
|
+
* every launch rides out a proxy restart the same way. `selector` then picks
|
|
15
|
+
* the account of each request by the fleet's selection policy.
|
|
6
16
|
*/
|
|
7
17
|
export function startAccountRelay(
|
|
8
18
|
endpoint: string,
|
|
9
|
-
prefix: string
|
|
19
|
+
prefix: string,
|
|
20
|
+
options: Pick<RelayOptions, 'selector' | 'alternatives'> = {}
|
|
10
21
|
): { url: string; stop: () => void } {
|
|
22
|
+
const pinned =
|
|
23
|
+
prefix && !options.selector
|
|
24
|
+
? new AccountSelector([prefix], { strategy: 'fill-first' })
|
|
25
|
+
: options.selector;
|
|
11
26
|
const server = Bun.serve({
|
|
12
27
|
hostname: '127.0.0.1',
|
|
13
28
|
port: 0,
|
|
14
29
|
// Model streams are idle for as long as a model thinks.
|
|
15
30
|
idleTimeout: 0,
|
|
16
|
-
fetch: (request) =>
|
|
31
|
+
fetch: (request) =>
|
|
32
|
+
forward(request, endpoint, prefix, {
|
|
33
|
+
...options,
|
|
34
|
+
...(pinned ? { selector: pinned } : {}),
|
|
35
|
+
}),
|
|
17
36
|
});
|
|
18
37
|
return {
|
|
19
38
|
url: `http://127.0.0.1:${server.port}`,
|
|
@@ -24,6 +43,11 @@ export function startAccountRelay(
|
|
|
24
43
|
type RelayOptions = {
|
|
25
44
|
send?: typeof fetch;
|
|
26
45
|
pause?: (seconds: number) => Promise<void>;
|
|
46
|
+
/** Unpinned: the accounts to try in turn. Pinned: remembers cooldowns. */
|
|
47
|
+
selector?: AccountSelector;
|
|
48
|
+
/** Pinned: the other instances of the provider, named in its error. */
|
|
49
|
+
alternatives?: string[];
|
|
50
|
+
now?: () => number;
|
|
27
51
|
};
|
|
28
52
|
const sleep = (seconds: number) =>
|
|
29
53
|
new Promise<void>((done) => setTimeout(done, seconds * 1000));
|
|
@@ -32,33 +56,54 @@ export async function forward(
|
|
|
32
56
|
request: Request,
|
|
33
57
|
endpoint: string,
|
|
34
58
|
prefix: string,
|
|
35
|
-
|
|
59
|
+
options: RelayOptions = {}
|
|
36
60
|
): Promise<Response> {
|
|
37
61
|
const url = new URL(request.url);
|
|
38
62
|
const headers = new Headers(request.headers);
|
|
39
63
|
// fetch sets these for the upstream request and decodes its response.
|
|
40
64
|
for (const name of ['host', 'content-length', 'accept-encoding'])
|
|
41
65
|
headers.delete(name);
|
|
42
|
-
|
|
43
|
-
if (request.body)
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
return new Response('The account relay needs an uncompressed request.', {
|
|
47
|
-
status: 415,
|
|
48
|
-
});
|
|
49
|
-
body = pinModel(await request.text(), prefix);
|
|
50
|
-
}
|
|
51
|
-
const attempt = () =>
|
|
52
|
-
send(endpoint + url.pathname + url.search, {
|
|
53
|
-
method: request.method,
|
|
54
|
-
headers,
|
|
55
|
-
...(body === undefined ? {} : { body }),
|
|
56
|
-
redirect: 'error',
|
|
57
|
-
signal: request.signal,
|
|
66
|
+
// An unreadable body could not be pinned; never forward it to the pool.
|
|
67
|
+
if (request.body && headers.has('content-encoding'))
|
|
68
|
+
return new Response('The account relay needs an uncompressed request.', {
|
|
69
|
+
status: 415,
|
|
58
70
|
});
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
71
|
+
const raw = request.body ? await request.text() : undefined;
|
|
72
|
+
const reach = reacher(
|
|
73
|
+
request,
|
|
74
|
+
headers,
|
|
75
|
+
endpoint + url.pathname + url.search,
|
|
76
|
+
options
|
|
77
|
+
);
|
|
78
|
+
const { selector } = options;
|
|
79
|
+
const reached =
|
|
80
|
+
!prefix && selector && raw !== undefined
|
|
81
|
+
? await failOver(selector, raw, reach)
|
|
82
|
+
: await pinnedReach(prefix, raw, reach, options);
|
|
83
|
+
if (reached instanceof Response) return reached;
|
|
84
|
+
return answer(reached);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Sends one body to the proxy, retrying while the route is down. */
|
|
88
|
+
function reacher(
|
|
89
|
+
request: Request,
|
|
90
|
+
headers: Headers,
|
|
91
|
+
target: string,
|
|
92
|
+
{ send = fetch, pause = sleep, now = Date.now }: RelayOptions
|
|
93
|
+
): (body: string | undefined) => Promise<Reached> {
|
|
94
|
+
return (body) =>
|
|
95
|
+
reachProxy(
|
|
96
|
+
() => send(target, upstreamInit(request, headers, body)),
|
|
97
|
+
request.signal,
|
|
98
|
+
pause,
|
|
99
|
+
now
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** The proxy's answer, decoded, or the distinct "unreachable" error. */
|
|
104
|
+
function answer(reached: Reached): Response {
|
|
105
|
+
const upstream = reached.response;
|
|
106
|
+
if (!upstream) return unreachable(reached.since);
|
|
62
107
|
const answered = new Headers(upstream.headers);
|
|
63
108
|
answered.delete('content-encoding');
|
|
64
109
|
answered.delete('content-length');
|
|
@@ -68,6 +113,109 @@ export async function forward(
|
|
|
68
113
|
});
|
|
69
114
|
}
|
|
70
115
|
|
|
116
|
+
/** A pinned account (or no pinning at all). A pinned account the provider
|
|
117
|
+
* limited answers at once, until its cooldown clears, with the time and the
|
|
118
|
+
* other instances of its provider.
|
|
119
|
+
*/
|
|
120
|
+
async function pinnedReach(
|
|
121
|
+
prefix: string,
|
|
122
|
+
raw: string | undefined,
|
|
123
|
+
reach: (body: string | undefined) => Promise<Reached>,
|
|
124
|
+
{ selector, alternatives = [] }: RelayOptions
|
|
125
|
+
): Promise<Reached | Response> {
|
|
126
|
+
const cooling = prefix && selector ? selector.coolingUntil(prefix) : null;
|
|
127
|
+
if (cooling !== null) return coolingDown(cooling, alternatives);
|
|
128
|
+
const reached = await reach(
|
|
129
|
+
raw !== undefined && prefix ? pinModel(raw, prefix) : raw
|
|
130
|
+
);
|
|
131
|
+
if (!prefix || !selector || reached.response?.status !== 429) return reached;
|
|
132
|
+
selector.limited(prefix, retryAfter(reached.response.headers));
|
|
133
|
+
await reached.response.body?.cancel().catch(() => undefined);
|
|
134
|
+
return coolingDown(selector.coolingUntil(prefix)!, alternatives);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
type Reached = { response: Response | null; since: number };
|
|
138
|
+
|
|
139
|
+
function upstreamInit(
|
|
140
|
+
request: Request,
|
|
141
|
+
headers: Headers,
|
|
142
|
+
body: string | undefined
|
|
143
|
+
): RequestInit {
|
|
144
|
+
return {
|
|
145
|
+
method: request.method,
|
|
146
|
+
headers,
|
|
147
|
+
...(body === undefined ? {} : { body }),
|
|
148
|
+
redirect: 'error',
|
|
149
|
+
signal: request.signal,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** An unpinned request goes to the first usable account by the fleet's
|
|
154
|
+
* policy; a quota or authentication refusal sends it on to the next. With
|
|
155
|
+
* every account cooling down, the proxy's own pool serves it.
|
|
156
|
+
*/
|
|
157
|
+
async function failOver(
|
|
158
|
+
selector: AccountSelector,
|
|
159
|
+
raw: string,
|
|
160
|
+
reach: (body: string | undefined) => Promise<Reached>
|
|
161
|
+
): Promise<Reached> {
|
|
162
|
+
const candidates = selector.candidates();
|
|
163
|
+
if (candidates.length === 0) return reach(raw);
|
|
164
|
+
let reached!: Reached;
|
|
165
|
+
for (const [position, candidate] of candidates.entries()) {
|
|
166
|
+
reached = await reach(pinModel(raw, candidate));
|
|
167
|
+
const status = reached.response?.status;
|
|
168
|
+
if (status === undefined || !accountRefused(status)) return reached;
|
|
169
|
+
selector.limited(candidate, retryAfter(reached.response!.headers));
|
|
170
|
+
if (position < candidates.length - 1)
|
|
171
|
+
await reached.response!.body?.cancel().catch(() => undefined);
|
|
172
|
+
}
|
|
173
|
+
return reached;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** One error both Anthropic and OpenAI clients show as a message. */
|
|
177
|
+
function agentError(
|
|
178
|
+
status: number,
|
|
179
|
+
type: string,
|
|
180
|
+
message: string,
|
|
181
|
+
retry: number
|
|
182
|
+
) {
|
|
183
|
+
return new Response(
|
|
184
|
+
JSON.stringify({ type: 'error', error: { type, message } }),
|
|
185
|
+
{
|
|
186
|
+
status,
|
|
187
|
+
headers: {
|
|
188
|
+
'content-type': 'application/json',
|
|
189
|
+
'retry-after': String(Math.max(1, Math.ceil(retry))),
|
|
190
|
+
},
|
|
191
|
+
}
|
|
192
|
+
);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function coolingDown(until: number, alternatives: string[]): Response {
|
|
196
|
+
const others = alternatives.length
|
|
197
|
+
? ` Pick another instance of this provider in T3: ${alternatives.join(', ')}.`
|
|
198
|
+
: '';
|
|
199
|
+
return agentError(
|
|
200
|
+
429,
|
|
201
|
+
'rate_limit_error',
|
|
202
|
+
`This account is cooling down until ${new Date(until).toISOString()}.${others}`,
|
|
203
|
+
(until - Date.now()) / 1000
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** The proxy did not answer within the retry window: a distinct, retryable
|
|
208
|
+
* error that says since when, never a provider's error.
|
|
209
|
+
*/
|
|
210
|
+
function unreachable(since: number): Response {
|
|
211
|
+
return agentError(
|
|
212
|
+
503,
|
|
213
|
+
'proxy_unreachable',
|
|
214
|
+
`The CLI proxy has been unreachable since ${new Date(since).toISOString()}. This is retryable; nothing is wrong with the account.`,
|
|
215
|
+
30
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
|
|
71
219
|
/** Seconds to wait before each further attempt (about a minute in total). */
|
|
72
220
|
export const RETRY_DELAYS = [1, 2, 4, 8, 15, 30];
|
|
73
221
|
/** The route to the proxy answered instead of the proxy: its connector or the
|
|
@@ -83,13 +231,15 @@ const routeDown = (status: number) =>
|
|
|
83
231
|
async function reachProxy(
|
|
84
232
|
attempt: () => Promise<Response>,
|
|
85
233
|
signal: AbortSignal,
|
|
86
|
-
pause: (seconds: number) => Promise<void
|
|
87
|
-
|
|
234
|
+
pause: (seconds: number) => Promise<void>,
|
|
235
|
+
now: () => number
|
|
236
|
+
): Promise<Reached> {
|
|
237
|
+
const since = now();
|
|
88
238
|
for (let tries = 0; ; tries++) {
|
|
89
239
|
const delay = RETRY_DELAYS[tries];
|
|
90
240
|
const response = await attempt().catch(() => null);
|
|
91
|
-
if (response && !routeDown(response.status)) return response;
|
|
92
|
-
if (delay === undefined || signal.aborted) return response;
|
|
241
|
+
if (response && !routeDown(response.status)) return { response, since };
|
|
242
|
+
if (delay === undefined || signal.aborted) return { response, since };
|
|
93
243
|
await response?.body?.cancel().catch(() => undefined);
|
|
94
244
|
await pause(delay);
|
|
95
245
|
}
|
|
@@ -135,6 +135,14 @@ const UPSTREAM: Record<string, string> = {
|
|
|
135
135
|
'The provider did not answer within the bounded verification deadline. Retry later.',
|
|
136
136
|
};
|
|
137
137
|
|
|
138
|
+
/** The declared route cannot work as it is: it fails, and stays failed,
|
|
139
|
+
* until the route is changed. Retrying or signing in again changes nothing.
|
|
140
|
+
*/
|
|
141
|
+
const INCOMPATIBLE: Record<string, string> = {
|
|
142
|
+
PROVIDER_PROTOCOL_INCOMPATIBLE:
|
|
143
|
+
'The gateway does not serve the protocol this route uses (no such endpoint, method or media type). Nothing on this machine can fix it: change the route in agents/registry.json to one listed under "Supported alternatives" in agents/setup.md, then run fleet-sync.',
|
|
144
|
+
};
|
|
145
|
+
|
|
138
146
|
export function inferenceFailure(provider: string, code: string): Check {
|
|
139
147
|
if (QUOTA.has(code))
|
|
140
148
|
return {
|
|
@@ -150,6 +158,7 @@ export function inferenceFailure(provider: string, code: string): Check {
|
|
|
150
158
|
remedy: {
|
|
151
159
|
action:
|
|
152
160
|
upstream ??
|
|
161
|
+
INCOMPATIBLE[code] ??
|
|
153
162
|
`Verify unattended Vault access and the declared ${provider} model on the singleton gateway. Preserve OAuth state; do not log native auth on proxy clients.`,
|
|
154
163
|
},
|
|
155
164
|
};
|
package/src/agents/launch.ts
CHANGED
|
@@ -6,6 +6,9 @@ import { accountPrefix, accountSlug, discoverAccounts } from './accounts';
|
|
|
6
6
|
import { startAccountRelay } from './account-relay';
|
|
7
7
|
import { proxyKey } from './secrets';
|
|
8
8
|
import { pinnedBinary, pinnedPath } from './toolchain';
|
|
9
|
+
import { AccountSelector } from '../accounts/selection';
|
|
10
|
+
import { fleet } from '../fleet-config';
|
|
11
|
+
import type { AccountSelection } from '../definition';
|
|
9
12
|
|
|
10
13
|
/** `account` (its email) pins a proxy launch to one signed-in proxy account. */
|
|
11
14
|
export async function launchProfile(
|
|
@@ -43,9 +46,11 @@ export async function launchProfile(
|
|
|
43
46
|
}
|
|
44
47
|
toolEnvironment(env, selection, directory);
|
|
45
48
|
const pinned =
|
|
46
|
-
|
|
47
|
-
?
|
|
48
|
-
:
|
|
49
|
+
selection.profile !== 'proxy'
|
|
50
|
+
? null
|
|
51
|
+
: account
|
|
52
|
+
? await pinAccount(env, selection, { account, directory })
|
|
53
|
+
: await routeThroughRelay(env, selection);
|
|
49
54
|
try {
|
|
50
55
|
return await spawnAgent(selection, directory, env, [
|
|
51
56
|
pinned?.args ?? [],
|
|
@@ -96,9 +101,14 @@ export async function pinAccount(
|
|
|
96
101
|
): Promise<{ args: string[]; stop: () => void }> {
|
|
97
102
|
const prefix = accountPrefix(selection.tool, account);
|
|
98
103
|
// What the account serves right now, preferred default first.
|
|
99
|
-
const
|
|
104
|
+
const roster = await discover(env['CLIPROXY_API_KEY'] ?? '');
|
|
105
|
+
const served = roster.find(
|
|
100
106
|
(entry) => entry.tool === selection.tool && entry.email === account
|
|
101
107
|
);
|
|
108
|
+
// Named in this account's "cooling down" error, so the person can switch.
|
|
109
|
+
const alternatives = roster
|
|
110
|
+
.filter((entry) => entry.tool === selection.tool && entry.email !== account)
|
|
111
|
+
.map((entry) => entry.email);
|
|
102
112
|
if (!served)
|
|
103
113
|
throw new AgentConfigurationError(
|
|
104
114
|
'This account is not signed in on the CLI proxy.'
|
|
@@ -108,7 +118,9 @@ export async function pinAccount(
|
|
|
108
118
|
env['OPENCODE_CONFIG_CONTENT'] = JSON.stringify(accountOpenCode(served));
|
|
109
119
|
return { args: [], stop: () => {} };
|
|
110
120
|
}
|
|
111
|
-
const { url, stop } = relay(registry.proxy.endpoint, prefix
|
|
121
|
+
const { url, stop } = relay(registry.proxy.endpoint, prefix, {
|
|
122
|
+
alternatives,
|
|
123
|
+
});
|
|
112
124
|
if (selection.tool === 'claude') {
|
|
113
125
|
env['ANTHROPIC_BASE_URL'] = url;
|
|
114
126
|
// Aliases and the default must name models a Claude account serves.
|
|
@@ -146,6 +158,73 @@ export async function pinAccount(
|
|
|
146
158
|
};
|
|
147
159
|
}
|
|
148
160
|
|
|
161
|
+
/** Which tools reach the proxy through an address this wrapper sets, and so
|
|
162
|
+
* can be sent through a relay. OpenCode and Grok read theirs from rendered
|
|
163
|
+
* configuration and still go to the proxy directly.
|
|
164
|
+
*/
|
|
165
|
+
const RELAYED = new Set(['claude', 'codex', 'gemini', 'antigravity']);
|
|
166
|
+
|
|
167
|
+
/** An unpinned launch goes through a relay as well: it retries a proxy that
|
|
168
|
+
* is restarting exactly as a pinned one does, reports "proxy unreachable"
|
|
169
|
+
* when it stays down, and picks each request's account by the fleet's
|
|
170
|
+
* selection policy when it declares one for the provider.
|
|
171
|
+
*/
|
|
172
|
+
export async function routeThroughRelay(
|
|
173
|
+
env: NodeJS.ProcessEnv,
|
|
174
|
+
selection: Selection,
|
|
175
|
+
{
|
|
176
|
+
relay = startAccountRelay,
|
|
177
|
+
discover = discoverAccounts,
|
|
178
|
+
policy = selectionPolicy,
|
|
179
|
+
} = {}
|
|
180
|
+
): Promise<{ args: string[]; stop: () => void } | null> {
|
|
181
|
+
if (!RELAYED.has(selection.tool)) return null;
|
|
182
|
+
const selector = await policySelector(env, selection, discover, policy);
|
|
183
|
+
const { url, stop } = relay(
|
|
184
|
+
registry.proxy.endpoint,
|
|
185
|
+
'',
|
|
186
|
+
selector ? { selector } : {}
|
|
187
|
+
);
|
|
188
|
+
if (selection.tool === 'claude') env['ANTHROPIC_BASE_URL'] = url;
|
|
189
|
+
if (selection.tool === 'gemini' || selection.tool === 'antigravity')
|
|
190
|
+
env['GOOGLE_GEMINI_BASE_URL'] = url;
|
|
191
|
+
const args =
|
|
192
|
+
selection.tool === 'codex'
|
|
193
|
+
? [
|
|
194
|
+
'-c',
|
|
195
|
+
`model_providers.personal_proxy.base_url=${JSON.stringify(`${url}/v1`)}`,
|
|
196
|
+
]
|
|
197
|
+
: [];
|
|
198
|
+
return { args, stop };
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** The fleet's selection policy for the tool's provider, over the accounts
|
|
202
|
+
* the proxy lists. The list is read only when a policy needs it; an
|
|
203
|
+
* unreadable list leaves the choice to the proxy's own pool.
|
|
204
|
+
*/
|
|
205
|
+
async function policySelector(
|
|
206
|
+
env: NodeJS.ProcessEnv,
|
|
207
|
+
selection: Selection,
|
|
208
|
+
discover: typeof discoverAccounts,
|
|
209
|
+
policy: (provider: string) => AccountSelection | undefined
|
|
210
|
+
): Promise<AccountSelector | undefined> {
|
|
211
|
+
const route = (
|
|
212
|
+
registry.proxy.accounts as Record<string, { provider: string }>
|
|
213
|
+
)[selection.tool];
|
|
214
|
+
const declared = route ? policy(route.provider) : undefined;
|
|
215
|
+
if (!declared) return undefined;
|
|
216
|
+
const accounts = await discover(env['CLIPROXY_API_KEY'] ?? '').catch(
|
|
217
|
+
() => []
|
|
218
|
+
);
|
|
219
|
+
const prefixes = accounts
|
|
220
|
+
.filter((account) => account.tool === selection.tool)
|
|
221
|
+
.map((account) => account.prefix);
|
|
222
|
+
return prefixes.length ? new AccountSelector(prefixes, declared) : undefined;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
const selectionPolicy = (provider: string): AccountSelection | undefined =>
|
|
226
|
+
fleet.services.proxy.accounts?.selection?.[provider];
|
|
227
|
+
|
|
149
228
|
/** OpenCode configuration that offers only one account's models. */
|
|
150
229
|
function accountOpenCode(route: { prefix: string; models: string[] }) {
|
|
151
230
|
const models = route.models.map((model) => `${route.prefix}/${model}`);
|