@maxiaochao/pi-toolkit 0.11.7 → 0.11.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +1 -0
- package/package.json +5 -3
- package/packages/openai-account-pool/CHANGELOG.md +11 -0
- package/packages/openai-account-pool/README.md +67 -0
- package/packages/openai-account-pool/TESTING.md +28 -0
- package/packages/openai-account-pool/docs/demo.gif +0 -0
- package/packages/openai-account-pool/docs/screenshot.png +0 -0
- package/packages/openai-account-pool/extensions/openai-account-pool.ts +368 -0
- package/packages/openai-account-pool/package.json +45 -0
- package/packages/openai-account-pool/src/auth.ts +418 -0
- package/packages/openai-account-pool/src/availability.ts +230 -0
- package/packages/openai-account-pool/src/command-completions.ts +18 -0
- package/packages/openai-account-pool/src/enrollment.ts +339 -0
- package/packages/openai-account-pool/src/lease.ts +128 -0
- package/packages/openai-account-pool/src/lock.ts +171 -0
- package/packages/openai-account-pool/src/login-slot.ts +158 -0
- package/packages/openai-account-pool/src/policy.ts +176 -0
- package/packages/openai-account-pool/src/provider.ts +511 -0
- package/packages/openai-account-pool/src/status-widget.ts +62 -0
- package/packages/openai-account-pool/src/store.ts +259 -0
- package/packages/openai-account-pool/src/usage.ts +291 -0
- package/packages/openai-account-pool/tests/command-completions.mjs +10 -0
- package/packages/openai-account-pool/tests/enrollment.mjs +322 -0
- package/packages/openai-account-pool/tests/failover-stream.mjs +554 -0
- package/packages/openai-account-pool/tests/helpers/fixtures.mjs +40 -0
- package/packages/openai-account-pool/tests/helpers/native-credential-store.mjs +58 -0
- package/packages/openai-account-pool/tests/helpers/pi-runtime.mjs +24 -0
- package/packages/openai-account-pool/tests/helpers/refresh-child.mjs +29 -0
- package/packages/openai-account-pool/tests/integration-login-ownership.mjs +299 -0
- package/packages/openai-account-pool/tests/login-slot.mjs +131 -0
- package/packages/openai-account-pool/tests/multiprocess.mjs +73 -0
- package/packages/openai-account-pool/tests/selection.mjs +286 -0
- package/packages/openai-account-pool/tests/store-ownership.mjs +369 -0
- package/packages/openai-account-pool/tests/usage.mjs +428 -0
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
availabilityFor,
|
|
5
|
+
claimProbeLease,
|
|
6
|
+
DEFAULT_PROBE_BACKOFF_MS,
|
|
7
|
+
readAvailability,
|
|
8
|
+
recordFailure,
|
|
9
|
+
recordSuccess,
|
|
10
|
+
releaseProbeLease,
|
|
11
|
+
} from "../src/availability.ts";
|
|
12
|
+
import { ACCOUNT_SCOPE, classifyFailure, decideFailover, selectAccount } from "../src/policy.ts";
|
|
13
|
+
import { poolEntryKey, poolStatePathFor, readPoolState } from "../src/store.ts";
|
|
14
|
+
import { makeAgentDir } from "./helpers/fixtures.mjs";
|
|
15
|
+
|
|
16
|
+
const NOW = Date.UTC(2026, 0, 2, 3, 4, 5);
|
|
17
|
+
const account = (id, { ordinal = 1, generation = 1 } = {}) => ({
|
|
18
|
+
id,
|
|
19
|
+
key: poolEntryKey(id),
|
|
20
|
+
ordinal,
|
|
21
|
+
addedAt: NOW - 1_000,
|
|
22
|
+
generation,
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
// Classification reads structured evidence, never the friendly error text.
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
{
|
|
29
|
+
const planLimit = classifyFailure(
|
|
30
|
+
{ status: 429, modelId: "gpt-5-codex", body: { type: "usage_limit_reached", plan_type: "plus", resets_at: (NOW + 3_600_000) / 1_000 } },
|
|
31
|
+
NOW,
|
|
32
|
+
);
|
|
33
|
+
assert.equal(planLimit.kind, "quota");
|
|
34
|
+
assert.equal(planLimit.scope, ACCOUNT_SCOPE, "a plan-level limit applies to every model on the account");
|
|
35
|
+
assert.equal(planLimit.serverResetAtMs, NOW + 3_600_000);
|
|
36
|
+
|
|
37
|
+
const modelLimit = classifyFailure({ status: 429, modelId: "gpt-5-codex", body: { code: "usage_limit_reached" } }, NOW);
|
|
38
|
+
assert.equal(modelLimit.kind, "quota");
|
|
39
|
+
assert.equal(modelLimit.scope, "model:gpt-5-codex", "without plan evidence the limit stays scoped to the model");
|
|
40
|
+
assert.equal(modelLimit.serverResetAtMs, null);
|
|
41
|
+
|
|
42
|
+
// pi formats a plain 429 with the same "ChatGPT usage limit" wording as a real subscription
|
|
43
|
+
// limit. Switching accounts on this would burn the pool on ordinary rate limiting.
|
|
44
|
+
const plain429 = classifyFailure(
|
|
45
|
+
{ status: 429, modelId: "gpt-5-codex", body: { message: "You have hit your ChatGPT usage limit. Try again later." } },
|
|
46
|
+
NOW,
|
|
47
|
+
);
|
|
48
|
+
assert.equal(plain429.kind, "transient");
|
|
49
|
+
|
|
50
|
+
const cases = [
|
|
51
|
+
[{ status: 429, body: { code: "rate_limit_exceeded" } }, "transient"],
|
|
52
|
+
[{ status: 403, body: { code: "usage_not_included" } }, "fatal"],
|
|
53
|
+
[{ status: 401 }, "auth_retry"],
|
|
54
|
+
[{ status: 403 }, "fatal"],
|
|
55
|
+
[{ status: 400 }, "fatal"],
|
|
56
|
+
[{ status: 500 }, "transient"],
|
|
57
|
+
[{ status: 503 }, "transient"],
|
|
58
|
+
[{ networkFailure: true }, "transient"],
|
|
59
|
+
[{ oauthRefresh: { invalidGrant: true } }, "auth_invalid"],
|
|
60
|
+
[{ oauthRefresh: { invalidGrant: false } }, "transient"],
|
|
61
|
+
[{ status: 401, body: { code: "invalid_grant" } }, "auth_invalid"],
|
|
62
|
+
[{ cancelled: true, status: 429, body: { code: "usage_limit_reached" } }, "cancelled"],
|
|
63
|
+
[{}, "unknown"],
|
|
64
|
+
];
|
|
65
|
+
for (const [evidence, expected] of cases) {
|
|
66
|
+
assert.equal(classifyFailure(evidence, NOW).kind, expected, `evidence ${JSON.stringify(evidence)}`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Only plausible reset times are trusted; anything else means "unknown, probe later".
|
|
70
|
+
const resets = [
|
|
71
|
+
[{ resets_at: 0 }, null],
|
|
72
|
+
[{ resets_at: -1 }, null],
|
|
73
|
+
[{ resets_at: 1e18 }, null],
|
|
74
|
+
[{ resets_at: "soon" }, null],
|
|
75
|
+
[{ resets_in_seconds: 3_600 }, NOW + 3_600_000],
|
|
76
|
+
[{ resets_in_seconds: -5 }, null],
|
|
77
|
+
[{ resets_in_seconds: 1e9 }, null],
|
|
78
|
+
];
|
|
79
|
+
for (const [extra, expected] of resets) {
|
|
80
|
+
const classified = classifyFailure({ body: { code: "usage_limit_reached", plan_type: "pro", ...extra } }, NOW);
|
|
81
|
+
assert.equal(classified.serverResetAtMs, expected, `reset evidence ${JSON.stringify(extra)}`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ---------------------------------------------------------------------------
|
|
86
|
+
// Selection is sticky, then falls back in the order accounts were added.
|
|
87
|
+
// ---------------------------------------------------------------------------
|
|
88
|
+
{
|
|
89
|
+
const usable = (id, ordinal) => ({ id, ordinal, needsProbe: false, authInvalid: false });
|
|
90
|
+
const blocked = (id, ordinal) => ({ ...usable(id, ordinal), unavailable: { reason: "usage_limit_reached", untilMs: NOW + 60_000, serverResetAtMs: null } });
|
|
91
|
+
|
|
92
|
+
const accounts = [usable("a", 1), usable("b", 2), usable("c", 3)];
|
|
93
|
+
assert.equal(selectAccount({ accounts, preferredId: "c", attemptedIds: [] })?.id, "c", "keep using the account that works");
|
|
94
|
+
assert.equal(selectAccount({ accounts, attemptedIds: [] })?.id, "a", "no preference means the first added account");
|
|
95
|
+
assert.equal(selectAccount({ accounts, preferredId: "c", attemptedIds: ["c"] })?.id, "a", "an attempted account is never retried");
|
|
96
|
+
assert.equal(
|
|
97
|
+
selectAccount({ accounts: [blocked("a", 1), usable("b", 2)], preferredId: "a", attemptedIds: [] })?.id,
|
|
98
|
+
"b",
|
|
99
|
+
"a cooled-down preference yields to the next usable account",
|
|
100
|
+
);
|
|
101
|
+
assert.equal(
|
|
102
|
+
selectAccount({ accounts: [{ ...usable("a", 1), authInvalid: true }, usable("b", 2)], attemptedIds: [] })?.id,
|
|
103
|
+
"b",
|
|
104
|
+
"an account needing re-authorization is skipped",
|
|
105
|
+
);
|
|
106
|
+
assert.equal(selectAccount({ accounts: [blocked("a", 1), blocked("b", 2)], attemptedIds: [] }), undefined);
|
|
107
|
+
assert.equal(selectAccount({ accounts, attemptedIds: ["a", "b", "c"] }), undefined);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// ---------------------------------------------------------------------------
|
|
111
|
+
// Failover is bounded: only proven quota or authorization failures switch accounts,
|
|
112
|
+
// and never after output already reached the caller.
|
|
113
|
+
// ---------------------------------------------------------------------------
|
|
114
|
+
{
|
|
115
|
+
const quota = classifyFailure({ body: { code: "usage_limit_reached", plan_type: "plus" } }, NOW);
|
|
116
|
+
const base = { emittedOutput: false, hasRemainingCandidate: true, repairUsed: false };
|
|
117
|
+
|
|
118
|
+
assert.deepEqual(decideFailover({ ...base, classification: quota }), { action: "failover", reason: "usage_limit_reached" });
|
|
119
|
+
assert.equal(decideFailover({ ...base, classification: quota, emittedOutput: true }).action, "stop");
|
|
120
|
+
assert.equal(decideFailover({ ...base, classification: quota, emittedOutput: true }).reason, "output_already_delivered");
|
|
121
|
+
assert.equal(decideFailover({ ...base, classification: quota, hasRemainingCandidate: false }).reason, "pool_exhausted");
|
|
122
|
+
|
|
123
|
+
const unauthorized = classifyFailure({ status: 401 }, NOW);
|
|
124
|
+
assert.equal(decideFailover({ ...base, classification: unauthorized }).action, "repair", "first 401 retries the same account");
|
|
125
|
+
assert.equal(
|
|
126
|
+
decideFailover({ ...base, classification: unauthorized, repairUsed: true }).action,
|
|
127
|
+
"failover",
|
|
128
|
+
"a 401 that survives one repair means this account is out",
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
for (const evidence of [{ status: 500 }, { status: 403 }, {}, { cancelled: true }]) {
|
|
132
|
+
assert.equal(decideFailover({ ...base, classification: classifyFailure(evidence, NOW) }).action, "stop", JSON.stringify(evidence));
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// ---------------------------------------------------------------------------
|
|
137
|
+
// Shared state: cooldowns, generations, probe leases.
|
|
138
|
+
// ---------------------------------------------------------------------------
|
|
139
|
+
{
|
|
140
|
+
const agent = makeAgentDir();
|
|
141
|
+
const statePath = poolStatePathFor(agent.directory);
|
|
142
|
+
try {
|
|
143
|
+
const first = account("account-aaaa", { ordinal: 1 });
|
|
144
|
+
const second = account("account-bbbb", { ordinal: 2 });
|
|
145
|
+
const accounts = [first, second];
|
|
146
|
+
|
|
147
|
+
// A limit with a server-stated reset blocks exactly until that time.
|
|
148
|
+
await recordFailure({
|
|
149
|
+
statePath,
|
|
150
|
+
account: first,
|
|
151
|
+
now: NOW,
|
|
152
|
+
classification: classifyFailure({ body: { code: "usage_limit_reached", plan_type: "plus", resets_at: (NOW + 600_000) / 1_000 } }, NOW),
|
|
153
|
+
});
|
|
154
|
+
let view = await readAvailability({ statePath, accounts, now: NOW + 1_000 });
|
|
155
|
+
assert.equal(view[0].unavailable?.reason, "usage_limit_reached");
|
|
156
|
+
assert.equal(view[0].unavailable?.untilMs, NOW + 600_000);
|
|
157
|
+
assert.equal(view[1].unavailable, undefined, "one account's limit says nothing about another");
|
|
158
|
+
assert.equal(selectAccount({ accounts: view, preferredId: first.id, attemptedIds: [] })?.id, second.id);
|
|
159
|
+
|
|
160
|
+
// Past the reset the account is a candidate again, flagged for a probe.
|
|
161
|
+
view = await readAvailability({ statePath, accounts, now: NOW + 600_001 });
|
|
162
|
+
assert.equal(view[0].unavailable, undefined);
|
|
163
|
+
assert.equal(view[0].needsProbe, true);
|
|
164
|
+
|
|
165
|
+
// A success from a request that started before the limit must not clear it.
|
|
166
|
+
await recordSuccess({ statePath, account: first, requestStartedAtMs: NOW - 5_000 });
|
|
167
|
+
view = await readAvailability({ statePath, accounts, now: NOW + 1_000 });
|
|
168
|
+
assert.equal(view[0].unavailable?.reason, "usage_limit_reached", "an older request's success is stale evidence");
|
|
169
|
+
|
|
170
|
+
// A success from a later request does clear it.
|
|
171
|
+
await recordSuccess({ statePath, account: first, requestStartedAtMs: NOW + 1 });
|
|
172
|
+
view = await readAvailability({ statePath, accounts, now: NOW + 1_000 });
|
|
173
|
+
assert.equal(view[0].unavailable, undefined);
|
|
174
|
+
assert.equal(view[0].needsProbe, false, "cleared evidence leaves no probe obligation");
|
|
175
|
+
}
|
|
176
|
+
finally {
|
|
177
|
+
agent.dispose();
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Re-authorizing an account retires every record written for the older generation.
|
|
182
|
+
{
|
|
183
|
+
const agent = makeAgentDir();
|
|
184
|
+
const statePath = poolStatePathFor(agent.directory);
|
|
185
|
+
try {
|
|
186
|
+
const before = account("account-aaaa", { generation: 3 });
|
|
187
|
+
await recordFailure({ statePath, account: before, now: NOW, classification: classifyFailure({ oauthRefresh: { invalidGrant: true } }, NOW) });
|
|
188
|
+
await recordFailure({ statePath, account: before, now: NOW, classification: classifyFailure({ body: { code: "usage_limit_reached", plan_type: "plus" } }, NOW) });
|
|
189
|
+
|
|
190
|
+
let view = await readAvailability({ statePath, accounts: [before], now: NOW + 1_000 });
|
|
191
|
+
assert.equal(view[0].authInvalid, true);
|
|
192
|
+
assert.equal(view[0].unavailable?.untilMs, NOW + DEFAULT_PROBE_BACKOFF_MS, "an unknown reset falls back to a short local backoff");
|
|
193
|
+
|
|
194
|
+
const after = account("account-aaaa", { generation: 4 });
|
|
195
|
+
view = await readAvailability({ statePath, accounts: [after], now: NOW + 1_000 });
|
|
196
|
+
assert.equal(view[0].authInvalid, false, "a fresh authorization clears the invalid mark");
|
|
197
|
+
assert.equal(view[0].unavailable, undefined, "evidence from the previous authorization no longer applies");
|
|
198
|
+
|
|
199
|
+
// Evidence arriving late from the old generation must not overwrite the newer record.
|
|
200
|
+
await recordFailure({ statePath, account: after, now: NOW + 2_000, classification: classifyFailure({ body: { code: "usage_limit_reached", plan_type: "plus" } }, NOW + 2_000) });
|
|
201
|
+
await recordFailure({ statePath, account: before, now: NOW + 3_000, classification: classifyFailure({ body: { code: "usage_limit_reached", plan_type: "plus" } }, NOW + 3_000) });
|
|
202
|
+
const state = await readPoolState(statePath);
|
|
203
|
+
assert.equal(state.accounts["account-aaaa"].cooldowns[ACCOUNT_SCOPE].generation, 4, "stale evidence never outranks a newer login");
|
|
204
|
+
}
|
|
205
|
+
finally {
|
|
206
|
+
agent.dispose();
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// A model-scoped limit leaves other models on the same account usable.
|
|
211
|
+
{
|
|
212
|
+
const agent = makeAgentDir();
|
|
213
|
+
const statePath = poolStatePathFor(agent.directory);
|
|
214
|
+
try {
|
|
215
|
+
const only = account("account-aaaa");
|
|
216
|
+
await recordFailure({ statePath, account: only, now: NOW, classification: classifyFailure({ modelId: "gpt-5-codex", body: { code: "usage_limit_reached" } }, NOW) });
|
|
217
|
+
const blockedModel = await readAvailability({ statePath, accounts: [only], now: NOW + 1_000, modelId: "gpt-5-codex" });
|
|
218
|
+
const otherModel = await readAvailability({ statePath, accounts: [only], now: NOW + 1_000, modelId: "gpt-5-mini" });
|
|
219
|
+
assert.equal(blockedModel[0].unavailable?.reason, "usage_limit_reached");
|
|
220
|
+
assert.equal(otherModel[0].unavailable, undefined);
|
|
221
|
+
}
|
|
222
|
+
finally {
|
|
223
|
+
agent.dispose();
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// Only one process probes a recovered account; the others keep using what works.
|
|
228
|
+
{
|
|
229
|
+
const agent = makeAgentDir();
|
|
230
|
+
const statePath = poolStatePathFor(agent.directory);
|
|
231
|
+
try {
|
|
232
|
+
const recovered = account("account-aaaa", { ordinal: 1 });
|
|
233
|
+
const healthy = account("account-bbbb", { ordinal: 2 });
|
|
234
|
+
const accounts = [recovered, healthy];
|
|
235
|
+
await recordFailure({ statePath, account: recovered, now: NOW, classification: classifyFailure({ body: { code: "usage_limit_reached", plan_type: "plus" } }, NOW) });
|
|
236
|
+
|
|
237
|
+
const probeTime = NOW + DEFAULT_PROBE_BACKOFF_MS + 1;
|
|
238
|
+
assert.equal(await claimProbeLease({ statePath, accountId: recovered.id, owner: "process-a", now: probeTime, ttlMs: 30_000 }), true);
|
|
239
|
+
assert.equal(
|
|
240
|
+
await claimProbeLease({ statePath, accountId: recovered.id, owner: "process-b", now: probeTime, ttlMs: 30_000 }),
|
|
241
|
+
false,
|
|
242
|
+
"a live probe lease is exclusive",
|
|
243
|
+
);
|
|
244
|
+
|
|
245
|
+
const state = await readPoolState(statePath);
|
|
246
|
+
const forOwner = availabilityFor({ accounts, state, now: probeTime, probeOwner: "process-a" });
|
|
247
|
+
const forOther = availabilityFor({ accounts, state, now: probeTime, probeOwner: "process-b" });
|
|
248
|
+
assert.equal(forOwner[0].unavailable, undefined, "the lease holder may probe");
|
|
249
|
+
assert.equal(forOther[0].unavailable?.reason, "probe_in_progress");
|
|
250
|
+
assert.equal(selectAccount({ accounts: forOther, attemptedIds: [] })?.id, healthy.id);
|
|
251
|
+
|
|
252
|
+
// An abandoned lease expires instead of blocking recovery forever.
|
|
253
|
+
assert.equal(await claimProbeLease({ statePath, accountId: recovered.id, owner: "process-b", now: probeTime + 30_001 }), true);
|
|
254
|
+
|
|
255
|
+
await releaseProbeLease({ statePath, accountId: recovered.id, owner: "process-a" });
|
|
256
|
+
assert.ok((await readPoolState(statePath)).accounts[recovered.id].probeLease, "only the owner may release the lease");
|
|
257
|
+
await releaseProbeLease({ statePath, accountId: recovered.id, owner: "process-b" });
|
|
258
|
+
assert.equal((await readPoolState(statePath)).accounts[recovered.id].probeLease, undefined);
|
|
259
|
+
|
|
260
|
+
// Recording a fresh limit drops the lease with it.
|
|
261
|
+
await claimProbeLease({ statePath, accountId: recovered.id, owner: "process-b", now: probeTime });
|
|
262
|
+
await recordFailure({ statePath, account: recovered, now: probeTime, classification: classifyFailure({ body: { code: "usage_limit_reached", plan_type: "plus" } }, probeTime) });
|
|
263
|
+
assert.equal((await readPoolState(statePath)).accounts[recovered.id].probeLease, undefined);
|
|
264
|
+
}
|
|
265
|
+
finally {
|
|
266
|
+
agent.dispose();
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// Transient and fatal failures leave no shared state behind.
|
|
271
|
+
{
|
|
272
|
+
const agent = makeAgentDir();
|
|
273
|
+
const statePath = poolStatePathFor(agent.directory);
|
|
274
|
+
try {
|
|
275
|
+
const only = account("account-aaaa");
|
|
276
|
+
for (const evidence of [{ status: 429 }, { status: 500 }, { status: 403 }, { cancelled: true }, {}]) {
|
|
277
|
+
await recordFailure({ statePath, account: only, now: NOW, classification: classifyFailure(evidence, NOW) });
|
|
278
|
+
}
|
|
279
|
+
assert.deepEqual((await readPoolState(statePath)).accounts, {}, "only proven limits and authorization failures are shared");
|
|
280
|
+
}
|
|
281
|
+
finally {
|
|
282
|
+
agent.dispose();
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
console.log("Selection policy tests passed");
|
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { mkdirSync, readFileSync, rmdirSync, symlinkSync, utimesSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
accountCredential,
|
|
7
|
+
commitAccount,
|
|
8
|
+
disablePool,
|
|
9
|
+
ensureFreshAccountCredential,
|
|
10
|
+
ExistingLoginNotIdentifiableError,
|
|
11
|
+
isPoolReferenceEntry,
|
|
12
|
+
loadPoolSnapshot,
|
|
13
|
+
PoolAccountMissingError,
|
|
14
|
+
PoolNotEnabledError,
|
|
15
|
+
removeAccount,
|
|
16
|
+
} from "../src/auth.ts";
|
|
17
|
+
import {
|
|
18
|
+
acquireEnrollmentLease,
|
|
19
|
+
assertNoActiveEnrollment,
|
|
20
|
+
EnrollmentInProgressError,
|
|
21
|
+
EnrollmentLeaseLostError,
|
|
22
|
+
withEnrollmentLease,
|
|
23
|
+
} from "../src/lease.ts";
|
|
24
|
+
import { acquireFileLock, lockDirectoryFor } from "../src/lock.ts";
|
|
25
|
+
import { poolEntryKey, poolStatePathFor, readPoolState, withAuthTransaction } from "../src/store.ts";
|
|
26
|
+
import { fakeAccessToken, fakeCredential, makeAgentDir, seedAuthFile } from "./helpers/fixtures.mjs";
|
|
27
|
+
import { NativeCredentialStore } from "./helpers/native-credential-store.mjs";
|
|
28
|
+
|
|
29
|
+
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
30
|
+
const readEntries = (authPath) => JSON.parse(readFileSync(authPath, "utf8"));
|
|
31
|
+
const storedCredential = (entries, accountId) => {
|
|
32
|
+
const entry = entries[poolEntryKey(accountId)];
|
|
33
|
+
return entry === undefined ? undefined : accountCredential(entry);
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
const UNRELATED = {
|
|
37
|
+
anthropic: { type: "api_key", key: "$ANTHROPIC_API_KEY", env: { AWS_REGION: "us-east-1" } },
|
|
38
|
+
"custom-gateway": { type: "oauth", access: "gateway-access", refresh: "gateway-refresh", expires: 7, vendorField: { nested: true } },
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
// Unrelated provider entries survive every ownership change.
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
{
|
|
45
|
+
const agent = makeAgentDir();
|
|
46
|
+
try {
|
|
47
|
+
seedAuthFile(agent.authPath, { ...UNRELATED, "openai-codex": fakeCredential("account-aaaa") });
|
|
48
|
+
await commitAccount({ authPath: agent.authPath, credential: fakeCredential("account-bbbb") });
|
|
49
|
+
await commitAccount({ authPath: agent.authPath, credential: fakeCredential("account-cccc") });
|
|
50
|
+
await removeAccount({ authPath: agent.authPath, accountId: "account-cccc" });
|
|
51
|
+
await disablePool({ authPath: agent.authPath, accountId: "account-bbbb" });
|
|
52
|
+
|
|
53
|
+
const entries = readEntries(agent.authPath);
|
|
54
|
+
for (const [providerId, entry] of Object.entries(UNRELATED)) {
|
|
55
|
+
assert.deepEqual(entries[providerId], entry, `${providerId} must be carried through untouched`);
|
|
56
|
+
}
|
|
57
|
+
} finally {
|
|
58
|
+
agent.dispose();
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// ---------------------------------------------------------------------------
|
|
63
|
+
// Pool transactions and pi's own credential writes share one lock.
|
|
64
|
+
// ---------------------------------------------------------------------------
|
|
65
|
+
{
|
|
66
|
+
const agent = makeAgentDir();
|
|
67
|
+
try {
|
|
68
|
+
seedAuthFile(agent.authPath, { ...UNRELATED });
|
|
69
|
+
const store = new NativeCredentialStore(agent.authPath);
|
|
70
|
+
const lock = await acquireFileLock(agent.authPath);
|
|
71
|
+
let nativeWriteFinished = false;
|
|
72
|
+
const pending = store.modify("anthropic", async () => ({ type: "api_key", key: "rewritten" })).then(() => { nativeWriteFinished = true; });
|
|
73
|
+
await delay(120);
|
|
74
|
+
assert.equal(nativeWriteFinished, false, "pi's store must wait while the pool holds the lock");
|
|
75
|
+
await lock.release();
|
|
76
|
+
await pending;
|
|
77
|
+
assert.equal(readEntries(agent.authPath).anthropic.key, "rewritten");
|
|
78
|
+
|
|
79
|
+
// And the other way round: a transaction waits for a lock held elsewhere.
|
|
80
|
+
const other = await acquireFileLock(agent.authPath);
|
|
81
|
+
let transactionFinished = false;
|
|
82
|
+
const transaction = withAuthTransaction(agent.authPath, (entries) => ({ value: { ...entries, probe: { type: "api_key", key: "x" } }, result: undefined }))
|
|
83
|
+
.then(() => { transactionFinished = true; });
|
|
84
|
+
await delay(120);
|
|
85
|
+
assert.equal(transactionFinished, false);
|
|
86
|
+
await other.release();
|
|
87
|
+
await transaction;
|
|
88
|
+
assert.equal(readEntries(agent.authPath).probe.key, "x");
|
|
89
|
+
} finally {
|
|
90
|
+
agent.dispose();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
// A waiting lock acquisition is cancellable; a stale lock is reclaimed.
|
|
96
|
+
// ---------------------------------------------------------------------------
|
|
97
|
+
{
|
|
98
|
+
const agent = makeAgentDir();
|
|
99
|
+
try {
|
|
100
|
+
seedAuthFile(agent.authPath, {});
|
|
101
|
+
const held = await acquireFileLock(agent.authPath);
|
|
102
|
+
const controller = new AbortController();
|
|
103
|
+
const waiting = acquireFileLock(agent.authPath, { signal: controller.signal, waitMs: 10_000 });
|
|
104
|
+
setTimeout(() => controller.abort(new Error("cancelled by test")), 40);
|
|
105
|
+
await assert.rejects(waiting, /cancelled by test/);
|
|
106
|
+
await held.release();
|
|
107
|
+
|
|
108
|
+
// A holder that stopped heart-beating is reclaimed instead of blocking forever.
|
|
109
|
+
const directory = lockDirectoryFor(agent.authPath);
|
|
110
|
+
mkdirSync(directory, { mode: 0o700 });
|
|
111
|
+
const longAgo = new Date(Date.now() - 120_000);
|
|
112
|
+
utimesSync(directory, longAgo, longAgo);
|
|
113
|
+
const reclaimed = await acquireFileLock(agent.authPath, { waitMs: 1_000 });
|
|
114
|
+
await reclaimed.release();
|
|
115
|
+
} finally {
|
|
116
|
+
agent.dispose();
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// ---------------------------------------------------------------------------
|
|
121
|
+
// A reclaimed lock refuses the write instead of corrupting the file.
|
|
122
|
+
// ---------------------------------------------------------------------------
|
|
123
|
+
{
|
|
124
|
+
const agent = makeAgentDir();
|
|
125
|
+
try {
|
|
126
|
+
seedAuthFile(agent.authPath, { ...UNRELATED });
|
|
127
|
+
const before = readFileSync(agent.authPath, "utf8");
|
|
128
|
+
await assert.rejects(
|
|
129
|
+
withAuthTransaction(agent.authPath, async (entries) => {
|
|
130
|
+
// Simulate another process reclaiming the lock directory mid-transaction.
|
|
131
|
+
const directory = lockDirectoryFor(agent.authPath);
|
|
132
|
+
rmdirSync(directory);
|
|
133
|
+
mkdirSync(directory, { mode: 0o700 });
|
|
134
|
+
// Detection happens at write time, so it does not depend on a heartbeat having fired.
|
|
135
|
+
return { value: { ...entries, injected: { type: "api_key", key: "must-not-be-written" } }, result: undefined };
|
|
136
|
+
}),
|
|
137
|
+
/Lost the lock/,
|
|
138
|
+
);
|
|
139
|
+
assert.equal(readFileSync(agent.authPath, "utf8"), before, "a compromised lock must not write");
|
|
140
|
+
rmdirSync(lockDirectoryFor(agent.authPath));
|
|
141
|
+
} finally {
|
|
142
|
+
agent.dispose();
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// ---------------------------------------------------------------------------
|
|
147
|
+
// Token rotation: refreshed once under the lock, persisted, membership kept.
|
|
148
|
+
// ---------------------------------------------------------------------------
|
|
149
|
+
{
|
|
150
|
+
const agent = makeAgentDir();
|
|
151
|
+
try {
|
|
152
|
+
seedAuthFile(agent.authPath, { ...UNRELATED });
|
|
153
|
+
const expiring = fakeCredential("account-aaaa", { expiresInMs: 1_000 });
|
|
154
|
+
const { account } = await commitAccount({ authPath: agent.authPath, credential: expiring });
|
|
155
|
+
|
|
156
|
+
let refreshCalls = 0;
|
|
157
|
+
const refresh = async (credential) => {
|
|
158
|
+
refreshCalls += 1;
|
|
159
|
+
assert.equal(credential.piOpenAIAccountPoolMembership, undefined, "the provider must see a plain credential");
|
|
160
|
+
await delay(40);
|
|
161
|
+
return { type: "oauth", access: fakeAccessToken("account-aaaa", "v2"), refresh: "rotated-refresh", expires: Date.now() + 3_600_000 };
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
const [first, second] = await Promise.all([
|
|
165
|
+
ensureFreshAccountCredential({ authPath: agent.authPath, account, refresh }),
|
|
166
|
+
ensureFreshAccountCredential({ authPath: agent.authPath, account, refresh }),
|
|
167
|
+
]);
|
|
168
|
+
assert.equal(refreshCalls, 1, "concurrent callers must exchange a rotating refresh token once");
|
|
169
|
+
assert.equal(first.credential.refresh, "rotated-refresh");
|
|
170
|
+
assert.equal(second.credential.refresh, "rotated-refresh");
|
|
171
|
+
assert.equal(first.credential.accountId, "account-aaaa", "identity survives a refresh that omits it");
|
|
172
|
+
|
|
173
|
+
const entries = readEntries(agent.authPath);
|
|
174
|
+
assert.equal(storedCredential(entries, "account-aaaa").refresh, "rotated-refresh");
|
|
175
|
+
assert.deepEqual(entries[account.key].piOpenAIAccountPoolMembership, { ordinal: account.ordinal, addedAt: account.addedAt });
|
|
176
|
+
|
|
177
|
+
// A still-valid credential is served without any refresh call.
|
|
178
|
+
const reused = await ensureFreshAccountCredential({
|
|
179
|
+
authPath: agent.authPath,
|
|
180
|
+
account,
|
|
181
|
+
refresh: async () => { throw new Error("must not refresh a valid credential"); },
|
|
182
|
+
});
|
|
183
|
+
assert.equal(reused.refreshed, false);
|
|
184
|
+
} finally {
|
|
185
|
+
agent.dispose();
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// ---------------------------------------------------------------------------
|
|
190
|
+
// A cancelled caller never discards a refresh that already started.
|
|
191
|
+
// ---------------------------------------------------------------------------
|
|
192
|
+
{
|
|
193
|
+
const agent = makeAgentDir();
|
|
194
|
+
try {
|
|
195
|
+
seedAuthFile(agent.authPath, {});
|
|
196
|
+
const { account } = await commitAccount({ authPath: agent.authPath, credential: fakeCredential("account-aaaa", { expiresInMs: 1_000 }) });
|
|
197
|
+
const controller = new AbortController();
|
|
198
|
+
const result = await ensureFreshAccountCredential({
|
|
199
|
+
authPath: agent.authPath,
|
|
200
|
+
account,
|
|
201
|
+
signal: controller.signal,
|
|
202
|
+
refresh: async () => {
|
|
203
|
+
controller.abort(new Error("request cancelled"));
|
|
204
|
+
await delay(30);
|
|
205
|
+
return { type: "oauth", access: fakeAccessToken("account-aaaa", "v2"), refresh: "rotated-after-cancel", expires: Date.now() + 3_600_000 };
|
|
206
|
+
},
|
|
207
|
+
});
|
|
208
|
+
assert.equal(result.refreshed, true);
|
|
209
|
+
assert.equal(storedCredential(readEntries(agent.authPath), "account-aaaa").refresh, "rotated-after-cancel");
|
|
210
|
+
} finally {
|
|
211
|
+
agent.dispose();
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// ---------------------------------------------------------------------------
|
|
216
|
+
// Ownership after a plain login, a logout, and a later re-enable.
|
|
217
|
+
// ---------------------------------------------------------------------------
|
|
218
|
+
{
|
|
219
|
+
const agent = makeAgentDir();
|
|
220
|
+
try {
|
|
221
|
+
seedAuthFile(agent.authPath, { ...UNRELATED });
|
|
222
|
+
const credentialA = fakeCredential("account-aaaa");
|
|
223
|
+
const credentialB = fakeCredential("account-bbbb");
|
|
224
|
+
await commitAccount({ authPath: agent.authPath, credential: credentialA });
|
|
225
|
+
await commitAccount({ authPath: agent.authPath, credential: credentialB });
|
|
226
|
+
|
|
227
|
+
const store = new NativeCredentialStore(agent.authPath);
|
|
228
|
+
|
|
229
|
+
// A plain login replaces the reference: the pool stops, accounts stay stored.
|
|
230
|
+
const plain = fakeCredential("account-dddd");
|
|
231
|
+
await store.modify("openai-codex", async () => plain);
|
|
232
|
+
let snapshot = await loadPoolSnapshot(agent.authPath);
|
|
233
|
+
assert.equal(snapshot.enabled, false);
|
|
234
|
+
assert.equal(snapshot.nativeLoginPresent, true);
|
|
235
|
+
assert.equal(snapshot.loggedOut, false);
|
|
236
|
+
|
|
237
|
+
// A plain logout leaves nothing to authenticate with, and the pool must not step in.
|
|
238
|
+
await store.delete("openai-codex");
|
|
239
|
+
snapshot = await loadPoolSnapshot(agent.authPath);
|
|
240
|
+
assert.equal(snapshot.enabled, false);
|
|
241
|
+
assert.equal(snapshot.loggedOut, true);
|
|
242
|
+
assert.deepEqual(snapshot.accounts, []);
|
|
243
|
+
await assert.rejects(disablePool({ authPath: agent.authPath, accountId: "account-aaaa" }), PoolNotEnabledError);
|
|
244
|
+
|
|
245
|
+
// Adding an account again re-enables the stored accounts, and says how many came back.
|
|
246
|
+
const reenabled = await commitAccount({ authPath: agent.authPath, credential: fakeCredential("account-eeee") });
|
|
247
|
+
assert.equal(reenabled.readoptedAccounts, 2, "previously stored accounts are brought back explicitly");
|
|
248
|
+
assert.equal(reenabled.accountCount, 3);
|
|
249
|
+
snapshot = await loadPoolSnapshot(agent.authPath);
|
|
250
|
+
assert.deepEqual(snapshot.accounts.map((account) => account.ordinal), [1, 2, 3]);
|
|
251
|
+
assert.deepEqual(storedCredential(readEntries(agent.authPath), "account-aaaa"), credentialA);
|
|
252
|
+
} finally {
|
|
253
|
+
agent.dispose();
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// ---------------------------------------------------------------------------
|
|
258
|
+
// Generations only move forward, and unknown accounts fail visibly.
|
|
259
|
+
// ---------------------------------------------------------------------------
|
|
260
|
+
{
|
|
261
|
+
const agent = makeAgentDir();
|
|
262
|
+
try {
|
|
263
|
+
seedAuthFile(agent.authPath, {});
|
|
264
|
+
const first = await commitAccount({ authPath: agent.authPath, credential: fakeCredential("account-aaaa") });
|
|
265
|
+
const second = await commitAccount({ authPath: agent.authPath, credential: fakeCredential("account-bbbb") });
|
|
266
|
+
const third = await commitAccount({ authPath: agent.authPath, credential: fakeCredential("account-aaaa", { marker: "v2" }) });
|
|
267
|
+
assert.ok(first.generation < second.generation && second.generation < third.generation);
|
|
268
|
+
|
|
269
|
+
await assert.rejects(removeAccount({ authPath: agent.authPath, accountId: "account-zzzz" }), PoolAccountMissingError);
|
|
270
|
+
await assert.rejects(disablePool({ authPath: agent.authPath, accountId: "account-zzzz" }), PoolAccountMissingError);
|
|
271
|
+
} finally {
|
|
272
|
+
agent.dispose();
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// ---------------------------------------------------------------------------
|
|
277
|
+
// Unidentifiable and non-OAuth credentials are rejected instead of guessed.
|
|
278
|
+
// ---------------------------------------------------------------------------
|
|
279
|
+
{
|
|
280
|
+
const agent = makeAgentDir();
|
|
281
|
+
try {
|
|
282
|
+
seedAuthFile(agent.authPath, { "openai-codex": { type: "oauth", access: "opaque-token", refresh: "opaque-refresh", expires: Date.now() + 1_000 } });
|
|
283
|
+
await assert.rejects(
|
|
284
|
+
commitAccount({ authPath: agent.authPath, credential: fakeCredential("account-bbbb") }),
|
|
285
|
+
ExistingLoginNotIdentifiableError,
|
|
286
|
+
);
|
|
287
|
+
assert.equal(isPoolReferenceEntry(readEntries(agent.authPath)["openai-codex"]), false, "a refused commit changes nothing");
|
|
288
|
+
|
|
289
|
+
await assert.rejects(
|
|
290
|
+
commitAccount({ authPath: agent.authPath, credential: { type: "api_key", key: "not-oauth" } }),
|
|
291
|
+
/not an OAuth credential/,
|
|
292
|
+
);
|
|
293
|
+
} finally {
|
|
294
|
+
agent.dispose();
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// ---------------------------------------------------------------------------
|
|
299
|
+
// Corrupt files and unsafe paths fail visibly and are never repaired silently.
|
|
300
|
+
// ---------------------------------------------------------------------------
|
|
301
|
+
{
|
|
302
|
+
const agent = makeAgentDir();
|
|
303
|
+
try {
|
|
304
|
+
writeFileSync(agent.authPath, "{ not json", "utf8");
|
|
305
|
+
await assert.rejects(commitAccount({ authPath: agent.authPath, credential: fakeCredential("account-aaaa") }), SyntaxError);
|
|
306
|
+
assert.equal(readFileSync(agent.authPath, "utf8"), "{ not json");
|
|
307
|
+
|
|
308
|
+
writeFileSync(agent.authPath, JSON.stringify({ "openai-codex": "not-an-object" }), "utf8");
|
|
309
|
+
await assert.rejects(loadPoolSnapshot(agent.authPath), /is not an object/);
|
|
310
|
+
|
|
311
|
+
const statePath = poolStatePathFor(agent.directory);
|
|
312
|
+
mkdirSync(join(agent.directory, "openai-account-pool"), { recursive: true });
|
|
313
|
+
writeFileSync(statePath, JSON.stringify({ schemaVersion: 99, policy: "sticky", accounts: {} }), "utf8");
|
|
314
|
+
await assert.rejects(readPoolState(statePath), /schemaVersion 99/);
|
|
315
|
+
} finally {
|
|
316
|
+
agent.dispose();
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
{
|
|
320
|
+
const agent = makeAgentDir();
|
|
321
|
+
try {
|
|
322
|
+
const real = join(agent.directory, "elsewhere.json");
|
|
323
|
+
writeFileSync(real, "{}", "utf8");
|
|
324
|
+
symlinkSync(real, agent.authPath);
|
|
325
|
+
await assert.rejects(commitAccount({ authPath: agent.authPath, credential: fakeCredential("account-aaaa") }), /not a regular file/);
|
|
326
|
+
} finally {
|
|
327
|
+
agent.dispose();
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// ---------------------------------------------------------------------------
|
|
332
|
+
// Enrollment leases: exclusive, recoverable, and released.
|
|
333
|
+
// ---------------------------------------------------------------------------
|
|
334
|
+
{
|
|
335
|
+
const agent = makeAgentDir();
|
|
336
|
+
try {
|
|
337
|
+
const statePath = poolStatePathFor(agent.directory);
|
|
338
|
+
const lease = await acquireEnrollmentLease({ statePath, purpose: "add account" });
|
|
339
|
+
await assert.rejects(acquireEnrollmentLease({ statePath, purpose: "off" }), EnrollmentInProgressError);
|
|
340
|
+
await assert.rejects(assertNoActiveEnrollment({ statePath }), /Another pi process/);
|
|
341
|
+
|
|
342
|
+
await lease.renew();
|
|
343
|
+
assert.ok(lease.record.expiresAtMs > Date.now());
|
|
344
|
+
await lease.release();
|
|
345
|
+
await assertNoActiveEnrollment({ statePath });
|
|
346
|
+
assert.equal((await readPoolState(statePath)).enrollment, undefined);
|
|
347
|
+
|
|
348
|
+
// An expired lease and a dead holder are both recoverable without manual cleanup.
|
|
349
|
+
const expired = await acquireEnrollmentLease({ statePath, purpose: "crashed wizard", ttlMs: -1 });
|
|
350
|
+
const replacement = await acquireEnrollmentLease({ statePath, purpose: "add account" });
|
|
351
|
+
await replacement.release();
|
|
352
|
+
await assert.rejects(expired.renew(), EnrollmentLeaseLostError);
|
|
353
|
+
|
|
354
|
+
const statePathWithDeadHolder = statePath;
|
|
355
|
+
await acquireEnrollmentLease({ statePath, purpose: "stuck" });
|
|
356
|
+
const state = await readPoolState(statePathWithDeadHolder);
|
|
357
|
+
writeFileSync(statePathWithDeadHolder, JSON.stringify({ ...state, enrollment: { ...state.enrollment, processId: 2 ** 22 } }, null, 2), "utf8");
|
|
358
|
+
const afterCrash = await acquireEnrollmentLease({ statePath, purpose: "add account" });
|
|
359
|
+
await afterCrash.release();
|
|
360
|
+
|
|
361
|
+
// The helper releases even when the body throws.
|
|
362
|
+
await assert.rejects(withEnrollmentLease({ statePath, purpose: "off" }, async () => { throw new Error("body failed"); }), /body failed/);
|
|
363
|
+
await assertNoActiveEnrollment({ statePath });
|
|
364
|
+
} finally {
|
|
365
|
+
agent.dispose();
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
console.log("Store ownership tests passed");
|