@timo972/cc-router 0.9.0 → 0.10.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -2
- package/README.md +2 -0
- package/dist/config/manager.js +15 -4
- package/dist/protocol/openai-response-to-anthropic.js +16 -1
- package/dist/protocol/openai-responses-collect.js +190 -11
- package/dist/protocol/openai-stream-to-anthropic.js +20 -2
- package/dist/protocol/sse.js +10 -1
- package/dist/providers/openai/account-state.js +377 -0
- package/dist/providers/openai/codex-transport.js +8 -0
- package/dist/providers/openai/failure-routing.js +141 -0
- package/dist/providers/openai/token-pool.js +401 -0
- package/dist/providers/openai/token-refresher.js +144 -11
- package/dist/providers/openai/usage.js +160 -0
- package/dist/proxy/account-add.js +10 -7
- package/dist/proxy/account-deletion.js +5 -1
- package/dist/proxy/account-patch.js +64 -0
- package/dist/proxy/account-pool.js +19 -0
- package/dist/proxy/anthropic-routing.js +2 -2
- package/dist/proxy/lease-lifecycle.js +6 -4
- package/dist/proxy/messages-cross-route.js +243 -62
- package/dist/proxy/openai-ingress.js +379 -0
- package/dist/proxy/openai-routing.js +61 -0
- package/dist/proxy/provider-routing.js +8 -4
- package/dist/proxy/responses-server.js +109 -48
- package/dist/proxy/server.js +241 -65
- package/dist/proxy/stats.js +30 -1
- package/dist/proxy/token-pool.js +3 -19
- package/dist/ui/Dashboard.js +93 -17
- package/package.json +1 -1
- package/dist/providers/openai/account-pool.js +0 -11
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
import { EmptyPoolError, NoEligibleAccountError, } from "../../proxy/account-pool.js";
|
|
2
|
+
import { bucketForModel, bucketIdForModel, sweepCodexRateLimits } from "./account-state.js";
|
|
3
|
+
import { DEFAULT_CODEX_LIMIT_ID } from "./usage.js";
|
|
4
|
+
const MAX_TRUSTED_RATE_LIMIT_RESET_MS = 8 * 24 * 60 * 60 * 1_000;
|
|
5
|
+
/** Matches the bucket-snapshot cap in account-state.ts: a real account has the
|
|
6
|
+
* default bucket plus a handful of metered ones, never dozens. */
|
|
7
|
+
const MAX_BUCKET_COOLDOWN_ENTRIES = 16;
|
|
8
|
+
export class OpenAITokenPool {
|
|
9
|
+
accounts;
|
|
10
|
+
inFlight = new Map();
|
|
11
|
+
cooldowns = new Map();
|
|
12
|
+
now;
|
|
13
|
+
currentIndex = 0;
|
|
14
|
+
onCapBypass;
|
|
15
|
+
onCooldownExpired;
|
|
16
|
+
constructor(accounts, options = {}) {
|
|
17
|
+
this.accounts = accounts;
|
|
18
|
+
this.now = options.now ?? Date.now;
|
|
19
|
+
}
|
|
20
|
+
acquireBest(activeSessions, context) {
|
|
21
|
+
if (this.accounts.length === 0) {
|
|
22
|
+
throw new EmptyPoolError("OpenAI token pool is empty — add an account first");
|
|
23
|
+
}
|
|
24
|
+
this.sweepExpiredCooldowns();
|
|
25
|
+
const hardBlocks = new Map();
|
|
26
|
+
const hardEligible = this.accounts.filter(account => {
|
|
27
|
+
const block = this.hardBlock(account, context);
|
|
28
|
+
if (block)
|
|
29
|
+
hardBlocks.set(account, block);
|
|
30
|
+
return block === null;
|
|
31
|
+
});
|
|
32
|
+
const withinUserCaps = hardEligible.filter(account => !this.overUserCap(account));
|
|
33
|
+
if (withinUserCaps.length > 0) {
|
|
34
|
+
const account = this.selectEligible(withinUserCaps, activeSessions, context);
|
|
35
|
+
this.advanceCursor(account);
|
|
36
|
+
return this.createLease(account, false);
|
|
37
|
+
}
|
|
38
|
+
if (hardEligible.length > 0) {
|
|
39
|
+
const account = this.selectEligible(hardEligible, activeSessions, context);
|
|
40
|
+
this.advanceCursor(account);
|
|
41
|
+
this.onCapBypass?.(account);
|
|
42
|
+
return this.createLease(account, true);
|
|
43
|
+
}
|
|
44
|
+
const rateLimited = [...hardBlocks.values()].filter(block => block.reason === "rate_limited");
|
|
45
|
+
const retryTimes = rateLimited
|
|
46
|
+
.map(block => block.retryAtMs)
|
|
47
|
+
.filter((retryAtMs) => retryAtMs !== undefined);
|
|
48
|
+
throw new NoEligibleAccountError(rateLimited.length > 0 ? "rate_limited" : "unavailable", this.accounts.length, retryTimes.length > 0 ? Math.min(...retryTimes) : undefined);
|
|
49
|
+
}
|
|
50
|
+
tryAcquire(accountId, context) {
|
|
51
|
+
const account = this.findById(accountId);
|
|
52
|
+
if (!account)
|
|
53
|
+
return null;
|
|
54
|
+
const now = this.now();
|
|
55
|
+
sweepCodexRateLimits(account, now, { isRetained: limitId => this.isBucketCoolingDown(account, limitId, now) });
|
|
56
|
+
this.clearExpiredCooldownState(account);
|
|
57
|
+
if (this.hardBlock(account, context) || this.overUserCap(account))
|
|
58
|
+
return null;
|
|
59
|
+
return this.createLease(account, false);
|
|
60
|
+
}
|
|
61
|
+
getInFlight(accountId) {
|
|
62
|
+
return this.inFlight.get(accountId) ?? 0;
|
|
63
|
+
}
|
|
64
|
+
setGlobalCooldownForAccount(account, durationMs, cause) {
|
|
65
|
+
const expiry = this.proposedExpiry(account, durationMs);
|
|
66
|
+
if (expiry === undefined)
|
|
67
|
+
return;
|
|
68
|
+
const state = this.cooldownsFor(account);
|
|
69
|
+
// Each cause extends its own horizon, so neither can misrepresent the
|
|
70
|
+
// other's duration in either direction.
|
|
71
|
+
state.globalUntil = Math.max(state.globalUntil, expiry);
|
|
72
|
+
if (cause === "rate_limit")
|
|
73
|
+
state.rateLimitedUntil = Math.max(state.rateLimitedUntil, expiry);
|
|
74
|
+
}
|
|
75
|
+
setBucketCooldownForAccount(account, limitId, durationMs) {
|
|
76
|
+
const expiry = this.proposedExpiry(account, durationMs);
|
|
77
|
+
if (expiry === undefined)
|
|
78
|
+
return;
|
|
79
|
+
const state = this.cooldownsFor(account);
|
|
80
|
+
if (!state.bucketUntil.has(limitId))
|
|
81
|
+
this.makeRoomForBucketCooldown(state);
|
|
82
|
+
state.bucketUntil.set(limitId, Math.max(state.bucketUntil.get(limitId) ?? 0, expiry));
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Keep `bucketUntil` bounded before a new limit id is added. Every distinct
|
|
86
|
+
* `x-codex-active-limit` value an upstream 429 reports creates an entry that
|
|
87
|
+
* can live for up to the trust horizon, so without a cap a buggy (or
|
|
88
|
+
* hostile) upstream cycling fresh ids would grow this map with request
|
|
89
|
+
* volume — the bucket snapshots and model mappings are already capped, this
|
|
90
|
+
* was the one unbounded piece of per-account state.
|
|
91
|
+
*
|
|
92
|
+
* Expired entries go first since they no longer block anything. If that is
|
|
93
|
+
* not enough, the soonest-to-expire entry is dropped: it is the one with the
|
|
94
|
+
* least protection left to lose.
|
|
95
|
+
*/
|
|
96
|
+
makeRoomForBucketCooldown(state) {
|
|
97
|
+
if (state.bucketUntil.size < MAX_BUCKET_COOLDOWN_ENTRIES)
|
|
98
|
+
return;
|
|
99
|
+
const now = this.now();
|
|
100
|
+
for (const [limitId, until] of state.bucketUntil) {
|
|
101
|
+
if (until <= now)
|
|
102
|
+
state.bucketUntil.delete(limitId);
|
|
103
|
+
}
|
|
104
|
+
if (state.bucketUntil.size < MAX_BUCKET_COOLDOWN_ENTRIES)
|
|
105
|
+
return;
|
|
106
|
+
let soonestId;
|
|
107
|
+
let soonestUntil = Number.POSITIVE_INFINITY;
|
|
108
|
+
for (const [limitId, until] of state.bucketUntil) {
|
|
109
|
+
if (until < soonestUntil) {
|
|
110
|
+
soonestId = limitId;
|
|
111
|
+
soonestUntil = until;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (soonestId !== undefined)
|
|
115
|
+
state.bucketUntil.delete(soonestId);
|
|
116
|
+
}
|
|
117
|
+
getCooldownView(accountId) {
|
|
118
|
+
const account = this.findById(accountId);
|
|
119
|
+
if (!account)
|
|
120
|
+
return { globalUntilMs: 0, bucketCooldowns: [] };
|
|
121
|
+
this.clearExpiredCooldownState(account);
|
|
122
|
+
const state = this.cooldowns.get(account);
|
|
123
|
+
if (!state)
|
|
124
|
+
return { globalUntilMs: 0, bucketCooldowns: [] };
|
|
125
|
+
return {
|
|
126
|
+
globalUntilMs: state.globalUntil,
|
|
127
|
+
bucketCooldowns: [...state.bucketUntil]
|
|
128
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
129
|
+
.slice(0, 12)
|
|
130
|
+
.map(([limitId, untilMs]) => ({ limitId, untilMs })),
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* The account-wide (global-scope) cooldown expiry only — never a
|
|
135
|
+
* bucket-scoped one. A niche model bucket's cooldown must not make the
|
|
136
|
+
* whole account appear busy, and a long global cooldown must not be
|
|
137
|
+
* reported as "recovering soon" just because some unrelated bucket
|
|
138
|
+
* cooldown happens to expire sooner. Per-bucket cooldowns remain visible
|
|
139
|
+
* individually via `getCooldownView`.
|
|
140
|
+
*/
|
|
141
|
+
getGlobalCooldownUntil(accountId) {
|
|
142
|
+
return this.getCooldownView(accountId).globalUntilMs;
|
|
143
|
+
}
|
|
144
|
+
isCoolingDown(accountId) {
|
|
145
|
+
return this.getGlobalCooldownUntil(accountId) > 0;
|
|
146
|
+
}
|
|
147
|
+
sweepExpiredCooldowns() {
|
|
148
|
+
const now = this.now();
|
|
149
|
+
for (const account of this.accounts) {
|
|
150
|
+
// Read cooldown retention before clearExpiredCooldownState (below)
|
|
151
|
+
// deletes any bucketUntil entry that just expired — a bucket must stay
|
|
152
|
+
// retained through this exact sweep for as long as its cooldown is
|
|
153
|
+
// still live at `now`, not react to the expiry the same call is about
|
|
154
|
+
// to perform.
|
|
155
|
+
const windowRecovered = sweepCodexRateLimits(account, now, {
|
|
156
|
+
isRetained: limitId => this.isBucketCoolingDown(account, limitId, now),
|
|
157
|
+
});
|
|
158
|
+
const cooldownRecovered = this.clearExpiredCooldownState(account);
|
|
159
|
+
// Read the account's global cooldown directly: the account object is
|
|
160
|
+
// already in hand here, so going through isCoolingDown() would re-scan
|
|
161
|
+
// the pool by id and rebuild the sorted public cooldown view once per
|
|
162
|
+
// account, making this sweep quadratic for no gain.
|
|
163
|
+
const globalUntil = this.cooldowns.get(account)?.globalUntil ?? 0;
|
|
164
|
+
if (account.rateLimits.status === "rate_limited" && globalUntil <= 0) {
|
|
165
|
+
const defaultBucket = account.rateLimits.buckets.get(DEFAULT_CODEX_LIMIT_ID);
|
|
166
|
+
const exhausted = [defaultBucket?.primary, defaultBucket?.secondary]
|
|
167
|
+
.some(window => window !== undefined && window.utilization >= 1);
|
|
168
|
+
if (!exhausted)
|
|
169
|
+
account.rateLimits.status = "ok";
|
|
170
|
+
}
|
|
171
|
+
if (windowRecovered || cooldownRecovered)
|
|
172
|
+
this.onCooldownExpired?.(account);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
findById(id) {
|
|
176
|
+
return this.accounts.find(account => account.id === id) ?? null;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Drop every piece of per-account routing state after the account has been
|
|
180
|
+
* removed from the shared `accounts` array. The array splice itself is owned
|
|
181
|
+
* by the deletion transaction (it persists a prospective state first), so
|
|
182
|
+
* this only clears what the pool holds on the side.
|
|
183
|
+
*
|
|
184
|
+
* Without this, `createLease`'s release guard (`findById(id) !== account`)
|
|
185
|
+
* means an in-flight request on a deleted account never decrements its
|
|
186
|
+
* counter, and re-adding the same id inherits a phantom in-flight count that
|
|
187
|
+
* permanently deprioritizes it in `selectEligible`. Mirrors the cleanup in
|
|
188
|
+
* the Anthropic `TokenPool.removeAccount`.
|
|
189
|
+
*/
|
|
190
|
+
forgetAccount(account) {
|
|
191
|
+
this.cooldowns.delete(account);
|
|
192
|
+
this.inFlight.delete(account.id);
|
|
193
|
+
this.currentIndex = this.accounts.length === 0
|
|
194
|
+
? 0
|
|
195
|
+
: this.currentIndex % this.accounts.length;
|
|
196
|
+
}
|
|
197
|
+
getAll() {
|
|
198
|
+
return this.accounts;
|
|
199
|
+
}
|
|
200
|
+
hardBlock(account, context) {
|
|
201
|
+
if (!account.enabled || !account.healthy)
|
|
202
|
+
return { reason: "unavailable" };
|
|
203
|
+
const nowMs = this.now();
|
|
204
|
+
const timedBlockers = [];
|
|
205
|
+
let hasIndefiniteBlocker = false;
|
|
206
|
+
// Two separate questions. `blocked` decides whether this account can take
|
|
207
|
+
// the request at all; `rateLimited` decides what a caller is told when no
|
|
208
|
+
// account can. Every blocker answers the first — only a spent quota
|
|
209
|
+
// answers the second, because only that is something the caller can wait
|
|
210
|
+
// out, and only that carries a `Retry-After` worth honouring.
|
|
211
|
+
let blocked = false;
|
|
212
|
+
let rateLimited = false;
|
|
213
|
+
const state = this.cooldowns.get(account);
|
|
214
|
+
if (state !== undefined && state.globalUntil > nowMs) {
|
|
215
|
+
blocked = true;
|
|
216
|
+
// A 401, a 503/529 overload, or a local refresh failure blocks just as
|
|
217
|
+
// hard, but answering 429 for quota the caller never spent sends it
|
|
218
|
+
// looking in entirely the wrong place. The retry hint follows the quota
|
|
219
|
+
// horizon rather than the whole block: it is only ever emitted with a
|
|
220
|
+
// rate-limit answer, and it should say when the quota frees up.
|
|
221
|
+
if (state.rateLimitedUntil > nowMs) {
|
|
222
|
+
rateLimited = true;
|
|
223
|
+
timedBlockers.push(state.rateLimitedUntil);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
const blockingWindows = [];
|
|
227
|
+
const defaultBucket = account.rateLimits.buckets.get(DEFAULT_CODEX_LIMIT_ID);
|
|
228
|
+
for (const window of [defaultBucket?.primary, defaultBucket?.secondary]) {
|
|
229
|
+
if (window !== undefined && window.utilization >= 1)
|
|
230
|
+
blockingWindows.push(window);
|
|
231
|
+
}
|
|
232
|
+
// A bucket cooldown can be learned from a header-only 429 that carried no
|
|
233
|
+
// rate-limit snapshot for that bucket (`setBucketCooldownForAccount` was
|
|
234
|
+
// called, but `account.rateLimits.buckets` has no entry for it yet).
|
|
235
|
+
// Resolve the model's mapped limitId independently of whether a bucket
|
|
236
|
+
// snapshot exists so that cooldown is still enforced; the window-based
|
|
237
|
+
// exhaustion check below still only applies when a snapshot is present.
|
|
238
|
+
const modelLimitId = bucketIdForModel(account, context?.requestedModel);
|
|
239
|
+
if (modelLimitId !== undefined) {
|
|
240
|
+
const bucketCooldown = state?.bucketUntil.get(modelLimitId) ?? 0;
|
|
241
|
+
if (bucketCooldown > nowMs) {
|
|
242
|
+
// Bucket cooldowns come only from a 429 naming an active limit.
|
|
243
|
+
blocked = true;
|
|
244
|
+
rateLimited = true;
|
|
245
|
+
timedBlockers.push(bucketCooldown);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
const modelBucket = this.modelBucket(account, context);
|
|
249
|
+
if (modelBucket !== undefined) {
|
|
250
|
+
for (const window of [modelBucket.primary, modelBucket.secondary]) {
|
|
251
|
+
if (window !== undefined && window.utilization >= 1)
|
|
252
|
+
blockingWindows.push(window);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
for (const window of blockingWindows) {
|
|
256
|
+
// An exhausted usage window is a spent quota by definition.
|
|
257
|
+
blocked = true;
|
|
258
|
+
rateLimited = true;
|
|
259
|
+
const resetMs = trustworthyResetMs(window.resetAt, nowMs);
|
|
260
|
+
if (resetMs !== undefined)
|
|
261
|
+
timedBlockers.push(resetMs);
|
|
262
|
+
else
|
|
263
|
+
hasIndefiniteBlocker = true;
|
|
264
|
+
}
|
|
265
|
+
if (!blocked)
|
|
266
|
+
return null;
|
|
267
|
+
if (!rateLimited)
|
|
268
|
+
return { reason: "unavailable" };
|
|
269
|
+
const retryAtMs = !hasIndefiniteBlocker && timedBlockers.length > 0
|
|
270
|
+
? Math.max(...timedBlockers)
|
|
271
|
+
: undefined;
|
|
272
|
+
return retryAtMs === undefined
|
|
273
|
+
? { reason: "rate_limited" }
|
|
274
|
+
: { reason: "rate_limited", retryAtMs };
|
|
275
|
+
}
|
|
276
|
+
modelBucket(account, context) {
|
|
277
|
+
return bucketForModel(account, context?.requestedModel);
|
|
278
|
+
}
|
|
279
|
+
/** True while `limitId` has an active bucket-scoped cooldown on `account`. */
|
|
280
|
+
isBucketCoolingDown(account, limitId, nowMs) {
|
|
281
|
+
return (this.cooldowns.get(account)?.bucketUntil.get(limitId) ?? 0) > nowMs;
|
|
282
|
+
}
|
|
283
|
+
overUserCap(account) {
|
|
284
|
+
const defaultBucket = account.rateLimits.buckets.get(DEFAULT_CODEX_LIMIT_ID);
|
|
285
|
+
const primaryUtil = defaultBucket?.primary?.utilization ?? 0;
|
|
286
|
+
const secondaryUtil = defaultBucket?.secondary?.utilization ?? 0;
|
|
287
|
+
return (account.sessionLimitPercent < 100 && primaryUtil * 100 >= account.sessionLimitPercent) ||
|
|
288
|
+
(account.weeklyLimitPercent < 100 && secondaryUtil * 100 >= account.weeklyLimitPercent);
|
|
289
|
+
}
|
|
290
|
+
headroomScore(account, context) {
|
|
291
|
+
const defaultBucket = account.rateLimits.buckets.get(DEFAULT_CODEX_LIMIT_ID);
|
|
292
|
+
const modelBucket = this.modelBucket(account, context);
|
|
293
|
+
return Math.max(capNormalizedUtilization(defaultBucket?.primary?.utilization ?? 0, account.sessionLimitPercent), capNormalizedUtilization(defaultBucket?.secondary?.utilization ?? 0, account.weeklyLimitPercent), modelBucket?.primary?.utilization ?? 0, modelBucket?.secondary?.utilization ?? 0);
|
|
294
|
+
}
|
|
295
|
+
selectEligible(candidates, activeSessions, context) {
|
|
296
|
+
return candidates.reduce((best, account) => {
|
|
297
|
+
const comparison = compareTuple([
|
|
298
|
+
this.getInFlight(account.id),
|
|
299
|
+
activeSessions.get(account.id) ?? 0,
|
|
300
|
+
this.headroomScore(account, context),
|
|
301
|
+
this.circularDistance(account),
|
|
302
|
+
], [
|
|
303
|
+
this.getInFlight(best.id),
|
|
304
|
+
activeSessions.get(best.id) ?? 0,
|
|
305
|
+
this.headroomScore(best, context),
|
|
306
|
+
this.circularDistance(best),
|
|
307
|
+
]);
|
|
308
|
+
return comparison < 0 ? account : best;
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
circularDistance(account) {
|
|
312
|
+
const index = this.accounts.indexOf(account);
|
|
313
|
+
return (index - this.currentIndex + this.accounts.length) % this.accounts.length;
|
|
314
|
+
}
|
|
315
|
+
advanceCursor(account) {
|
|
316
|
+
const index = this.accounts.indexOf(account);
|
|
317
|
+
this.currentIndex = (index + 1) % this.accounts.length;
|
|
318
|
+
}
|
|
319
|
+
proposedExpiry(account, durationMs) {
|
|
320
|
+
if (this.findById(account.id) !== account)
|
|
321
|
+
return undefined;
|
|
322
|
+
if (!Number.isFinite(durationMs) || durationMs <= 0)
|
|
323
|
+
return undefined;
|
|
324
|
+
const expiry = this.now() + durationMs;
|
|
325
|
+
return Number.isFinite(expiry) ? expiry : undefined;
|
|
326
|
+
}
|
|
327
|
+
cooldownsFor(account) {
|
|
328
|
+
let state = this.cooldowns.get(account);
|
|
329
|
+
if (!state) {
|
|
330
|
+
state = { globalUntil: 0, rateLimitedUntil: 0, bucketUntil: new Map() };
|
|
331
|
+
this.cooldowns.set(account, state);
|
|
332
|
+
}
|
|
333
|
+
return state;
|
|
334
|
+
}
|
|
335
|
+
/** Returns true when an active cooldown scope just expired. */
|
|
336
|
+
clearExpiredCooldownState(account) {
|
|
337
|
+
const state = this.cooldowns.get(account);
|
|
338
|
+
if (!state)
|
|
339
|
+
return false;
|
|
340
|
+
const now = this.now();
|
|
341
|
+
let recovered = false;
|
|
342
|
+
if (state.rateLimitedUntil > 0 && state.rateLimitedUntil <= now)
|
|
343
|
+
state.rateLimitedUntil = 0;
|
|
344
|
+
if (state.globalUntil > 0 && state.globalUntil <= now) {
|
|
345
|
+
state.globalUntil = 0;
|
|
346
|
+
recovered = true;
|
|
347
|
+
}
|
|
348
|
+
for (const [limitId, until] of state.bucketUntil) {
|
|
349
|
+
if (until <= now) {
|
|
350
|
+
state.bucketUntil.delete(limitId);
|
|
351
|
+
recovered = true;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
if (state.globalUntil === 0 && state.bucketUntil.size === 0)
|
|
355
|
+
this.cooldowns.delete(account);
|
|
356
|
+
return recovered;
|
|
357
|
+
}
|
|
358
|
+
createLease(account, fallback) {
|
|
359
|
+
this.inFlight.set(account.id, this.getInFlight(account.id) + 1);
|
|
360
|
+
account.requestCount++;
|
|
361
|
+
account.lastUsed = this.now();
|
|
362
|
+
let released = false;
|
|
363
|
+
return {
|
|
364
|
+
account,
|
|
365
|
+
fallback,
|
|
366
|
+
release: () => {
|
|
367
|
+
if (released)
|
|
368
|
+
return;
|
|
369
|
+
released = true;
|
|
370
|
+
if (this.findById(account.id) !== account)
|
|
371
|
+
return;
|
|
372
|
+
const remaining = Math.max(0, this.getInFlight(account.id) - 1);
|
|
373
|
+
if (remaining === 0)
|
|
374
|
+
this.inFlight.delete(account.id);
|
|
375
|
+
else
|
|
376
|
+
this.inFlight.set(account.id, remaining);
|
|
377
|
+
},
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
function trustworthyResetMs(resetAtSeconds, nowMs) {
|
|
382
|
+
if (!Number.isFinite(resetAtSeconds) || resetAtSeconds <= 0)
|
|
383
|
+
return undefined;
|
|
384
|
+
const resetAtMs = Math.floor(resetAtSeconds) * 1_000;
|
|
385
|
+
if (resetAtMs <= nowMs)
|
|
386
|
+
return undefined;
|
|
387
|
+
return resetAtMs - nowMs <= MAX_TRUSTED_RATE_LIMIT_RESET_MS ? resetAtMs : undefined;
|
|
388
|
+
}
|
|
389
|
+
function capNormalizedUtilization(utilization, capPercent) {
|
|
390
|
+
const cap = Number.isFinite(capPercent) ? Math.max(0, capPercent / 100) : 1;
|
|
391
|
+
return cap === 0 ? Number.POSITIVE_INFINITY : utilization / cap;
|
|
392
|
+
}
|
|
393
|
+
function compareTuple(left, right) {
|
|
394
|
+
for (let i = 0; i < left.length; i++) {
|
|
395
|
+
const a = left[i] ?? 0;
|
|
396
|
+
const b = right[i] ?? 0;
|
|
397
|
+
if (a !== b)
|
|
398
|
+
return a - b;
|
|
399
|
+
}
|
|
400
|
+
return 0;
|
|
401
|
+
}
|
|
@@ -1,7 +1,64 @@
|
|
|
1
|
+
import { decodeOpenAIPlan } from "./usage.js";
|
|
1
2
|
const TOKEN_ENDPOINT = "https://auth.openai.com/oauth/token";
|
|
2
3
|
const REFRESH_BUFFER_MS = 10 * 60 * 1000;
|
|
3
4
|
const CHECK_INTERVAL_MS = 5 * 60 * 1000;
|
|
4
5
|
const refreshLocks = new Map();
|
|
6
|
+
/**
|
|
7
|
+
* Accounts whose most recently rotated credentials have NOT been confirmed
|
|
8
|
+
* on disk. Identity-keyed (by object reference, not `account.id`) so a
|
|
9
|
+
* deleted-then-re-added account can never inherit another object's dirty
|
|
10
|
+
* state just because it reused the same id, and a `WeakSet` so an account
|
|
11
|
+
* that's later removed from the pool can't leak here.
|
|
12
|
+
*/
|
|
13
|
+
const pendingCredentialWrites = new WeakSet();
|
|
14
|
+
/** True when `account` has a rotated credential that has not yet been durably persisted. */
|
|
15
|
+
export function hasPendingCredentialWrite(account) {
|
|
16
|
+
return pendingCredentialWrites.has(account);
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Record that every account in `accounts` has reached disk.
|
|
20
|
+
*
|
|
21
|
+
* A whole-pool write makes all of them durable, not just whichever account
|
|
22
|
+
* prompted it — and those writes do not all come from a refresh. Adding,
|
|
23
|
+
* patching, or deleting an account rewrites the same file from the same live
|
|
24
|
+
* array, so a rotation that failed to persist earlier is on disk once any of
|
|
25
|
+
* those succeed. Without this, health keeps reporting `credentialsPendingWrite`
|
|
26
|
+
* for an account whose credentials are already saved, and later requests keep
|
|
27
|
+
* retrying a write that has already landed.
|
|
28
|
+
*
|
|
29
|
+
* Call it only after the write has actually returned.
|
|
30
|
+
*/
|
|
31
|
+
export function markOpenAICredentialsPersisted(accounts) {
|
|
32
|
+
for (const account of accounts)
|
|
33
|
+
pendingCredentialWrites.delete(account);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Centralizes every write of the account pool to disk so every caller shares
|
|
37
|
+
* the same durability bookkeeping: on success, clears the account's pending
|
|
38
|
+
* flag; on a throw (e.g. disk full, a bad custom `--accounts` path), (re)sets
|
|
39
|
+
* it and logs, but never propagates — the caller's fresh in-memory token
|
|
40
|
+
* remains usable even though it isn't on disk yet.
|
|
41
|
+
*
|
|
42
|
+
* Residual risk: if the process crashes before any retry of a dirty account
|
|
43
|
+
* succeeds, the rotated refresh token is lost from disk — and since OpenAI
|
|
44
|
+
* already invalidated the old one, that forces re-authentication. The retry
|
|
45
|
+
* cadence (the very next request via `prepareOpenAIAccountForRequest`, or at
|
|
46
|
+
* worst the refresh loop's next ≤5-minute tick) bounds how long that window
|
|
47
|
+
* stays open, but does not close it.
|
|
48
|
+
*/
|
|
49
|
+
function persistCredentials(account, allAccounts, saveAccounts) {
|
|
50
|
+
try {
|
|
51
|
+
saveAccounts(allAccounts);
|
|
52
|
+
markOpenAICredentialsPersisted(allAccounts);
|
|
53
|
+
pendingCredentialWrites.delete(account);
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
pendingCredentialWrites.add(account);
|
|
58
|
+
console.error(error);
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
5
62
|
export function needsOpenAIRefresh(account) {
|
|
6
63
|
return account.expiresAt - Date.now() < REFRESH_BUFFER_MS;
|
|
7
64
|
}
|
|
@@ -19,17 +76,45 @@ export async function refreshOpenAISubscriptionToken(account) {
|
|
|
19
76
|
}
|
|
20
77
|
}
|
|
21
78
|
export async function prepareOpenAIAccountForRequest(account, allAccounts, saveAccounts) {
|
|
22
|
-
if (!needsOpenAIRefresh(account))
|
|
79
|
+
if (!needsOpenAIRefresh(account)) {
|
|
80
|
+
// No refresh due, but a previous rotation from this account never made it
|
|
81
|
+
// to disk (e.g. a transient disk-full). This is the retry path: piggyback
|
|
82
|
+
// on this otherwise-idle request to flush the still-current in-memory
|
|
83
|
+
// token, without blocking or failing the request either way.
|
|
84
|
+
if (hasPendingCredentialWrite(account))
|
|
85
|
+
persistCredentials(account, allAccounts, saveAccounts);
|
|
23
86
|
return true;
|
|
87
|
+
}
|
|
88
|
+
// The refreshed token is already live in memory, so a persistence failure
|
|
89
|
+
// must not fail this request — it only means the new token isn't on disk
|
|
90
|
+
// yet. `persistCredentials` marks the account dirty so the write is retried
|
|
91
|
+
// (here, or by the refresh loop) instead of silently losing the rotation.
|
|
92
|
+
return refreshAndPersistOpenAIAccount(account, allAccounts, saveAccounts);
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Refreshes `account`'s token if needed and persists the pool. Used both by
|
|
96
|
+
* `prepareOpenAIAccountForRequest`'s refresh-due branch and by callers that
|
|
97
|
+
* want to force a refresh outside of that gate (e.g. reacting to an upstream
|
|
98
|
+
* 401). Always returns the refresh outcome — a persistence failure never
|
|
99
|
+
* turns a successful refresh into a `false` result.
|
|
100
|
+
*/
|
|
101
|
+
export async function refreshAndPersistOpenAIAccount(account, allAccounts, saveAccounts) {
|
|
24
102
|
const ok = await refreshOpenAISubscriptionToken(account);
|
|
25
103
|
if (ok)
|
|
26
|
-
|
|
104
|
+
persistCredentials(account, allAccounts, saveAccounts);
|
|
27
105
|
return ok;
|
|
28
106
|
}
|
|
29
107
|
export function startOpenAIRefreshLoop(accounts, saveAccounts) {
|
|
30
108
|
const check = async () => {
|
|
31
109
|
for (const account of accounts) {
|
|
32
|
-
|
|
110
|
+
// One account's refresh throwing must not skip every account after it
|
|
111
|
+
// in this tick — isolate failures per-account.
|
|
112
|
+
try {
|
|
113
|
+
await prepareOpenAIAccountForRequest(account, accounts, saveAccounts);
|
|
114
|
+
}
|
|
115
|
+
catch (error) {
|
|
116
|
+
console.error(error);
|
|
117
|
+
}
|
|
33
118
|
}
|
|
34
119
|
};
|
|
35
120
|
const timer = setInterval(() => { check().catch(console.error); }, CHECK_INTERVAL_MS);
|
|
@@ -41,16 +126,64 @@ async function doRefresh(account) {
|
|
|
41
126
|
grant_type: "refresh_token",
|
|
42
127
|
refresh_token: account.refreshToken,
|
|
43
128
|
});
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
129
|
+
let data;
|
|
130
|
+
try {
|
|
131
|
+
const res = await fetch(TOKEN_ENDPOINT, {
|
|
132
|
+
method: "POST",
|
|
133
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
134
|
+
body: body.toString(),
|
|
135
|
+
});
|
|
136
|
+
if (!res.ok)
|
|
137
|
+
return false;
|
|
138
|
+
data = await res.json();
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
// Network failure (or malformed response body) must resolve to `false`,
|
|
142
|
+
// exactly like a non-ok HTTP response — never propagate as a rejection.
|
|
143
|
+
return false;
|
|
144
|
+
}
|
|
145
|
+
// A 200 with an unusable payload is a failed refresh, not a successful one.
|
|
146
|
+
// Writing it through would leave `expiresAt` as NaN, which then reads as
|
|
147
|
+
// "never needs refreshing" in `needsOpenAIRefresh` and permanently strands
|
|
148
|
+
// the account on a broken token.
|
|
149
|
+
if (typeof data?.access_token !== "string" || data.access_token.length === 0)
|
|
150
|
+
return false;
|
|
151
|
+
// The lifetime has to be positive and has to still name a finite instant
|
|
152
|
+
// once converted. A zero or negative `expires_in` would report success on a
|
|
153
|
+
// token that is already due for another refresh, so every request re-enters
|
|
154
|
+
// the refresh path; a value big enough to overflow the multiplication would
|
|
155
|
+
// set `expiresAt` to Infinity, which `needsOpenAIRefresh` can never reach —
|
|
156
|
+
// the same permanent strand as NaN, from the opposite direction.
|
|
157
|
+
if (typeof data.expires_in !== "number" || !Number.isFinite(data.expires_in))
|
|
158
|
+
return false;
|
|
159
|
+
if (data.expires_in <= 0)
|
|
160
|
+
return false;
|
|
161
|
+
const expiresAt = Date.now() + data.expires_in * 1000;
|
|
162
|
+
if (!Number.isFinite(expiresAt))
|
|
50
163
|
return false;
|
|
51
|
-
const data = await res.json();
|
|
52
164
|
account.accessToken = data.access_token;
|
|
53
165
|
account.refreshToken = data.refresh_token ?? account.refreshToken;
|
|
54
|
-
account.expiresAt =
|
|
166
|
+
account.expiresAt = expiresAt;
|
|
167
|
+
// A successful refresh recovers an account the pool previously excluded for
|
|
168
|
+
// being unhealthy (e.g. after a prior failed refresh). Without this, the pool's
|
|
169
|
+
// hard `!healthy` block means the account never gets acquired again — and thus
|
|
170
|
+
// never gets another chance to refresh — so it stays excluded until restart.
|
|
171
|
+
const runtime = account;
|
|
172
|
+
if (runtime.healthy !== undefined)
|
|
173
|
+
runtime.healthy = true;
|
|
174
|
+
if (runtime.consecutiveErrors !== undefined)
|
|
175
|
+
runtime.consecutiveErrors = 0;
|
|
176
|
+
if (runtime.lastRefresh !== undefined)
|
|
177
|
+
runtime.lastRefresh = Date.now();
|
|
178
|
+
// The rotated access token can carry a different plan than the one decoded
|
|
179
|
+
// at account creation (e.g. a Plus->Pro upgrade). Mirrors createOpenAIAccount's
|
|
180
|
+
// semantics: only overwrite when the new token actually decodes a plan claim —
|
|
181
|
+
// an undecodable token leaves the previously known plan in place rather than
|
|
182
|
+
// erasing it, since a missing claim means "unknown", not "no plan".
|
|
183
|
+
if (runtime.rateLimits) {
|
|
184
|
+
const plan = decodeOpenAIPlan(account.accessToken);
|
|
185
|
+
if (plan)
|
|
186
|
+
runtime.rateLimits.plan = plan;
|
|
187
|
+
}
|
|
55
188
|
return true;
|
|
56
189
|
}
|