@timo972/cc-router 0.9.0 → 0.10.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,401 @@
1
+ import { EmptyPoolError, NoEligibleAccountError, } from "../../proxy/account-pool.js";
2
+ import { bucketForModel, bucketIdForModel, sweepCodexRateLimits } from "./account-state.js";
3
+ import { DEFAULT_CODEX_LIMIT_ID } from "./usage.js";
4
+ const MAX_TRUSTED_RATE_LIMIT_RESET_MS = 8 * 24 * 60 * 60 * 1_000;
5
+ /** Matches the bucket-snapshot cap in account-state.ts: a real account has the
6
+ * default bucket plus a handful of metered ones, never dozens. */
7
+ const MAX_BUCKET_COOLDOWN_ENTRIES = 16;
8
+ export class OpenAITokenPool {
9
+ accounts;
10
+ inFlight = new Map();
11
+ cooldowns = new Map();
12
+ now;
13
+ currentIndex = 0;
14
+ onCapBypass;
15
+ onCooldownExpired;
16
+ constructor(accounts, options = {}) {
17
+ this.accounts = accounts;
18
+ this.now = options.now ?? Date.now;
19
+ }
20
+ acquireBest(activeSessions, context) {
21
+ if (this.accounts.length === 0) {
22
+ throw new EmptyPoolError("OpenAI token pool is empty — add an account first");
23
+ }
24
+ this.sweepExpiredCooldowns();
25
+ const hardBlocks = new Map();
26
+ const hardEligible = this.accounts.filter(account => {
27
+ const block = this.hardBlock(account, context);
28
+ if (block)
29
+ hardBlocks.set(account, block);
30
+ return block === null;
31
+ });
32
+ const withinUserCaps = hardEligible.filter(account => !this.overUserCap(account));
33
+ if (withinUserCaps.length > 0) {
34
+ const account = this.selectEligible(withinUserCaps, activeSessions, context);
35
+ this.advanceCursor(account);
36
+ return this.createLease(account, false);
37
+ }
38
+ if (hardEligible.length > 0) {
39
+ const account = this.selectEligible(hardEligible, activeSessions, context);
40
+ this.advanceCursor(account);
41
+ this.onCapBypass?.(account);
42
+ return this.createLease(account, true);
43
+ }
44
+ const rateLimited = [...hardBlocks.values()].filter(block => block.reason === "rate_limited");
45
+ const retryTimes = rateLimited
46
+ .map(block => block.retryAtMs)
47
+ .filter((retryAtMs) => retryAtMs !== undefined);
48
+ throw new NoEligibleAccountError(rateLimited.length > 0 ? "rate_limited" : "unavailable", this.accounts.length, retryTimes.length > 0 ? Math.min(...retryTimes) : undefined);
49
+ }
50
+ tryAcquire(accountId, context) {
51
+ const account = this.findById(accountId);
52
+ if (!account)
53
+ return null;
54
+ const now = this.now();
55
+ sweepCodexRateLimits(account, now, { isRetained: limitId => this.isBucketCoolingDown(account, limitId, now) });
56
+ this.clearExpiredCooldownState(account);
57
+ if (this.hardBlock(account, context) || this.overUserCap(account))
58
+ return null;
59
+ return this.createLease(account, false);
60
+ }
61
+ getInFlight(accountId) {
62
+ return this.inFlight.get(accountId) ?? 0;
63
+ }
64
+ setGlobalCooldownForAccount(account, durationMs, cause) {
65
+ const expiry = this.proposedExpiry(account, durationMs);
66
+ if (expiry === undefined)
67
+ return;
68
+ const state = this.cooldownsFor(account);
69
+ // Each cause extends its own horizon, so neither can misrepresent the
70
+ // other's duration in either direction.
71
+ state.globalUntil = Math.max(state.globalUntil, expiry);
72
+ if (cause === "rate_limit")
73
+ state.rateLimitedUntil = Math.max(state.rateLimitedUntil, expiry);
74
+ }
75
+ setBucketCooldownForAccount(account, limitId, durationMs) {
76
+ const expiry = this.proposedExpiry(account, durationMs);
77
+ if (expiry === undefined)
78
+ return;
79
+ const state = this.cooldownsFor(account);
80
+ if (!state.bucketUntil.has(limitId))
81
+ this.makeRoomForBucketCooldown(state);
82
+ state.bucketUntil.set(limitId, Math.max(state.bucketUntil.get(limitId) ?? 0, expiry));
83
+ }
84
+ /**
85
+ * Keep `bucketUntil` bounded before a new limit id is added. Every distinct
86
+ * `x-codex-active-limit` value an upstream 429 reports creates an entry that
87
+ * can live for up to the trust horizon, so without a cap a buggy (or
88
+ * hostile) upstream cycling fresh ids would grow this map with request
89
+ * volume — the bucket snapshots and model mappings are already capped, this
90
+ * was the one unbounded piece of per-account state.
91
+ *
92
+ * Expired entries go first since they no longer block anything. If that is
93
+ * not enough, the soonest-to-expire entry is dropped: it is the one with the
94
+ * least protection left to lose.
95
+ */
96
+ makeRoomForBucketCooldown(state) {
97
+ if (state.bucketUntil.size < MAX_BUCKET_COOLDOWN_ENTRIES)
98
+ return;
99
+ const now = this.now();
100
+ for (const [limitId, until] of state.bucketUntil) {
101
+ if (until <= now)
102
+ state.bucketUntil.delete(limitId);
103
+ }
104
+ if (state.bucketUntil.size < MAX_BUCKET_COOLDOWN_ENTRIES)
105
+ return;
106
+ let soonestId;
107
+ let soonestUntil = Number.POSITIVE_INFINITY;
108
+ for (const [limitId, until] of state.bucketUntil) {
109
+ if (until < soonestUntil) {
110
+ soonestId = limitId;
111
+ soonestUntil = until;
112
+ }
113
+ }
114
+ if (soonestId !== undefined)
115
+ state.bucketUntil.delete(soonestId);
116
+ }
117
+ getCooldownView(accountId) {
118
+ const account = this.findById(accountId);
119
+ if (!account)
120
+ return { globalUntilMs: 0, bucketCooldowns: [] };
121
+ this.clearExpiredCooldownState(account);
122
+ const state = this.cooldowns.get(account);
123
+ if (!state)
124
+ return { globalUntilMs: 0, bucketCooldowns: [] };
125
+ return {
126
+ globalUntilMs: state.globalUntil,
127
+ bucketCooldowns: [...state.bucketUntil]
128
+ .sort(([left], [right]) => left.localeCompare(right))
129
+ .slice(0, 12)
130
+ .map(([limitId, untilMs]) => ({ limitId, untilMs })),
131
+ };
132
+ }
133
+ /**
134
+ * The account-wide (global-scope) cooldown expiry only — never a
135
+ * bucket-scoped one. A niche model bucket's cooldown must not make the
136
+ * whole account appear busy, and a long global cooldown must not be
137
+ * reported as "recovering soon" just because some unrelated bucket
138
+ * cooldown happens to expire sooner. Per-bucket cooldowns remain visible
139
+ * individually via `getCooldownView`.
140
+ */
141
+ getGlobalCooldownUntil(accountId) {
142
+ return this.getCooldownView(accountId).globalUntilMs;
143
+ }
144
+ isCoolingDown(accountId) {
145
+ return this.getGlobalCooldownUntil(accountId) > 0;
146
+ }
147
+ sweepExpiredCooldowns() {
148
+ const now = this.now();
149
+ for (const account of this.accounts) {
150
+ // Read cooldown retention before clearExpiredCooldownState (below)
151
+ // deletes any bucketUntil entry that just expired — a bucket must stay
152
+ // retained through this exact sweep for as long as its cooldown is
153
+ // still live at `now`, not react to the expiry the same call is about
154
+ // to perform.
155
+ const windowRecovered = sweepCodexRateLimits(account, now, {
156
+ isRetained: limitId => this.isBucketCoolingDown(account, limitId, now),
157
+ });
158
+ const cooldownRecovered = this.clearExpiredCooldownState(account);
159
+ // Read the account's global cooldown directly: the account object is
160
+ // already in hand here, so going through isCoolingDown() would re-scan
161
+ // the pool by id and rebuild the sorted public cooldown view once per
162
+ // account, making this sweep quadratic for no gain.
163
+ const globalUntil = this.cooldowns.get(account)?.globalUntil ?? 0;
164
+ if (account.rateLimits.status === "rate_limited" && globalUntil <= 0) {
165
+ const defaultBucket = account.rateLimits.buckets.get(DEFAULT_CODEX_LIMIT_ID);
166
+ const exhausted = [defaultBucket?.primary, defaultBucket?.secondary]
167
+ .some(window => window !== undefined && window.utilization >= 1);
168
+ if (!exhausted)
169
+ account.rateLimits.status = "ok";
170
+ }
171
+ if (windowRecovered || cooldownRecovered)
172
+ this.onCooldownExpired?.(account);
173
+ }
174
+ }
175
+ findById(id) {
176
+ return this.accounts.find(account => account.id === id) ?? null;
177
+ }
178
+ /**
179
+ * Drop every piece of per-account routing state after the account has been
180
+ * removed from the shared `accounts` array. The array splice itself is owned
181
+ * by the deletion transaction (it persists a prospective state first), so
182
+ * this only clears what the pool holds on the side.
183
+ *
184
+ * Without this, `createLease`'s release guard (`findById(id) !== account`)
185
+ * means an in-flight request on a deleted account never decrements its
186
+ * counter, and re-adding the same id inherits a phantom in-flight count that
187
+ * permanently deprioritizes it in `selectEligible`. Mirrors the cleanup in
188
+ * the Anthropic `TokenPool.removeAccount`.
189
+ */
190
+ forgetAccount(account) {
191
+ this.cooldowns.delete(account);
192
+ this.inFlight.delete(account.id);
193
+ this.currentIndex = this.accounts.length === 0
194
+ ? 0
195
+ : this.currentIndex % this.accounts.length;
196
+ }
197
+ getAll() {
198
+ return this.accounts;
199
+ }
200
+ hardBlock(account, context) {
201
+ if (!account.enabled || !account.healthy)
202
+ return { reason: "unavailable" };
203
+ const nowMs = this.now();
204
+ const timedBlockers = [];
205
+ let hasIndefiniteBlocker = false;
206
+ // Two separate questions. `blocked` decides whether this account can take
207
+ // the request at all; `rateLimited` decides what a caller is told when no
208
+ // account can. Every blocker answers the first — only a spent quota
209
+ // answers the second, because only that is something the caller can wait
210
+ // out, and only that carries a `Retry-After` worth honouring.
211
+ let blocked = false;
212
+ let rateLimited = false;
213
+ const state = this.cooldowns.get(account);
214
+ if (state !== undefined && state.globalUntil > nowMs) {
215
+ blocked = true;
216
+ // A 401, a 503/529 overload, or a local refresh failure blocks just as
217
+ // hard, but answering 429 for quota the caller never spent sends it
218
+ // looking in entirely the wrong place. The retry hint follows the quota
219
+ // horizon rather than the whole block: it is only ever emitted with a
220
+ // rate-limit answer, and it should say when the quota frees up.
221
+ if (state.rateLimitedUntil > nowMs) {
222
+ rateLimited = true;
223
+ timedBlockers.push(state.rateLimitedUntil);
224
+ }
225
+ }
226
+ const blockingWindows = [];
227
+ const defaultBucket = account.rateLimits.buckets.get(DEFAULT_CODEX_LIMIT_ID);
228
+ for (const window of [defaultBucket?.primary, defaultBucket?.secondary]) {
229
+ if (window !== undefined && window.utilization >= 1)
230
+ blockingWindows.push(window);
231
+ }
232
+ // A bucket cooldown can be learned from a header-only 429 that carried no
233
+ // rate-limit snapshot for that bucket (`setBucketCooldownForAccount` was
234
+ // called, but `account.rateLimits.buckets` has no entry for it yet).
235
+ // Resolve the model's mapped limitId independently of whether a bucket
236
+ // snapshot exists so that cooldown is still enforced; the window-based
237
+ // exhaustion check below still only applies when a snapshot is present.
238
+ const modelLimitId = bucketIdForModel(account, context?.requestedModel);
239
+ if (modelLimitId !== undefined) {
240
+ const bucketCooldown = state?.bucketUntil.get(modelLimitId) ?? 0;
241
+ if (bucketCooldown > nowMs) {
242
+ // Bucket cooldowns come only from a 429 naming an active limit.
243
+ blocked = true;
244
+ rateLimited = true;
245
+ timedBlockers.push(bucketCooldown);
246
+ }
247
+ }
248
+ const modelBucket = this.modelBucket(account, context);
249
+ if (modelBucket !== undefined) {
250
+ for (const window of [modelBucket.primary, modelBucket.secondary]) {
251
+ if (window !== undefined && window.utilization >= 1)
252
+ blockingWindows.push(window);
253
+ }
254
+ }
255
+ for (const window of blockingWindows) {
256
+ // An exhausted usage window is a spent quota by definition.
257
+ blocked = true;
258
+ rateLimited = true;
259
+ const resetMs = trustworthyResetMs(window.resetAt, nowMs);
260
+ if (resetMs !== undefined)
261
+ timedBlockers.push(resetMs);
262
+ else
263
+ hasIndefiniteBlocker = true;
264
+ }
265
+ if (!blocked)
266
+ return null;
267
+ if (!rateLimited)
268
+ return { reason: "unavailable" };
269
+ const retryAtMs = !hasIndefiniteBlocker && timedBlockers.length > 0
270
+ ? Math.max(...timedBlockers)
271
+ : undefined;
272
+ return retryAtMs === undefined
273
+ ? { reason: "rate_limited" }
274
+ : { reason: "rate_limited", retryAtMs };
275
+ }
276
+ modelBucket(account, context) {
277
+ return bucketForModel(account, context?.requestedModel);
278
+ }
279
+ /** True while `limitId` has an active bucket-scoped cooldown on `account`. */
280
+ isBucketCoolingDown(account, limitId, nowMs) {
281
+ return (this.cooldowns.get(account)?.bucketUntil.get(limitId) ?? 0) > nowMs;
282
+ }
283
+ overUserCap(account) {
284
+ const defaultBucket = account.rateLimits.buckets.get(DEFAULT_CODEX_LIMIT_ID);
285
+ const primaryUtil = defaultBucket?.primary?.utilization ?? 0;
286
+ const secondaryUtil = defaultBucket?.secondary?.utilization ?? 0;
287
+ return (account.sessionLimitPercent < 100 && primaryUtil * 100 >= account.sessionLimitPercent) ||
288
+ (account.weeklyLimitPercent < 100 && secondaryUtil * 100 >= account.weeklyLimitPercent);
289
+ }
290
+ headroomScore(account, context) {
291
+ const defaultBucket = account.rateLimits.buckets.get(DEFAULT_CODEX_LIMIT_ID);
292
+ const modelBucket = this.modelBucket(account, context);
293
+ return Math.max(capNormalizedUtilization(defaultBucket?.primary?.utilization ?? 0, account.sessionLimitPercent), capNormalizedUtilization(defaultBucket?.secondary?.utilization ?? 0, account.weeklyLimitPercent), modelBucket?.primary?.utilization ?? 0, modelBucket?.secondary?.utilization ?? 0);
294
+ }
295
+ selectEligible(candidates, activeSessions, context) {
296
+ return candidates.reduce((best, account) => {
297
+ const comparison = compareTuple([
298
+ this.getInFlight(account.id),
299
+ activeSessions.get(account.id) ?? 0,
300
+ this.headroomScore(account, context),
301
+ this.circularDistance(account),
302
+ ], [
303
+ this.getInFlight(best.id),
304
+ activeSessions.get(best.id) ?? 0,
305
+ this.headroomScore(best, context),
306
+ this.circularDistance(best),
307
+ ]);
308
+ return comparison < 0 ? account : best;
309
+ });
310
+ }
311
+ circularDistance(account) {
312
+ const index = this.accounts.indexOf(account);
313
+ return (index - this.currentIndex + this.accounts.length) % this.accounts.length;
314
+ }
315
+ advanceCursor(account) {
316
+ const index = this.accounts.indexOf(account);
317
+ this.currentIndex = (index + 1) % this.accounts.length;
318
+ }
319
+ proposedExpiry(account, durationMs) {
320
+ if (this.findById(account.id) !== account)
321
+ return undefined;
322
+ if (!Number.isFinite(durationMs) || durationMs <= 0)
323
+ return undefined;
324
+ const expiry = this.now() + durationMs;
325
+ return Number.isFinite(expiry) ? expiry : undefined;
326
+ }
327
+ cooldownsFor(account) {
328
+ let state = this.cooldowns.get(account);
329
+ if (!state) {
330
+ state = { globalUntil: 0, rateLimitedUntil: 0, bucketUntil: new Map() };
331
+ this.cooldowns.set(account, state);
332
+ }
333
+ return state;
334
+ }
335
+ /** Returns true when an active cooldown scope just expired. */
336
+ clearExpiredCooldownState(account) {
337
+ const state = this.cooldowns.get(account);
338
+ if (!state)
339
+ return false;
340
+ const now = this.now();
341
+ let recovered = false;
342
+ if (state.rateLimitedUntil > 0 && state.rateLimitedUntil <= now)
343
+ state.rateLimitedUntil = 0;
344
+ if (state.globalUntil > 0 && state.globalUntil <= now) {
345
+ state.globalUntil = 0;
346
+ recovered = true;
347
+ }
348
+ for (const [limitId, until] of state.bucketUntil) {
349
+ if (until <= now) {
350
+ state.bucketUntil.delete(limitId);
351
+ recovered = true;
352
+ }
353
+ }
354
+ if (state.globalUntil === 0 && state.bucketUntil.size === 0)
355
+ this.cooldowns.delete(account);
356
+ return recovered;
357
+ }
358
+ createLease(account, fallback) {
359
+ this.inFlight.set(account.id, this.getInFlight(account.id) + 1);
360
+ account.requestCount++;
361
+ account.lastUsed = this.now();
362
+ let released = false;
363
+ return {
364
+ account,
365
+ fallback,
366
+ release: () => {
367
+ if (released)
368
+ return;
369
+ released = true;
370
+ if (this.findById(account.id) !== account)
371
+ return;
372
+ const remaining = Math.max(0, this.getInFlight(account.id) - 1);
373
+ if (remaining === 0)
374
+ this.inFlight.delete(account.id);
375
+ else
376
+ this.inFlight.set(account.id, remaining);
377
+ },
378
+ };
379
+ }
380
+ }
381
+ function trustworthyResetMs(resetAtSeconds, nowMs) {
382
+ if (!Number.isFinite(resetAtSeconds) || resetAtSeconds <= 0)
383
+ return undefined;
384
+ const resetAtMs = Math.floor(resetAtSeconds) * 1_000;
385
+ if (resetAtMs <= nowMs)
386
+ return undefined;
387
+ return resetAtMs - nowMs <= MAX_TRUSTED_RATE_LIMIT_RESET_MS ? resetAtMs : undefined;
388
+ }
389
+ function capNormalizedUtilization(utilization, capPercent) {
390
+ const cap = Number.isFinite(capPercent) ? Math.max(0, capPercent / 100) : 1;
391
+ return cap === 0 ? Number.POSITIVE_INFINITY : utilization / cap;
392
+ }
393
+ function compareTuple(left, right) {
394
+ for (let i = 0; i < left.length; i++) {
395
+ const a = left[i] ?? 0;
396
+ const b = right[i] ?? 0;
397
+ if (a !== b)
398
+ return a - b;
399
+ }
400
+ return 0;
401
+ }
@@ -1,7 +1,64 @@
1
+ import { decodeOpenAIPlan } from "./usage.js";
1
2
  const TOKEN_ENDPOINT = "https://auth.openai.com/oauth/token";
2
3
  const REFRESH_BUFFER_MS = 10 * 60 * 1000;
3
4
  const CHECK_INTERVAL_MS = 5 * 60 * 1000;
4
5
  const refreshLocks = new Map();
6
+ /**
7
+ * Accounts whose most recently rotated credentials have NOT been confirmed
8
+ * on disk. Identity-keyed (by object reference, not `account.id`) so a
9
+ * deleted-then-re-added account can never inherit another object's dirty
10
+ * state just because it reused the same id, and a `WeakSet` so an account
11
+ * that's later removed from the pool can't leak here.
12
+ */
13
+ const pendingCredentialWrites = new WeakSet();
14
+ /** True when `account` has a rotated credential that has not yet been durably persisted. */
15
+ export function hasPendingCredentialWrite(account) {
16
+ return pendingCredentialWrites.has(account);
17
+ }
18
+ /**
19
+ * Record that every account in `accounts` has reached disk.
20
+ *
21
+ * A whole-pool write makes all of them durable, not just whichever account
22
+ * prompted it — and those writes do not all come from a refresh. Adding,
23
+ * patching, or deleting an account rewrites the same file from the same live
24
+ * array, so a rotation that failed to persist earlier is on disk once any of
25
+ * those succeed. Without this, health keeps reporting `credentialsPendingWrite`
26
+ * for an account whose credentials are already saved, and later requests keep
27
+ * retrying a write that has already landed.
28
+ *
29
+ * Call it only after the write has actually returned.
30
+ */
31
+ export function markOpenAICredentialsPersisted(accounts) {
32
+ for (const account of accounts)
33
+ pendingCredentialWrites.delete(account);
34
+ }
35
+ /**
36
+ * Centralizes every write of the account pool to disk so every caller shares
37
+ * the same durability bookkeeping: on success, clears the account's pending
38
+ * flag; on a throw (e.g. disk full, a bad custom `--accounts` path), (re)sets
39
+ * it and logs, but never propagates — the caller's fresh in-memory token
40
+ * remains usable even though it isn't on disk yet.
41
+ *
42
+ * Residual risk: if the process crashes before any retry of a dirty account
43
+ * succeeds, the rotated refresh token is lost from disk — and since OpenAI
44
+ * already invalidated the old one, that forces re-authentication. The retry
45
+ * cadence (the very next request via `prepareOpenAIAccountForRequest`, or at
46
+ * worst the refresh loop's next ≤5-minute tick) bounds how long that window
47
+ * stays open, but does not close it.
48
+ */
49
+ function persistCredentials(account, allAccounts, saveAccounts) {
50
+ try {
51
+ saveAccounts(allAccounts);
52
+ markOpenAICredentialsPersisted(allAccounts);
53
+ pendingCredentialWrites.delete(account);
54
+ return true;
55
+ }
56
+ catch (error) {
57
+ pendingCredentialWrites.add(account);
58
+ console.error(error);
59
+ return false;
60
+ }
61
+ }
5
62
  export function needsOpenAIRefresh(account) {
6
63
  return account.expiresAt - Date.now() < REFRESH_BUFFER_MS;
7
64
  }
@@ -19,17 +76,45 @@ export async function refreshOpenAISubscriptionToken(account) {
19
76
  }
20
77
  }
21
78
  export async function prepareOpenAIAccountForRequest(account, allAccounts, saveAccounts) {
22
- if (!needsOpenAIRefresh(account))
79
+ if (!needsOpenAIRefresh(account)) {
80
+ // No refresh due, but a previous rotation from this account never made it
81
+ // to disk (e.g. a transient disk-full). This is the retry path: piggyback
82
+ // on this otherwise-idle request to flush the still-current in-memory
83
+ // token, without blocking or failing the request either way.
84
+ if (hasPendingCredentialWrite(account))
85
+ persistCredentials(account, allAccounts, saveAccounts);
23
86
  return true;
87
+ }
88
+ // The refreshed token is already live in memory, so a persistence failure
89
+ // must not fail this request — it only means the new token isn't on disk
90
+ // yet. `persistCredentials` marks the account dirty so the write is retried
91
+ // (here, or by the refresh loop) instead of silently losing the rotation.
92
+ return refreshAndPersistOpenAIAccount(account, allAccounts, saveAccounts);
93
+ }
94
+ /**
95
+ * Refreshes `account`'s token if needed and persists the pool. Used both by
96
+ * `prepareOpenAIAccountForRequest`'s refresh-due branch and by callers that
97
+ * want to force a refresh outside of that gate (e.g. reacting to an upstream
98
+ * 401). Always returns the refresh outcome — a persistence failure never
99
+ * turns a successful refresh into a `false` result.
100
+ */
101
+ export async function refreshAndPersistOpenAIAccount(account, allAccounts, saveAccounts) {
24
102
  const ok = await refreshOpenAISubscriptionToken(account);
25
103
  if (ok)
26
- saveAccounts(allAccounts);
104
+ persistCredentials(account, allAccounts, saveAccounts);
27
105
  return ok;
28
106
  }
29
107
  export function startOpenAIRefreshLoop(accounts, saveAccounts) {
30
108
  const check = async () => {
31
109
  for (const account of accounts) {
32
- await prepareOpenAIAccountForRequest(account, accounts, saveAccounts);
110
+ // One account's refresh throwing must not skip every account after it
111
+ // in this tick — isolate failures per-account.
112
+ try {
113
+ await prepareOpenAIAccountForRequest(account, accounts, saveAccounts);
114
+ }
115
+ catch (error) {
116
+ console.error(error);
117
+ }
33
118
  }
34
119
  };
35
120
  const timer = setInterval(() => { check().catch(console.error); }, CHECK_INTERVAL_MS);
@@ -41,16 +126,64 @@ async function doRefresh(account) {
41
126
  grant_type: "refresh_token",
42
127
  refresh_token: account.refreshToken,
43
128
  });
44
- const res = await fetch(TOKEN_ENDPOINT, {
45
- method: "POST",
46
- headers: { "Content-Type": "application/x-www-form-urlencoded" },
47
- body: body.toString(),
48
- });
49
- if (!res.ok)
129
+ let data;
130
+ try {
131
+ const res = await fetch(TOKEN_ENDPOINT, {
132
+ method: "POST",
133
+ headers: { "Content-Type": "application/x-www-form-urlencoded" },
134
+ body: body.toString(),
135
+ });
136
+ if (!res.ok)
137
+ return false;
138
+ data = await res.json();
139
+ }
140
+ catch {
141
+ // Network failure (or malformed response body) must resolve to `false`,
142
+ // exactly like a non-ok HTTP response — never propagate as a rejection.
143
+ return false;
144
+ }
145
+ // A 200 with an unusable payload is a failed refresh, not a successful one.
146
+ // Writing it through would leave `expiresAt` as NaN, which then reads as
147
+ // "never needs refreshing" in `needsOpenAIRefresh` and permanently strands
148
+ // the account on a broken token.
149
+ if (typeof data?.access_token !== "string" || data.access_token.length === 0)
150
+ return false;
151
+ // The lifetime has to be positive and has to still name a finite instant
152
+ // once converted. A zero or negative `expires_in` would report success on a
153
+ // token that is already due for another refresh, so every request re-enters
154
+ // the refresh path; a value big enough to overflow the multiplication would
155
+ // set `expiresAt` to Infinity, which `needsOpenAIRefresh` can never reach —
156
+ // the same permanent strand as NaN, from the opposite direction.
157
+ if (typeof data.expires_in !== "number" || !Number.isFinite(data.expires_in))
158
+ return false;
159
+ if (data.expires_in <= 0)
160
+ return false;
161
+ const expiresAt = Date.now() + data.expires_in * 1000;
162
+ if (!Number.isFinite(expiresAt))
50
163
  return false;
51
- const data = await res.json();
52
164
  account.accessToken = data.access_token;
53
165
  account.refreshToken = data.refresh_token ?? account.refreshToken;
54
- account.expiresAt = Date.now() + data.expires_in * 1000;
166
+ account.expiresAt = expiresAt;
167
+ // A successful refresh recovers an account the pool previously excluded for
168
+ // being unhealthy (e.g. after a prior failed refresh). Without this, the pool's
169
+ // hard `!healthy` block means the account never gets acquired again — and thus
170
+ // never gets another chance to refresh — so it stays excluded until restart.
171
+ const runtime = account;
172
+ if (runtime.healthy !== undefined)
173
+ runtime.healthy = true;
174
+ if (runtime.consecutiveErrors !== undefined)
175
+ runtime.consecutiveErrors = 0;
176
+ if (runtime.lastRefresh !== undefined)
177
+ runtime.lastRefresh = Date.now();
178
+ // The rotated access token can carry a different plan than the one decoded
179
+ // at account creation (e.g. a Plus->Pro upgrade). Mirrors createOpenAIAccount's
180
+ // semantics: only overwrite when the new token actually decodes a plan claim —
181
+ // an undecodable token leaves the previously known plan in place rather than
182
+ // erasing it, since a missing claim means "unknown", not "no plan".
183
+ if (runtime.rateLimits) {
184
+ const plan = decodeOpenAIPlan(account.accessToken);
185
+ if (plan)
186
+ runtime.rateLimits.plan = plan;
187
+ }
55
188
  return true;
56
189
  }