maxpool 1.5.11 → 1.5.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "maxpool",
3
- "version": "1.5.11",
3
+ "version": "1.5.13",
4
4
  "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
@@ -1,4 +1,4 @@
1
- import { refreshAccessToken, isTokenExpiringSoon } from './oauth.js';
1
+ import { refreshAccessToken, isTokenExpiringSoon, modelFamily } from './oauth.js';
2
2
 
3
3
  // Bounded re-poll hold for an account blocked ONLY by a transient, self-clearing
4
4
  // condition whose exact recovery time is unknown: (a) a weekly-critical account
@@ -36,8 +36,10 @@ function emptyQuota() {
36
36
  unified7dRaw: null, // upstream-reported utilization before display clamp
37
37
  unified5hReset: null, // ms timestamp
38
38
  unified7dReset: null, // ms timestamp
39
- unified7dSonnet: null, // Sonnet-specific weekly utilization (from usage probe)
40
- unified7dSonnetReset: null, // ms timestamp
39
+ // Per-model weekly sub-limits Anthropic enforces SEPARATELY from the unified
40
+ // weekly (e.g. Fable can be 100% while unified is 56%). Keyed by model family:
41
+ // { fable:{utilization,resetAt,severity,isActive}, opus:{...}, sonnet:{...} }.
42
+ scopedWeekly: {},
41
43
  unifiedStatus: null, // allowed | allowed_warning | rejected
42
44
  resetsAt: null,
43
45
  };
@@ -91,7 +93,7 @@ const FIVE_HOUR_MS = 5 * 60 * 60 * 1000;
91
93
  // (probing, requalify, rateLimitedUntil) and credentials are intentionally
92
94
  // excluded. A stale restored window is wiped on first use by _clearExpiredQuotas.
93
95
  const PERSISTED_QUOTA_FIELDS = [
94
- 'unified5h', 'unified7d', 'unified5hReset', 'unified7dReset', 'unifiedStatus',
96
+ 'unified5h', 'unified7d', 'unified5hReset', 'unified7dReset', 'unifiedStatus', 'scopedWeekly',
95
97
  'tokensLimit', 'tokensRemaining', 'requestsLimit', 'requestsRemaining', 'resetsAt',
96
98
  ];
97
99
 
@@ -310,7 +312,7 @@ export class AccountManager {
310
312
  // finite hold. It must NEVER contribute the WEEKLY reset (days) — that is the
311
313
  // multi-day-hang the bounded path fences off; see the !fairnessOnlyBlock
312
314
  // guards on the weekly branches below.
313
- if (!fairnessOnlyBlock && this._isAvailable(account, { allowWeeklyReserve: true })) {
315
+ if (!fairnessOnlyBlock && this._isAvailable(account, { allowWeeklyReserve: true, model: requestInfo.model })) {
314
316
  return {
315
317
  available: true,
316
318
  retryAfterMs: 0,
@@ -319,15 +321,15 @@ export class AccountManager {
319
321
  matchingRoutes,
320
322
  };
321
323
  }
322
- if (fairnessOnlyBlock && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
324
+ if (fairnessOnlyBlock && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true, model: requestInfo.model })) {
323
325
  soonestBoundedHold = Math.min(soonestBoundedHold, BOUNDED_REPOLL_HOLD_MS);
324
326
  if (!boundedHoldCause) boundedHoldCause = 'queued_behind_fairness';
325
327
  continue;
326
328
  }
327
329
 
328
- const retry = this._retryInfo(account);
330
+ const retry = this._retryInfo(account, requestInfo.model);
329
331
  note(retry.cause);
330
- if (!fairnessOnlyBlock && retry.weeklyCritical && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
332
+ if (!fairnessOnlyBlock && retry.weeklyCritical && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true, model: requestInfo.model })) {
331
333
  return {
332
334
  available: true,
333
335
  retryAfterMs: 0,
@@ -434,7 +436,7 @@ export class AccountManager {
434
436
  return weeklyPasses.some(options => this.accounts.some(account => {
435
437
  if (excludedIndexes.has(account.index)) return false;
436
438
  if (!this._matchesRequest(account, profile, requestInfo)) return false;
437
- return this._isAvailable(account, options);
439
+ return this._isAvailable(account, { ...options, model: requestInfo.model });
438
440
  }));
439
441
  }
440
442
 
@@ -452,7 +454,7 @@ export class AccountManager {
452
454
  const weight = Math.max(1, Number(requestInfo.weight) || 1);
453
455
  const upstreamThrottleProbe = account.type !== 'provider' && this._claimUpstreamThrottleProbe();
454
456
  if (requestInfo.sessionKey) {
455
- this._bindSession(requestInfo.sessionKey, account);
457
+ this._bindSession(requestInfo.sessionKey, account, requestInfo.model);
456
458
  }
457
459
  account.inFlight++;
458
460
  account.activeWeight += weight;
@@ -492,6 +494,11 @@ export class AccountManager {
492
494
 
493
495
  if (outcome.neutral) return;
494
496
 
497
+ // A per-model weekly cap is NOT an account failure — the account is healthy for
498
+ // every other model. Don't poison its failure counters / scoring penalty (mirror
499
+ // the network-blip carve-out); the scoped bench is already set in markRateLimited.
500
+ if (outcome.error === 'model_rate_limited') return;
501
+
495
502
  if (outcome.success) {
496
503
  account.completedRequests++;
497
504
  account.consecutiveFailures = 0;
@@ -558,6 +565,21 @@ export class AccountManager {
558
565
  };
559
566
  }
560
567
 
568
+ // True (with the scoped reset) when THIS request's model family has hit its
569
+ // per-model weekly sub-limit on this account — a block SEPARATE from the unified
570
+ // weekly (Fable can be 100% while unified is 56%). A model-agnostic request (no
571
+ // family) or an account with no scoped data fails OPEN (returns null), matching
572
+ // prior behavior. Gate: is_active AND (severity critical OR util ≥ exhausted).
573
+ _scopedExhausted(account, model) {
574
+ const fam = modelFamily(model);
575
+ if (!fam) return null;
576
+ const e = account.quota?.scopedWeekly?.[fam];
577
+ if (!e || e.isActive === false) return null;
578
+ const exhausted = e.severity === 'critical'
579
+ || (e.utilization != null && e.utilization >= this.scheduler.weeklyExhaustedThreshold);
580
+ return exhausted ? { resetAt: e.resetAt || null, family: fam } : null;
581
+ }
582
+
561
583
  _isAvailable(account, options = {}) {
562
584
  if (!account) return false;
563
585
  if (!account.enabled) return false;
@@ -606,6 +628,11 @@ export class AccountManager {
606
628
  if (weeklyState === 'critical' && !options.allowWeeklyCritical) return false;
607
629
  if (weeklyState === 'reserve' && !options.allowWeeklyReserve) return false;
608
630
 
631
+ // Per-model weekly cap for THIS request's model — unavailable for this request
632
+ // (but the account still serves its other models). options.model is threaded in
633
+ // by request-path callers; model-agnostic call sites skip this gate.
634
+ if (options.model && this._scopedExhausted(account, options.model)) return false;
635
+
609
636
  return true;
610
637
  }
611
638
 
@@ -873,6 +900,18 @@ export class AccountManager {
873
900
  changed = true;
874
901
  }
875
902
 
903
+ // Expire per-model weekly sub-limits on their own reset (a family whose scoped
904
+ // window has passed is usable again for that model, independent of unified).
905
+ if (q.scopedWeekly && typeof q.scopedWeekly === 'object') {
906
+ for (const [fam, e] of Object.entries(q.scopedWeekly)) {
907
+ if (e && e.resetAt && now >= e.resetAt) {
908
+ console.log(`[Maxpool] Account "${account.name}" ${fam} weekly sub-limit reset`);
909
+ delete q.scopedWeekly[fam];
910
+ changed = true;
911
+ }
912
+ }
913
+ }
914
+
876
915
  // Clear expired standard quotas
877
916
  if (q.resetsAt && now >= new Date(q.resetsAt).getTime()) {
878
917
  q.tokensRemaining = null;
@@ -1028,8 +1067,8 @@ export class AccountManager {
1028
1067
  if (account.index === bound.index) continue;
1029
1068
  if (excludedIndexes.has(account.index)) continue;
1030
1069
  if (!this._matchesRequest(account, profile, requestInfo)) continue;
1031
- // Genuinely-healthy alternatives only (normal/soft/unknown weekly).
1032
- if (!this._isAvailable(account, { allowWeeklyReserve: false, allowWeeklyCritical: false })) continue;
1070
+ // Genuinely-healthy alternatives only (normal/soft/unknown weekly + model headroom).
1071
+ if (!this._isAvailable(account, { allowWeeklyReserve: false, allowWeeklyCritical: false, model: requestInfo.model })) continue;
1033
1072
  const score = this._scoreAccount(account, requestInfo, scoringCtx);
1034
1073
  if (score < bestScore) {
1035
1074
  bestScore = score;
@@ -1043,7 +1082,7 @@ export class AccountManager {
1043
1082
  && bestTier < boundTier;
1044
1083
  }
1045
1084
 
1046
- _retryInfo(account) {
1085
+ _retryInfo(account, model = null) {
1047
1086
  const now = Date.now();
1048
1087
  const q = account.quota || {};
1049
1088
 
@@ -1056,6 +1095,14 @@ export class AccountManager {
1056
1095
  if (account.status === 'error') return { cause: 'error', retryAt: null, queueable: false };
1057
1096
  if (account.status === 'exhausted') return { cause: 'exhausted', retryAt: null, queueable: false };
1058
1097
 
1098
+ // Per-model weekly cap for THIS request's model: a hard block until the scoped
1099
+ // weekly reset (days), exactly like weekly_exhausted but scoped to one model —
1100
+ // reuse that cause so the oracle's weekly-reset branches hold on the scoped
1101
+ // reset consistently (agreeing with _isAvailable's model gate above; the oracle
1102
+ // still returns available:true off any SIBLING account with model headroom).
1103
+ const scoped = model ? this._scopedExhausted(account, model) : null;
1104
+ if (scoped) return { cause: 'weekly_exhausted', retryAt: scoped.resetAt, queueable: false, modelScoped: true };
1105
+
1059
1106
  // RAW weekly state, so the retry oracle agrees with _isAvailable's raw gate.
1060
1107
  // (Pace must NOT classify a raw-healthy account as weekly_critical here, or
1061
1108
  // the queue keys on a far-future reset instead of the account's real
@@ -1169,7 +1216,7 @@ export class AccountManager {
1169
1216
  const pinned = this.accounts.find(a => a.name === requestInfo.pinnedAccountName);
1170
1217
  if (pinned && !excludedIndexes.has(pinned.index)
1171
1218
  && this._matchesRequest(pinned, profile, requestInfo)
1172
- && this._isAvailable(pinned, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
1219
+ && this._isAvailable(pinned, { allowWeeklyReserve: true, allowWeeklyCritical: true, model: requestInfo.model })) {
1173
1220
  this.currentIndex = pinned.index;
1174
1221
  return pinned;
1175
1222
  }
@@ -1182,7 +1229,7 @@ export class AccountManager {
1182
1229
  { allowWeeklyReserve: true, allowWeeklyCritical: false },
1183
1230
  { allowWeeklyReserve: true, allowWeeklyCritical: true },
1184
1231
  ];
1185
- if (preferredPasses.some(options => this._isAvailable(preferred, options))) {
1232
+ if (preferredPasses.some(options => this._isAvailable(preferred, { ...options, model: requestInfo.model }))) {
1186
1233
  this.currentIndex = preferred.index;
1187
1234
  return preferred;
1188
1235
  }
@@ -1217,7 +1264,7 @@ export class AccountManager {
1217
1264
  const account = this.accounts[idx];
1218
1265
  if (excludedIndexes.has(account.index)) continue;
1219
1266
  if (!this._matchesRequest(account, profile, requestInfo)) continue;
1220
- if (!this._isAvailable(account, weeklyOptions)) continue;
1267
+ if (!this._isAvailable(account, { ...weeklyOptions, model: requestInfo.model })) continue;
1221
1268
 
1222
1269
  const priority = Number.isFinite(account.priority) ? account.priority : 0;
1223
1270
  const score = this._scoreAccount(account, requestInfo, scoringCtx);
@@ -1296,11 +1343,11 @@ export class AccountManager {
1296
1343
  if (!account) return null;
1297
1344
  if (excludedIndexes.has(account.index)) return null;
1298
1345
  if (!this._matchesRequest(account, profile, requestInfo)) return null;
1299
- if (!this._isAvailable(account, options)) return null;
1346
+ if (!this._isAvailable(account, { ...options, model: requestInfo.model })) return null;
1300
1347
  return account;
1301
1348
  }
1302
1349
 
1303
- _bindSession(sessionKey, account) {
1350
+ _bindSession(sessionKey, account, model = null) {
1304
1351
  const priority = this._priority(account);
1305
1352
  const binding = this._sessionBinding(sessionKey) || {
1306
1353
  homeName: account.name,
@@ -1317,8 +1364,11 @@ export class AccountManager {
1317
1364
  // _boundAccount (which prefers homeName) doesn't snap the session straight
1318
1365
  // back to the hot home next request. If the old home is UNAVAILABLE this is a
1319
1366
  // FAILOVER → keep homeName so the session returns to it once it recovers.
1367
+ // Model-aware: a move off a home that's capped for THIS model (but healthy
1368
+ // for others) is a FAILOVER, not a by-choice rebalance — keep homeName so the
1369
+ // session snaps back once the model's scoped cap resets.
1320
1370
  const oldHome = this.accounts.find(a => a.name === binding.homeName);
1321
- if (oldHome && this._isAvailable(oldHome, { allowWeeklyReserve: true })) {
1371
+ if (oldHome && this._isAvailable(oldHome, { allowWeeklyReserve: true, model })) {
1322
1372
  binding.homeName = account.name;
1323
1373
  }
1324
1374
  }
@@ -1349,7 +1399,7 @@ export class AccountManager {
1349
1399
  if (excludedIndexes.has(account.index)) return false;
1350
1400
  if (!this._matchesRequest(account, profile, requestInfo)) return false;
1351
1401
  const priority = this._priority(account);
1352
- return priority < boundPriority && this._isAvailable(account, { allowWeeklyReserve: true });
1402
+ return priority < boundPriority && this._isAvailable(account, { allowWeeklyReserve: true, model: requestInfo.model });
1353
1403
  });
1354
1404
  }
1355
1405
 
@@ -1619,9 +1669,21 @@ export class AccountManager {
1619
1669
  if (usage.sevenDay.utilization != null) q.unified7d = clamp01(usage.sevenDay.utilization);
1620
1670
  if (usage.sevenDay.resetAt != null) q.unified7dReset = usage.sevenDay.resetAt;
1621
1671
  }
1622
- if (usage.sevenDaySonnet) {
1623
- if (usage.sevenDaySonnet.utilization != null) q.unified7dSonnet = clamp01(usage.sevenDaySonnet.utilization);
1624
- if (usage.sevenDaySonnet.resetAt != null) q.unified7dSonnetReset = usage.sevenDaySonnet.resetAt;
1672
+ // Per-model weekly sub-limits (Fable, Opus, ...). Replace wholesale with the
1673
+ // fresh probe set so a family that dropped out of the response doesn't linger
1674
+ // stale; expiry on reset is a backstop for the between-probe window.
1675
+ if (usage.scopedWeekly && typeof usage.scopedWeekly === 'object') {
1676
+ const fresh = {};
1677
+ for (const [fam, e] of Object.entries(usage.scopedWeekly)) {
1678
+ if (!e) continue;
1679
+ fresh[fam] = {
1680
+ utilization: e.utilization != null ? clamp01(e.utilization) : null,
1681
+ resetAt: e.resetAt != null ? e.resetAt : null,
1682
+ severity: e.severity || null,
1683
+ isActive: e.isActive !== false,
1684
+ };
1685
+ }
1686
+ q.scopedWeekly = fresh;
1625
1687
  }
1626
1688
 
1627
1689
  // If we just learned this account's weekly window while probing, re-evaluate
@@ -1745,6 +1807,27 @@ export class AccountManager {
1745
1807
  const account = this.accounts[accountIndex];
1746
1808
  if (!account) return;
1747
1809
  const retryAfter = clampRetryAfterSeconds(retryAfterSeconds);
1810
+
1811
+ // Model-scoped rate-limit: a per-model weekly cap (e.g. Fable) rejected this
1812
+ // request while the account's UNIFIED quota is healthy. Record ONLY the scoped
1813
+ // exhaustion — do NOT bench the whole account (its other models still have
1814
+ // headroom). The just-429'd request fails over to a headroom account via the
1815
+ // per-request excludedIndexes; future same-model requests are steered by the
1816
+ // scoped gate; future other-model requests keep using this account.
1817
+ if (options.modelScope) {
1818
+ account.quota.scopedWeekly = account.quota.scopedWeekly || {};
1819
+ account.quota.scopedWeekly[options.modelScope] = {
1820
+ utilization: 1,
1821
+ resetAt: Date.now() + (retryAfter * 1000),
1822
+ severity: 'critical',
1823
+ isActive: true,
1824
+ };
1825
+ account.lastStatus = options.status || 429;
1826
+ account.lastErrorAt = Date.now();
1827
+ console.log(`[Maxpool] Account "${account.name}" ${options.modelScope} weekly limit hit — scoped bench ${retryAfter}s (account stays active for other models)`);
1828
+ return;
1829
+ }
1830
+
1748
1831
  account.status = 'throttled';
1749
1832
  account.rateLimitedUntil = Date.now() + (retryAfter * 1000);
1750
1833
  account.lastStatus = options.status || 429;
@@ -2099,6 +2182,16 @@ export class AccountManager {
2099
2182
  // could otherwise pin the account unavailable until the first live response.
2100
2183
  if (account.quota.unified5hReset == null) account.quota.unified5h = null;
2101
2184
  if (account.quota.unified7dReset == null) account.quota.unified7d = null;
2185
+ // Drop restored scoped entries lacking a clearable reset window — they can't
2186
+ // be expired by _clearExpiredQuotas and would otherwise pin an account
2187
+ // unavailable-for-family until the first live probe.
2188
+ if (account.quota.scopedWeekly && typeof account.quota.scopedWeekly === 'object') {
2189
+ for (const [fam, e] of Object.entries(account.quota.scopedWeekly)) {
2190
+ if (!e || e.resetAt == null) delete account.quota.scopedWeekly[fam];
2191
+ }
2192
+ } else {
2193
+ account.quota.scopedWeekly = {};
2194
+ }
2102
2195
  // We already know this account's weekly window, so it isn't "probing".
2103
2196
  if (account.quota.unified7dReset != null) account.probing = false;
2104
2197
  }
package/src/index.js CHANGED
@@ -156,10 +156,20 @@ async function supervisorCommand() {
156
156
  // supervisor's acceptor is only LIVE during the brief cutover gap (it stops
157
157
  // once a worker confirms it is the sole acceptor). A bare handler is required
158
158
  // so the rare gap-accepted socket isn't left dangling.
159
- const relistenMaster = () => new Promise((resolve, reject) => {
159
+ const relistenMaster = (retriesOnBusy = 20) => new Promise((resolve, reject) => {
160
160
  if (masterServer.listening) { resolve(); return; }
161
- const onErr = err => { masterServer.removeListener('listening', onListen); reject(err); };
162
161
  const onListen = () => { masterServer.removeListener('error', onErr); resolve(); };
162
+ const onErr = err => {
163
+ masterServer.removeListener('listening', onListen);
164
+ // A just-SIGKILLed worker (seamless-reload fallback) may not have released the
165
+ // listening fd yet — SIGKILL is async; the port frees within a few ms. Retry
166
+ // bounded (~1s) rather than crashing the supervisor via handleServerListenError.
167
+ if (err.code === 'EADDRINUSE' && retriesOnBusy > 0) {
168
+ setTimeout(() => relistenMaster(retriesOnBusy - 1).then(resolve, reject), 50);
169
+ return;
170
+ }
171
+ reject(err);
172
+ };
163
173
  masterServer.once('error', onErr);
164
174
  masterServer.once('listening', onListen);
165
175
  masterServer.listen(port, host);
@@ -347,6 +357,19 @@ async function supervisorCommand() {
347
357
 
348
358
  try {
349
359
  while (true) {
360
+ // Ensure the master socket is LISTENING before handing it to a cold worker.
361
+ // After the previous primary worker sent MSG_PRIMARY the supervisor ran
362
+ // closeMasterAccept() (masterServer.close()), and the exiting worker released
363
+ // the last fd — so on every respawn masterServer is closed. Without this,
364
+ // spawnWorker sends a dead handle, the worker ignores it (falsy-handle guard),
365
+ // and never becomes primary → the interactive `r` restart hangs blank. Idempotent
366
+ // (no-op when already listening, e.g. the very first iteration after line 172).
367
+ try {
368
+ await relistenMaster();
369
+ } catch (err) {
370
+ handleServerListenError(err, host, port);
371
+ return;
372
+ }
350
373
  const worker = spawnWorker({ reload: false });
351
374
  const startedAt = Date.now();
352
375
  const result = await superviseTurn(worker);
@@ -725,7 +748,10 @@ async function serverWorkerCommand() {
725
748
  if (draining) return;
726
749
  // Interactive (live TUI) → full cold restart so the fresh worker re-renders the
727
750
  // TUI; headless/service → zero-downtime seamless baton (nothing visual to lose).
728
- if (reloadStrategy({ supervised, useTUI }) === 'seamless') {
751
+ // Test-only: force the interactive cold-restart path headless (a pty can't be
752
+ // allocated in-suite), so the exit-75 respawn is exercised without a real TUI.
753
+ const forceCold = process.env.MAXPOOL_TEST_FORCE_COLD_RESTART === '1';
754
+ if (!forceCold && reloadStrategy({ supervised, useTUI }) === 'seamless') {
729
755
  try {
730
756
  process.send({ type: MSG_RELOAD_REQUEST });
731
757
  return;
@@ -991,6 +1017,12 @@ async function serverWorkerCommand() {
991
1017
  // Cold start: take the socket and go primary immediately.
992
1018
  await listenOnHandle(handle);
993
1019
  await becomePrimary({ viaTakeover: false });
1020
+ } else if (msg?.type === MSG_LISTEN && !handle) {
1021
+ // Defense-in-depth for the stale-handle hang: if the supervisor ever hands
1022
+ // a cold worker a dead/absent socket, fail LOUD and exit-75 so it respawns —
1023
+ // never sit alive-but-not-primary (the blank-screen `r`-restart hang).
1024
+ console.error('[Maxpool] Cold worker received MSG_LISTEN with no socket handle — exiting for respawn.');
1025
+ process.exit(SERVER_RESTART_EXIT_CODE);
994
1026
  } else if (msg?.type === MSG_PROBE_READY) {
995
1027
  // Headless reload worker: we've booted the new code and restored quota
996
1028
  // in memory. Confirm readiness (we are NOT yet accepting / writing).
@@ -1013,9 +1045,13 @@ async function serverWorkerCommand() {
1013
1045
  await releaseBatonAndDrain();
1014
1046
  }
1015
1047
  } catch (err) {
1016
- // A failed reload worker must NOT exit(1) (kills the supervisor loop).
1017
- // Report failure so the supervisor rolls back; stay alive harmlessly.
1018
- console.error(`[Maxpool] Reload worker error: ${err.message}`);
1048
+ console.error(`[Maxpool] Worker message error: ${err.message}`);
1049
+ // A COLD worker that failed to listen / become primary has no baton to roll
1050
+ // back to — it would strand blank. Exit-75 so the supervisor respawns it
1051
+ // (closes the silent-non-primary hang class). A RELOAD worker mid-baton must
1052
+ // NOT exit (that kills the supervisor loop) — report MSG_FAILED so the
1053
+ // supervisor rolls back and the old worker stays primary (zero disruption).
1054
+ if (!isReloadWorker) process.exit(SERVER_RESTART_EXIT_CODE);
1019
1055
  try { process.send({ type: MSG_FAILED, reason: err.message }); } catch { /* ignore */ }
1020
1056
  }
1021
1057
  });
package/src/oauth.js CHANGED
@@ -136,9 +136,34 @@ export async function fetchProfile(accessToken) {
136
136
  const USAGE_URL = 'https://api.anthropic.com/api/oauth/usage';
137
137
  const OAUTH_USAGE_BETA = 'oauth-2025-04-20';
138
138
 
139
+ /** Parse a reset timestamp (epoch-seconds, epoch-ms, or ISO string) to ms, or null. */
140
+ export function parseResetTimestamp(rawReset) {
141
+ if (typeof rawReset === 'number') return rawReset < 1e12 ? rawReset * 1000 : rawReset;
142
+ if (typeof rawReset === 'string') {
143
+ const asNum = Number(rawReset);
144
+ if (Number.isFinite(asNum) && rawReset.trim() !== '') return asNum < 1e12 ? asNum * 1000 : asNum;
145
+ const parsed = Date.parse(rawReset);
146
+ if (Number.isFinite(parsed)) return parsed;
147
+ }
148
+ return null;
149
+ }
150
+
151
+ /** Map a model id or usage display name to a coarse family key, or null. Matches
152
+ * BOTH request models (`claude-fable-5`) and usage `scope.model.display_name`
153
+ * (`Fable`). Unmatched → null (caller treats as never model-scoped). */
154
+ export function modelFamily(model) {
155
+ const s = String(model || '').toLowerCase();
156
+ if (s.includes('fable') || s.includes('mythos')) return 'fable';
157
+ if (s.includes('opus')) return 'opus';
158
+ if (s.includes('sonnet')) return 'sonnet';
159
+ if (s.includes('haiku')) return 'haiku';
160
+ return null;
161
+ }
162
+
139
163
  /** Normalize one usage bucket from the /api/oauth/usage response to
140
164
  * { utilization: 0..1, resetAt: ms-timestamp } (tolerant of field-name and
141
- * percentage/epoch-vs-iso variations). */
165
+ * percentage/epoch-vs-iso variations). Legacy top-level buckets only — the
166
+ * `limits[]` path uses parseLimitEntry (clean 0-100 percent, no ambiguity). */
142
167
  export function normalizeUsageBucket(bucket) {
143
168
  if (!bucket || typeof bucket !== 'object') return null;
144
169
 
@@ -148,25 +173,26 @@ export function normalizeUsageBucket(bucket) {
148
173
  ? (parsedPct > 1 ? parsedPct / 100 : parsedPct)
149
174
  : null;
150
175
 
151
- const rawReset = bucket.resets_at ?? bucket.resetsAt ?? bucket.reset_at ?? bucket.resetAt;
152
- let resetAt = null;
153
- if (typeof rawReset === 'number') {
154
- resetAt = rawReset < 1e12 ? rawReset * 1000 : rawReset;
155
- } else if (typeof rawReset === 'string') {
156
- const asNum = Number(rawReset);
157
- if (Number.isFinite(asNum) && rawReset.trim() !== '') {
158
- resetAt = asNum < 1e12 ? asNum * 1000 : asNum;
159
- } else {
160
- const parsed = Date.parse(rawReset);
161
- if (Number.isFinite(parsed)) resetAt = parsed;
162
- }
163
- }
176
+ return { utilization, resetAt: parseResetTimestamp(bucket.resets_at ?? bucket.resetsAt ?? bucket.reset_at ?? bucket.resetAt) };
177
+ }
164
178
 
165
- return { utilization, resetAt };
179
+ /** Normalize one `limits[]` entry. `percent` is a clean 0-100 integer (unlike the
180
+ * ambiguous top-level buckets), so utilization = percent/100 always. */
181
+ export function parseLimitEntry(l) {
182
+ if (!l || typeof l !== 'object') return null;
183
+ const pct = typeof l.percent === 'number' ? l.percent : parseFloat(l.percent);
184
+ return {
185
+ utilization: Number.isFinite(pct) ? Math.max(0, Math.min(1, pct / 100)) : null,
186
+ resetAt: parseResetTimestamp(l.resets_at ?? l.resetsAt),
187
+ severity: typeof l.severity === 'string' ? l.severity : null,
188
+ isActive: l.is_active !== false,
189
+ };
166
190
  }
167
191
 
168
192
  /** Read an account's quota from the zero-spend /api/oauth/usage endpoint.
169
- * Returns { fiveHour, sevenDay, sevenDaySonnet } buckets, or { error, status }. */
193
+ * Returns { fiveHour, sevenDay, scopedWeekly } buckets, or { error, status }.
194
+ * Per-model + unified limits now live in the `limits[]` array (the top-level
195
+ * seven_day_opus / seven_day_sonnet keys are deprecated → always null). */
170
196
  export async function fetchUsage(accessToken) {
171
197
  try {
172
198
  const res = await fetch(USAGE_URL, {
@@ -189,10 +215,30 @@ export async function fetchUsage(accessToken) {
189
215
  }
190
216
 
191
217
  const data = await res.json();
218
+
219
+ // The `limits[]` array is the authoritative source: `session` (5h), unscoped
220
+ // `weekly_all` (unified 7d), and `weekly_scoped` entries carrying a
221
+ // `scope.model` — the PER-MODEL weekly sub-limits (e.g. Fable) Anthropic
222
+ // enforces separately from the unified weekly. Prefer it over the legacy
223
+ // top-level buckets (whose 0-100 percent misreads a genuine 1% as 100%).
224
+ const scopedWeekly = {};
225
+ let limitsSession = null;
226
+ let limitsWeeklyAll = null;
227
+ for (const l of (Array.isArray(data?.limits) ? data.limits : [])) {
228
+ if (l?.kind === 'weekly_scoped') {
229
+ const fam = l?.scope?.model?.display_name ? modelFamily(l.scope.model.display_name) : null;
230
+ if (fam) scopedWeekly[fam] = parseLimitEntry(l);
231
+ } else if (l?.kind === 'session') {
232
+ limitsSession = parseLimitEntry(l);
233
+ } else if (l?.kind === 'weekly_all') {
234
+ limitsWeeklyAll = parseLimitEntry(l);
235
+ }
236
+ }
237
+
192
238
  return {
193
- fiveHour: normalizeUsageBucket(data?.five_hour),
194
- sevenDay: normalizeUsageBucket(data?.seven_day),
195
- sevenDaySonnet: normalizeUsageBucket(data?.seven_day_sonnet),
239
+ fiveHour: limitsSession || normalizeUsageBucket(data?.five_hour),
240
+ sevenDay: limitsWeeklyAll || normalizeUsageBucket(data?.seven_day),
241
+ scopedWeekly,
196
242
  };
197
243
  } catch (err) {
198
244
  return { error: err.message || String(err), status: null };
package/src/server.js CHANGED
@@ -1,6 +1,7 @@
1
1
  import http from 'node:http';
2
2
  import { writeFile, mkdir } from 'node:fs/promises';
3
3
  import { join } from 'node:path';
4
+ import { modelFamily } from './oauth.js';
4
5
 
5
6
 
6
7
  const HOP_BY_HOP_HEADERS = new Set([
@@ -453,7 +454,7 @@ async function forwardRequest(
453
454
  const errorBody = await readErrorBody(upstreamRes);
454
455
  const retryAfter = parseRetryAfter(upstreamRes.headers.get('retry-after'))
455
456
  || parseProviderRetryAfter(errorBody, account.provider);
456
- const rateLimit = classifyRateLimit(account, rateLimitHeaders, errorBody);
457
+ const rateLimit = classifyRateLimit(account, rateLimitHeaders, errorBody, { model: requestInfo.model, retryAfter });
457
458
  if (rateLimit.scope === 'upstream') {
458
459
  const parsedError = parseJsonError(errorBody);
459
460
  const fingerprint = `429:${rateLimit.fingerprint || overloadFingerprint(errorBody, body)}`;
@@ -552,8 +553,9 @@ async function forwardRequest(
552
553
  status: 429,
553
554
  recordFailure: false,
554
555
  fingerprint: rateLimit.scope === 'unknown' ? rateLimit.fingerprint : null,
556
+ modelScope: rateLimit.modelScope || null,
555
557
  });
556
- accountManager.releaseAccount(lease, { status: 429, error: 'rate_limited' });
558
+ accountManager.releaseAccount(lease, { status: 429, error: rateLimit.modelScope ? 'model_rate_limited' : 'rate_limited' });
557
559
 
558
560
  if (logDir) {
559
561
  logSections.push(`=== RESPONSE 429 — "${account.name}" rate-limited ${retryAfter}s ===\n${formatHeaders(upstreamRes.headers)}`);
@@ -1008,7 +1010,7 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
1008
1010
  return `All ${n} accounts exhausted. Retry in ${retryAfter}s.`;
1009
1011
  }
1010
1012
 
1011
- export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, describeRequest };
1013
+ export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, describeRequest, classifyRateLimit };
1012
1014
 
1013
1015
  async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
1014
1016
  if (!upstreamRes.body) return '';
@@ -1073,7 +1075,7 @@ function parseJsonError(body) {
1073
1075
  }
1074
1076
  }
1075
1077
 
1076
- function classifyRateLimit(account, headers, body) {
1078
+ function classifyRateLimit(account, headers, body, opts = {}) {
1077
1079
  if (account.type === 'provider') return { scope: 'account', fingerprint: null };
1078
1080
 
1079
1081
  const parsed = parseJsonError(body);
@@ -1085,18 +1087,38 @@ function classifyRateLimit(account, headers, body) {
1085
1087
  const tokensRemaining = Number(headers['anthropic-ratelimit-tokens-remaining']);
1086
1088
  const requestsRemaining = Number(headers['anthropic-ratelimit-requests-remaining']);
1087
1089
 
1090
+ // A PER-MODEL weekly sub-limit (e.g. Fable) rejects while the account's UNIFIED
1091
+ // quota is healthy. We can't rely on a captured Fable-429 body shape, so detect
1092
+ // by the CONTRADICTION: a weekly-class (long) reset AND both unified buckets NOT
1093
+ // exhausted AND the request carries a known model family. Tagging modelScope
1094
+ // makes the failover bench only (account, model) — not the whole account (which
1095
+ // still has headroom for its other models). retryAfter is in SECONDS.
1096
+ const fam = modelFamily(opts.model);
1097
+ const retryAfter = Number(opts.retryAfter);
1098
+ // Model-scope requires POSITIVE evidence the unified quota has headroom: at least
1099
+ // one unified utilization header present AND both present ones below the floor. A
1100
+ // rejection with NO utilization headers is treated as account-wide (safer — a
1101
+ // genuine account cap with stripped headers must bench the whole account, not one
1102
+ // model). Neither bucket may be at/above the exhaustion floor.
1103
+ const haveUnifiedEvidence = Number.isFinite(weekly) || Number.isFinite(fiveHour);
1104
+ const unifiedNotExhausted =
1105
+ (!Number.isFinite(weekly) || weekly < 0.985) && (!Number.isFinite(fiveHour) || fiveHour < 0.985);
1106
+ const modelScope =
1107
+ (fam && haveUnifiedEvidence && unifiedNotExhausted && Number.isFinite(retryAfter) && retryAfter >= 30 * 60)
1108
+ ? fam : null;
1109
+
1088
1110
  const quotaHeaderExhaustion =
1089
1111
  unifiedStatus === 'rejected'
1090
1112
  || (Number.isFinite(fiveHour) && fiveHour >= 0.985)
1091
1113
  || (Number.isFinite(weekly) && weekly >= 0.985)
1092
1114
  || (headers['anthropic-ratelimit-tokens-remaining'] != null && tokensRemaining <= 0)
1093
1115
  || (headers['anthropic-ratelimit-requests-remaining'] != null && requestsRemaining <= 0);
1094
- if (quotaHeaderExhaustion) return { scope: 'account', fingerprint: null };
1116
+ if (quotaHeaderExhaustion) return { scope: 'account', fingerprint: null, modelScope };
1095
1117
 
1096
1118
  const quotaBodyExhaustion =
1097
1119
  /\b(account|plan|session|weekly|quota)\b.{0,40}\b(exhausted|limit|exceeded|reached)\b/i.test(message)
1098
1120
  || /\busage\b.{0,40}\b(exhausted|exceeded|reached)\b/i.test(message);
1099
- if (quotaBodyExhaustion) return { scope: 'account', fingerprint: null };
1121
+ if (quotaBodyExhaustion) return { scope: 'account', fingerprint: null, modelScope };
1100
1122
 
1101
1123
  const explicitSharedThrottle =
1102
1124
  message.includes('not your usage limit')
package/src/tui.js CHANGED
@@ -986,6 +986,15 @@ export class TUI {
986
986
  }
987
987
  const weekly = weeklyPolicyText(this.am, a);
988
988
  if (weekly) line += ` ${weekly}`;
989
+ // Per-model weekly caps (e.g. Fable maxed while the unified weekly still has
990
+ // headroom) — surfaced separately so a capped model on an otherwise-healthy
991
+ // account is visible, and routing away from it is explained.
992
+ const exhaustedFloor = this.am.scheduler?.weeklyExhaustedThreshold ?? 0.985;
993
+ const capped = Object.entries(q.scopedWeekly || {})
994
+ .filter(([, e]) => e && e.isActive !== false
995
+ && (e.severity === 'critical' || (e.utilization != null && e.utilization >= exhaustedFloor)))
996
+ .map(([fam]) => fam.charAt(0).toUpperCase() + fam.slice(1));
997
+ if (capped.length) line += ` ${red(`${capped.join(',')} maxed`)}`;
989
998
  line += ` ${dim(loadText(this._accountLoad(a)))}`;
990
999
  return line;
991
1000
  }