maxpool 1.5.11 → 1.5.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +116 -23
- package/src/index.js +42 -6
- package/src/oauth.js +65 -19
- package/src/server.js +28 -6
- package/src/tui.js +9 -0
package/package.json
CHANGED
package/src/account-manager.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { refreshAccessToken, isTokenExpiringSoon } from './oauth.js';
|
|
1
|
+
import { refreshAccessToken, isTokenExpiringSoon, modelFamily } from './oauth.js';
|
|
2
2
|
|
|
3
3
|
// Bounded re-poll hold for an account blocked ONLY by a transient, self-clearing
|
|
4
4
|
// condition whose exact recovery time is unknown: (a) a weekly-critical account
|
|
@@ -36,8 +36,10 @@ function emptyQuota() {
|
|
|
36
36
|
unified7dRaw: null, // upstream-reported utilization before display clamp
|
|
37
37
|
unified5hReset: null, // ms timestamp
|
|
38
38
|
unified7dReset: null, // ms timestamp
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
// Per-model weekly sub-limits Anthropic enforces SEPARATELY from the unified
|
|
40
|
+
// weekly (e.g. Fable can be 100% while unified is 56%). Keyed by model family:
|
|
41
|
+
// { fable:{utilization,resetAt,severity,isActive}, opus:{...}, sonnet:{...} }.
|
|
42
|
+
scopedWeekly: {},
|
|
41
43
|
unifiedStatus: null, // allowed | allowed_warning | rejected
|
|
42
44
|
resetsAt: null,
|
|
43
45
|
};
|
|
@@ -91,7 +93,7 @@ const FIVE_HOUR_MS = 5 * 60 * 60 * 1000;
|
|
|
91
93
|
// (probing, requalify, rateLimitedUntil) and credentials are intentionally
|
|
92
94
|
// excluded. A stale restored window is wiped on first use by _clearExpiredQuotas.
|
|
93
95
|
const PERSISTED_QUOTA_FIELDS = [
|
|
94
|
-
'unified5h', 'unified7d', 'unified5hReset', 'unified7dReset', 'unifiedStatus',
|
|
96
|
+
'unified5h', 'unified7d', 'unified5hReset', 'unified7dReset', 'unifiedStatus', 'scopedWeekly',
|
|
95
97
|
'tokensLimit', 'tokensRemaining', 'requestsLimit', 'requestsRemaining', 'resetsAt',
|
|
96
98
|
];
|
|
97
99
|
|
|
@@ -310,7 +312,7 @@ export class AccountManager {
|
|
|
310
312
|
// finite hold. It must NEVER contribute the WEEKLY reset (days) — that is the
|
|
311
313
|
// multi-day-hang the bounded path fences off; see the !fairnessOnlyBlock
|
|
312
314
|
// guards on the weekly branches below.
|
|
313
|
-
if (!fairnessOnlyBlock && this._isAvailable(account, { allowWeeklyReserve: true })) {
|
|
315
|
+
if (!fairnessOnlyBlock && this._isAvailable(account, { allowWeeklyReserve: true, model: requestInfo.model })) {
|
|
314
316
|
return {
|
|
315
317
|
available: true,
|
|
316
318
|
retryAfterMs: 0,
|
|
@@ -319,15 +321,15 @@ export class AccountManager {
|
|
|
319
321
|
matchingRoutes,
|
|
320
322
|
};
|
|
321
323
|
}
|
|
322
|
-
if (fairnessOnlyBlock && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
|
|
324
|
+
if (fairnessOnlyBlock && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true, model: requestInfo.model })) {
|
|
323
325
|
soonestBoundedHold = Math.min(soonestBoundedHold, BOUNDED_REPOLL_HOLD_MS);
|
|
324
326
|
if (!boundedHoldCause) boundedHoldCause = 'queued_behind_fairness';
|
|
325
327
|
continue;
|
|
326
328
|
}
|
|
327
329
|
|
|
328
|
-
const retry = this._retryInfo(account);
|
|
330
|
+
const retry = this._retryInfo(account, requestInfo.model);
|
|
329
331
|
note(retry.cause);
|
|
330
|
-
if (!fairnessOnlyBlock && retry.weeklyCritical && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
|
|
332
|
+
if (!fairnessOnlyBlock && retry.weeklyCritical && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true, model: requestInfo.model })) {
|
|
331
333
|
return {
|
|
332
334
|
available: true,
|
|
333
335
|
retryAfterMs: 0,
|
|
@@ -434,7 +436,7 @@ export class AccountManager {
|
|
|
434
436
|
return weeklyPasses.some(options => this.accounts.some(account => {
|
|
435
437
|
if (excludedIndexes.has(account.index)) return false;
|
|
436
438
|
if (!this._matchesRequest(account, profile, requestInfo)) return false;
|
|
437
|
-
return this._isAvailable(account, options);
|
|
439
|
+
return this._isAvailable(account, { ...options, model: requestInfo.model });
|
|
438
440
|
}));
|
|
439
441
|
}
|
|
440
442
|
|
|
@@ -452,7 +454,7 @@ export class AccountManager {
|
|
|
452
454
|
const weight = Math.max(1, Number(requestInfo.weight) || 1);
|
|
453
455
|
const upstreamThrottleProbe = account.type !== 'provider' && this._claimUpstreamThrottleProbe();
|
|
454
456
|
if (requestInfo.sessionKey) {
|
|
455
|
-
this._bindSession(requestInfo.sessionKey, account);
|
|
457
|
+
this._bindSession(requestInfo.sessionKey, account, requestInfo.model);
|
|
456
458
|
}
|
|
457
459
|
account.inFlight++;
|
|
458
460
|
account.activeWeight += weight;
|
|
@@ -492,6 +494,11 @@ export class AccountManager {
|
|
|
492
494
|
|
|
493
495
|
if (outcome.neutral) return;
|
|
494
496
|
|
|
497
|
+
// A per-model weekly cap is NOT an account failure — the account is healthy for
|
|
498
|
+
// every other model. Don't poison its failure counters / scoring penalty (mirror
|
|
499
|
+
// the network-blip carve-out); the scoped bench is already set in markRateLimited.
|
|
500
|
+
if (outcome.error === 'model_rate_limited') return;
|
|
501
|
+
|
|
495
502
|
if (outcome.success) {
|
|
496
503
|
account.completedRequests++;
|
|
497
504
|
account.consecutiveFailures = 0;
|
|
@@ -558,6 +565,21 @@ export class AccountManager {
|
|
|
558
565
|
};
|
|
559
566
|
}
|
|
560
567
|
|
|
568
|
+
// True (with the scoped reset) when THIS request's model family has hit its
|
|
569
|
+
// per-model weekly sub-limit on this account — a block SEPARATE from the unified
|
|
570
|
+
// weekly (Fable can be 100% while unified is 56%). A model-agnostic request (no
|
|
571
|
+
// family) or an account with no scoped data fails OPEN (returns null), matching
|
|
572
|
+
// prior behavior. Gate: is_active AND (severity critical OR util ≥ exhausted).
|
|
573
|
+
_scopedExhausted(account, model) {
|
|
574
|
+
const fam = modelFamily(model);
|
|
575
|
+
if (!fam) return null;
|
|
576
|
+
const e = account.quota?.scopedWeekly?.[fam];
|
|
577
|
+
if (!e || e.isActive === false) return null;
|
|
578
|
+
const exhausted = e.severity === 'critical'
|
|
579
|
+
|| (e.utilization != null && e.utilization >= this.scheduler.weeklyExhaustedThreshold);
|
|
580
|
+
return exhausted ? { resetAt: e.resetAt || null, family: fam } : null;
|
|
581
|
+
}
|
|
582
|
+
|
|
561
583
|
_isAvailable(account, options = {}) {
|
|
562
584
|
if (!account) return false;
|
|
563
585
|
if (!account.enabled) return false;
|
|
@@ -606,6 +628,11 @@ export class AccountManager {
|
|
|
606
628
|
if (weeklyState === 'critical' && !options.allowWeeklyCritical) return false;
|
|
607
629
|
if (weeklyState === 'reserve' && !options.allowWeeklyReserve) return false;
|
|
608
630
|
|
|
631
|
+
// Per-model weekly cap for THIS request's model — unavailable for this request
|
|
632
|
+
// (but the account still serves its other models). options.model is threaded in
|
|
633
|
+
// by request-path callers; model-agnostic call sites skip this gate.
|
|
634
|
+
if (options.model && this._scopedExhausted(account, options.model)) return false;
|
|
635
|
+
|
|
609
636
|
return true;
|
|
610
637
|
}
|
|
611
638
|
|
|
@@ -873,6 +900,18 @@ export class AccountManager {
|
|
|
873
900
|
changed = true;
|
|
874
901
|
}
|
|
875
902
|
|
|
903
|
+
// Expire per-model weekly sub-limits on their own reset (a family whose scoped
|
|
904
|
+
// window has passed is usable again for that model, independent of unified).
|
|
905
|
+
if (q.scopedWeekly && typeof q.scopedWeekly === 'object') {
|
|
906
|
+
for (const [fam, e] of Object.entries(q.scopedWeekly)) {
|
|
907
|
+
if (e && e.resetAt && now >= e.resetAt) {
|
|
908
|
+
console.log(`[Maxpool] Account "${account.name}" ${fam} weekly sub-limit reset`);
|
|
909
|
+
delete q.scopedWeekly[fam];
|
|
910
|
+
changed = true;
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
|
|
876
915
|
// Clear expired standard quotas
|
|
877
916
|
if (q.resetsAt && now >= new Date(q.resetsAt).getTime()) {
|
|
878
917
|
q.tokensRemaining = null;
|
|
@@ -1028,8 +1067,8 @@ export class AccountManager {
|
|
|
1028
1067
|
if (account.index === bound.index) continue;
|
|
1029
1068
|
if (excludedIndexes.has(account.index)) continue;
|
|
1030
1069
|
if (!this._matchesRequest(account, profile, requestInfo)) continue;
|
|
1031
|
-
// Genuinely-healthy alternatives only (normal/soft/unknown weekly).
|
|
1032
|
-
if (!this._isAvailable(account, { allowWeeklyReserve: false, allowWeeklyCritical: false })) continue;
|
|
1070
|
+
// Genuinely-healthy alternatives only (normal/soft/unknown weekly + model headroom).
|
|
1071
|
+
if (!this._isAvailable(account, { allowWeeklyReserve: false, allowWeeklyCritical: false, model: requestInfo.model })) continue;
|
|
1033
1072
|
const score = this._scoreAccount(account, requestInfo, scoringCtx);
|
|
1034
1073
|
if (score < bestScore) {
|
|
1035
1074
|
bestScore = score;
|
|
@@ -1043,7 +1082,7 @@ export class AccountManager {
|
|
|
1043
1082
|
&& bestTier < boundTier;
|
|
1044
1083
|
}
|
|
1045
1084
|
|
|
1046
|
-
_retryInfo(account) {
|
|
1085
|
+
_retryInfo(account, model = null) {
|
|
1047
1086
|
const now = Date.now();
|
|
1048
1087
|
const q = account.quota || {};
|
|
1049
1088
|
|
|
@@ -1056,6 +1095,14 @@ export class AccountManager {
|
|
|
1056
1095
|
if (account.status === 'error') return { cause: 'error', retryAt: null, queueable: false };
|
|
1057
1096
|
if (account.status === 'exhausted') return { cause: 'exhausted', retryAt: null, queueable: false };
|
|
1058
1097
|
|
|
1098
|
+
// Per-model weekly cap for THIS request's model: a hard block until the scoped
|
|
1099
|
+
// weekly reset (days), exactly like weekly_exhausted but scoped to one model —
|
|
1100
|
+
// reuse that cause so the oracle's weekly-reset branches hold on the scoped
|
|
1101
|
+
// reset consistently (agreeing with _isAvailable's model gate above; the oracle
|
|
1102
|
+
// still returns available:true off any SIBLING account with model headroom).
|
|
1103
|
+
const scoped = model ? this._scopedExhausted(account, model) : null;
|
|
1104
|
+
if (scoped) return { cause: 'weekly_exhausted', retryAt: scoped.resetAt, queueable: false, modelScoped: true };
|
|
1105
|
+
|
|
1059
1106
|
// RAW weekly state, so the retry oracle agrees with _isAvailable's raw gate.
|
|
1060
1107
|
// (Pace must NOT classify a raw-healthy account as weekly_critical here, or
|
|
1061
1108
|
// the queue keys on a far-future reset instead of the account's real
|
|
@@ -1169,7 +1216,7 @@ export class AccountManager {
|
|
|
1169
1216
|
const pinned = this.accounts.find(a => a.name === requestInfo.pinnedAccountName);
|
|
1170
1217
|
if (pinned && !excludedIndexes.has(pinned.index)
|
|
1171
1218
|
&& this._matchesRequest(pinned, profile, requestInfo)
|
|
1172
|
-
&& this._isAvailable(pinned, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
|
|
1219
|
+
&& this._isAvailable(pinned, { allowWeeklyReserve: true, allowWeeklyCritical: true, model: requestInfo.model })) {
|
|
1173
1220
|
this.currentIndex = pinned.index;
|
|
1174
1221
|
return pinned;
|
|
1175
1222
|
}
|
|
@@ -1182,7 +1229,7 @@ export class AccountManager {
|
|
|
1182
1229
|
{ allowWeeklyReserve: true, allowWeeklyCritical: false },
|
|
1183
1230
|
{ allowWeeklyReserve: true, allowWeeklyCritical: true },
|
|
1184
1231
|
];
|
|
1185
|
-
if (preferredPasses.some(options => this._isAvailable(preferred, options))) {
|
|
1232
|
+
if (preferredPasses.some(options => this._isAvailable(preferred, { ...options, model: requestInfo.model }))) {
|
|
1186
1233
|
this.currentIndex = preferred.index;
|
|
1187
1234
|
return preferred;
|
|
1188
1235
|
}
|
|
@@ -1217,7 +1264,7 @@ export class AccountManager {
|
|
|
1217
1264
|
const account = this.accounts[idx];
|
|
1218
1265
|
if (excludedIndexes.has(account.index)) continue;
|
|
1219
1266
|
if (!this._matchesRequest(account, profile, requestInfo)) continue;
|
|
1220
|
-
if (!this._isAvailable(account, weeklyOptions)) continue;
|
|
1267
|
+
if (!this._isAvailable(account, { ...weeklyOptions, model: requestInfo.model })) continue;
|
|
1221
1268
|
|
|
1222
1269
|
const priority = Number.isFinite(account.priority) ? account.priority : 0;
|
|
1223
1270
|
const score = this._scoreAccount(account, requestInfo, scoringCtx);
|
|
@@ -1296,11 +1343,11 @@ export class AccountManager {
|
|
|
1296
1343
|
if (!account) return null;
|
|
1297
1344
|
if (excludedIndexes.has(account.index)) return null;
|
|
1298
1345
|
if (!this._matchesRequest(account, profile, requestInfo)) return null;
|
|
1299
|
-
if (!this._isAvailable(account, options)) return null;
|
|
1346
|
+
if (!this._isAvailable(account, { ...options, model: requestInfo.model })) return null;
|
|
1300
1347
|
return account;
|
|
1301
1348
|
}
|
|
1302
1349
|
|
|
1303
|
-
_bindSession(sessionKey, account) {
|
|
1350
|
+
_bindSession(sessionKey, account, model = null) {
|
|
1304
1351
|
const priority = this._priority(account);
|
|
1305
1352
|
const binding = this._sessionBinding(sessionKey) || {
|
|
1306
1353
|
homeName: account.name,
|
|
@@ -1317,8 +1364,11 @@ export class AccountManager {
|
|
|
1317
1364
|
// _boundAccount (which prefers homeName) doesn't snap the session straight
|
|
1318
1365
|
// back to the hot home next request. If the old home is UNAVAILABLE this is a
|
|
1319
1366
|
// FAILOVER → keep homeName so the session returns to it once it recovers.
|
|
1367
|
+
// Model-aware: a move off a home that's capped for THIS model (but healthy
|
|
1368
|
+
// for others) is a FAILOVER, not a by-choice rebalance — keep homeName so the
|
|
1369
|
+
// session snaps back once the model's scoped cap resets.
|
|
1320
1370
|
const oldHome = this.accounts.find(a => a.name === binding.homeName);
|
|
1321
|
-
if (oldHome && this._isAvailable(oldHome, { allowWeeklyReserve: true })) {
|
|
1371
|
+
if (oldHome && this._isAvailable(oldHome, { allowWeeklyReserve: true, model })) {
|
|
1322
1372
|
binding.homeName = account.name;
|
|
1323
1373
|
}
|
|
1324
1374
|
}
|
|
@@ -1349,7 +1399,7 @@ export class AccountManager {
|
|
|
1349
1399
|
if (excludedIndexes.has(account.index)) return false;
|
|
1350
1400
|
if (!this._matchesRequest(account, profile, requestInfo)) return false;
|
|
1351
1401
|
const priority = this._priority(account);
|
|
1352
|
-
return priority < boundPriority && this._isAvailable(account, { allowWeeklyReserve: true });
|
|
1402
|
+
return priority < boundPriority && this._isAvailable(account, { allowWeeklyReserve: true, model: requestInfo.model });
|
|
1353
1403
|
});
|
|
1354
1404
|
}
|
|
1355
1405
|
|
|
@@ -1619,9 +1669,21 @@ export class AccountManager {
|
|
|
1619
1669
|
if (usage.sevenDay.utilization != null) q.unified7d = clamp01(usage.sevenDay.utilization);
|
|
1620
1670
|
if (usage.sevenDay.resetAt != null) q.unified7dReset = usage.sevenDay.resetAt;
|
|
1621
1671
|
}
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1672
|
+
// Per-model weekly sub-limits (Fable, Opus, ...). Replace wholesale with the
|
|
1673
|
+
// fresh probe set so a family that dropped out of the response doesn't linger
|
|
1674
|
+
// stale; expiry on reset is a backstop for the between-probe window.
|
|
1675
|
+
if (usage.scopedWeekly && typeof usage.scopedWeekly === 'object') {
|
|
1676
|
+
const fresh = {};
|
|
1677
|
+
for (const [fam, e] of Object.entries(usage.scopedWeekly)) {
|
|
1678
|
+
if (!e) continue;
|
|
1679
|
+
fresh[fam] = {
|
|
1680
|
+
utilization: e.utilization != null ? clamp01(e.utilization) : null,
|
|
1681
|
+
resetAt: e.resetAt != null ? e.resetAt : null,
|
|
1682
|
+
severity: e.severity || null,
|
|
1683
|
+
isActive: e.isActive !== false,
|
|
1684
|
+
};
|
|
1685
|
+
}
|
|
1686
|
+
q.scopedWeekly = fresh;
|
|
1625
1687
|
}
|
|
1626
1688
|
|
|
1627
1689
|
// If we just learned this account's weekly window while probing, re-evaluate
|
|
@@ -1745,6 +1807,27 @@ export class AccountManager {
|
|
|
1745
1807
|
const account = this.accounts[accountIndex];
|
|
1746
1808
|
if (!account) return;
|
|
1747
1809
|
const retryAfter = clampRetryAfterSeconds(retryAfterSeconds);
|
|
1810
|
+
|
|
1811
|
+
// Model-scoped rate-limit: a per-model weekly cap (e.g. Fable) rejected this
|
|
1812
|
+
// request while the account's UNIFIED quota is healthy. Record ONLY the scoped
|
|
1813
|
+
// exhaustion — do NOT bench the whole account (its other models still have
|
|
1814
|
+
// headroom). The just-429'd request fails over to a headroom account via the
|
|
1815
|
+
// per-request excludedIndexes; future same-model requests are steered by the
|
|
1816
|
+
// scoped gate; future other-model requests keep using this account.
|
|
1817
|
+
if (options.modelScope) {
|
|
1818
|
+
account.quota.scopedWeekly = account.quota.scopedWeekly || {};
|
|
1819
|
+
account.quota.scopedWeekly[options.modelScope] = {
|
|
1820
|
+
utilization: 1,
|
|
1821
|
+
resetAt: Date.now() + (retryAfter * 1000),
|
|
1822
|
+
severity: 'critical',
|
|
1823
|
+
isActive: true,
|
|
1824
|
+
};
|
|
1825
|
+
account.lastStatus = options.status || 429;
|
|
1826
|
+
account.lastErrorAt = Date.now();
|
|
1827
|
+
console.log(`[Maxpool] Account "${account.name}" ${options.modelScope} weekly limit hit — scoped bench ${retryAfter}s (account stays active for other models)`);
|
|
1828
|
+
return;
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1748
1831
|
account.status = 'throttled';
|
|
1749
1832
|
account.rateLimitedUntil = Date.now() + (retryAfter * 1000);
|
|
1750
1833
|
account.lastStatus = options.status || 429;
|
|
@@ -2099,6 +2182,16 @@ export class AccountManager {
|
|
|
2099
2182
|
// could otherwise pin the account unavailable until the first live response.
|
|
2100
2183
|
if (account.quota.unified5hReset == null) account.quota.unified5h = null;
|
|
2101
2184
|
if (account.quota.unified7dReset == null) account.quota.unified7d = null;
|
|
2185
|
+
// Drop restored scoped entries lacking a clearable reset window — they can't
|
|
2186
|
+
// be expired by _clearExpiredQuotas and would otherwise pin an account
|
|
2187
|
+
// unavailable-for-family until the first live probe.
|
|
2188
|
+
if (account.quota.scopedWeekly && typeof account.quota.scopedWeekly === 'object') {
|
|
2189
|
+
for (const [fam, e] of Object.entries(account.quota.scopedWeekly)) {
|
|
2190
|
+
if (!e || e.resetAt == null) delete account.quota.scopedWeekly[fam];
|
|
2191
|
+
}
|
|
2192
|
+
} else {
|
|
2193
|
+
account.quota.scopedWeekly = {};
|
|
2194
|
+
}
|
|
2102
2195
|
// We already know this account's weekly window, so it isn't "probing".
|
|
2103
2196
|
if (account.quota.unified7dReset != null) account.probing = false;
|
|
2104
2197
|
}
|
package/src/index.js
CHANGED
|
@@ -156,10 +156,20 @@ async function supervisorCommand() {
|
|
|
156
156
|
// supervisor's acceptor is only LIVE during the brief cutover gap (it stops
|
|
157
157
|
// once a worker confirms it is the sole acceptor). A bare handler is required
|
|
158
158
|
// so the rare gap-accepted socket isn't left dangling.
|
|
159
|
-
const relistenMaster = () => new Promise((resolve, reject) => {
|
|
159
|
+
const relistenMaster = (retriesOnBusy = 20) => new Promise((resolve, reject) => {
|
|
160
160
|
if (masterServer.listening) { resolve(); return; }
|
|
161
|
-
const onErr = err => { masterServer.removeListener('listening', onListen); reject(err); };
|
|
162
161
|
const onListen = () => { masterServer.removeListener('error', onErr); resolve(); };
|
|
162
|
+
const onErr = err => {
|
|
163
|
+
masterServer.removeListener('listening', onListen);
|
|
164
|
+
// A just-SIGKILLed worker (seamless-reload fallback) may not have released the
|
|
165
|
+
// listening fd yet — SIGKILL is async; the port frees within a few ms. Retry
|
|
166
|
+
// bounded (~1s) rather than crashing the supervisor via handleServerListenError.
|
|
167
|
+
if (err.code === 'EADDRINUSE' && retriesOnBusy > 0) {
|
|
168
|
+
setTimeout(() => relistenMaster(retriesOnBusy - 1).then(resolve, reject), 50);
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
reject(err);
|
|
172
|
+
};
|
|
163
173
|
masterServer.once('error', onErr);
|
|
164
174
|
masterServer.once('listening', onListen);
|
|
165
175
|
masterServer.listen(port, host);
|
|
@@ -347,6 +357,19 @@ async function supervisorCommand() {
|
|
|
347
357
|
|
|
348
358
|
try {
|
|
349
359
|
while (true) {
|
|
360
|
+
// Ensure the master socket is LISTENING before handing it to a cold worker.
|
|
361
|
+
// After the previous primary worker sent MSG_PRIMARY the supervisor ran
|
|
362
|
+
// closeMasterAccept() (masterServer.close()), and the exiting worker released
|
|
363
|
+
// the last fd — so on every respawn masterServer is closed. Without this,
|
|
364
|
+
// spawnWorker sends a dead handle, the worker ignores it (falsy-handle guard),
|
|
365
|
+
// and never becomes primary → the interactive `r` restart hangs blank. Idempotent
|
|
366
|
+
// (no-op when already listening, e.g. the very first iteration after line 172).
|
|
367
|
+
try {
|
|
368
|
+
await relistenMaster();
|
|
369
|
+
} catch (err) {
|
|
370
|
+
handleServerListenError(err, host, port);
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
350
373
|
const worker = spawnWorker({ reload: false });
|
|
351
374
|
const startedAt = Date.now();
|
|
352
375
|
const result = await superviseTurn(worker);
|
|
@@ -725,7 +748,10 @@ async function serverWorkerCommand() {
|
|
|
725
748
|
if (draining) return;
|
|
726
749
|
// Interactive (live TUI) → full cold restart so the fresh worker re-renders the
|
|
727
750
|
// TUI; headless/service → zero-downtime seamless baton (nothing visual to lose).
|
|
728
|
-
|
|
751
|
+
// Test-only: force the interactive cold-restart path headless (a pty can't be
|
|
752
|
+
// allocated in-suite), so the exit-75 respawn is exercised without a real TUI.
|
|
753
|
+
const forceCold = process.env.MAXPOOL_TEST_FORCE_COLD_RESTART === '1';
|
|
754
|
+
if (!forceCold && reloadStrategy({ supervised, useTUI }) === 'seamless') {
|
|
729
755
|
try {
|
|
730
756
|
process.send({ type: MSG_RELOAD_REQUEST });
|
|
731
757
|
return;
|
|
@@ -991,6 +1017,12 @@ async function serverWorkerCommand() {
|
|
|
991
1017
|
// Cold start: take the socket and go primary immediately.
|
|
992
1018
|
await listenOnHandle(handle);
|
|
993
1019
|
await becomePrimary({ viaTakeover: false });
|
|
1020
|
+
} else if (msg?.type === MSG_LISTEN && !handle) {
|
|
1021
|
+
// Defense-in-depth for the stale-handle hang: if the supervisor ever hands
|
|
1022
|
+
// a cold worker a dead/absent socket, fail LOUD and exit-75 so it respawns —
|
|
1023
|
+
// never sit alive-but-not-primary (the blank-screen `r`-restart hang).
|
|
1024
|
+
console.error('[Maxpool] Cold worker received MSG_LISTEN with no socket handle — exiting for respawn.');
|
|
1025
|
+
process.exit(SERVER_RESTART_EXIT_CODE);
|
|
994
1026
|
} else if (msg?.type === MSG_PROBE_READY) {
|
|
995
1027
|
// Headless reload worker: we've booted the new code and restored quota
|
|
996
1028
|
// in memory. Confirm readiness (we are NOT yet accepting / writing).
|
|
@@ -1013,9 +1045,13 @@ async function serverWorkerCommand() {
|
|
|
1013
1045
|
await releaseBatonAndDrain();
|
|
1014
1046
|
}
|
|
1015
1047
|
} catch (err) {
|
|
1016
|
-
|
|
1017
|
-
//
|
|
1018
|
-
|
|
1048
|
+
console.error(`[Maxpool] Worker message error: ${err.message}`);
|
|
1049
|
+
// A COLD worker that failed to listen / become primary has no baton to roll
|
|
1050
|
+
// back to — it would strand blank. Exit-75 so the supervisor respawns it
|
|
1051
|
+
// (closes the silent-non-primary hang class). A RELOAD worker mid-baton must
|
|
1052
|
+
// NOT exit (that kills the supervisor loop) — report MSG_FAILED so the
|
|
1053
|
+
// supervisor rolls back and the old worker stays primary (zero disruption).
|
|
1054
|
+
if (!isReloadWorker) process.exit(SERVER_RESTART_EXIT_CODE);
|
|
1019
1055
|
try { process.send({ type: MSG_FAILED, reason: err.message }); } catch { /* ignore */ }
|
|
1020
1056
|
}
|
|
1021
1057
|
});
|
package/src/oauth.js
CHANGED
|
@@ -136,9 +136,34 @@ export async function fetchProfile(accessToken) {
|
|
|
136
136
|
const USAGE_URL = 'https://api.anthropic.com/api/oauth/usage';
|
|
137
137
|
const OAUTH_USAGE_BETA = 'oauth-2025-04-20';
|
|
138
138
|
|
|
139
|
+
/** Parse a reset timestamp (epoch-seconds, epoch-ms, or ISO string) to ms, or null. */
|
|
140
|
+
export function parseResetTimestamp(rawReset) {
|
|
141
|
+
if (typeof rawReset === 'number') return rawReset < 1e12 ? rawReset * 1000 : rawReset;
|
|
142
|
+
if (typeof rawReset === 'string') {
|
|
143
|
+
const asNum = Number(rawReset);
|
|
144
|
+
if (Number.isFinite(asNum) && rawReset.trim() !== '') return asNum < 1e12 ? asNum * 1000 : asNum;
|
|
145
|
+
const parsed = Date.parse(rawReset);
|
|
146
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
147
|
+
}
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** Map a model id or usage display name to a coarse family key, or null. Matches
|
|
152
|
+
* BOTH request models (`claude-fable-5`) and usage `scope.model.display_name`
|
|
153
|
+
* (`Fable`). Unmatched → null (caller treats as never model-scoped). */
|
|
154
|
+
export function modelFamily(model) {
|
|
155
|
+
const s = String(model || '').toLowerCase();
|
|
156
|
+
if (s.includes('fable') || s.includes('mythos')) return 'fable';
|
|
157
|
+
if (s.includes('opus')) return 'opus';
|
|
158
|
+
if (s.includes('sonnet')) return 'sonnet';
|
|
159
|
+
if (s.includes('haiku')) return 'haiku';
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
|
|
139
163
|
/** Normalize one usage bucket from the /api/oauth/usage response to
|
|
140
164
|
* { utilization: 0..1, resetAt: ms-timestamp } (tolerant of field-name and
|
|
141
|
-
* percentage/epoch-vs-iso variations).
|
|
165
|
+
* percentage/epoch-vs-iso variations). Legacy top-level buckets only — the
|
|
166
|
+
* `limits[]` path uses parseLimitEntry (clean 0-100 percent, no ambiguity). */
|
|
142
167
|
export function normalizeUsageBucket(bucket) {
|
|
143
168
|
if (!bucket || typeof bucket !== 'object') return null;
|
|
144
169
|
|
|
@@ -148,25 +173,26 @@ export function normalizeUsageBucket(bucket) {
|
|
|
148
173
|
? (parsedPct > 1 ? parsedPct / 100 : parsedPct)
|
|
149
174
|
: null;
|
|
150
175
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
if (typeof rawReset === 'number') {
|
|
154
|
-
resetAt = rawReset < 1e12 ? rawReset * 1000 : rawReset;
|
|
155
|
-
} else if (typeof rawReset === 'string') {
|
|
156
|
-
const asNum = Number(rawReset);
|
|
157
|
-
if (Number.isFinite(asNum) && rawReset.trim() !== '') {
|
|
158
|
-
resetAt = asNum < 1e12 ? asNum * 1000 : asNum;
|
|
159
|
-
} else {
|
|
160
|
-
const parsed = Date.parse(rawReset);
|
|
161
|
-
if (Number.isFinite(parsed)) resetAt = parsed;
|
|
162
|
-
}
|
|
163
|
-
}
|
|
176
|
+
return { utilization, resetAt: parseResetTimestamp(bucket.resets_at ?? bucket.resetsAt ?? bucket.reset_at ?? bucket.resetAt) };
|
|
177
|
+
}
|
|
164
178
|
|
|
165
|
-
|
|
179
|
+
/** Normalize one `limits[]` entry. `percent` is a clean 0-100 integer (unlike the
|
|
180
|
+
* ambiguous top-level buckets), so utilization = percent/100 always. */
|
|
181
|
+
export function parseLimitEntry(l) {
|
|
182
|
+
if (!l || typeof l !== 'object') return null;
|
|
183
|
+
const pct = typeof l.percent === 'number' ? l.percent : parseFloat(l.percent);
|
|
184
|
+
return {
|
|
185
|
+
utilization: Number.isFinite(pct) ? Math.max(0, Math.min(1, pct / 100)) : null,
|
|
186
|
+
resetAt: parseResetTimestamp(l.resets_at ?? l.resetsAt),
|
|
187
|
+
severity: typeof l.severity === 'string' ? l.severity : null,
|
|
188
|
+
isActive: l.is_active !== false,
|
|
189
|
+
};
|
|
166
190
|
}
|
|
167
191
|
|
|
168
192
|
/** Read an account's quota from the zero-spend /api/oauth/usage endpoint.
|
|
169
|
-
* Returns { fiveHour, sevenDay,
|
|
193
|
+
* Returns { fiveHour, sevenDay, scopedWeekly } buckets, or { error, status }.
|
|
194
|
+
* Per-model + unified limits now live in the `limits[]` array (the top-level
|
|
195
|
+
* seven_day_opus / seven_day_sonnet keys are deprecated → always null). */
|
|
170
196
|
export async function fetchUsage(accessToken) {
|
|
171
197
|
try {
|
|
172
198
|
const res = await fetch(USAGE_URL, {
|
|
@@ -189,10 +215,30 @@ export async function fetchUsage(accessToken) {
|
|
|
189
215
|
}
|
|
190
216
|
|
|
191
217
|
const data = await res.json();
|
|
218
|
+
|
|
219
|
+
// The `limits[]` array is the authoritative source: `session` (5h), unscoped
|
|
220
|
+
// `weekly_all` (unified 7d), and `weekly_scoped` entries carrying a
|
|
221
|
+
// `scope.model` — the PER-MODEL weekly sub-limits (e.g. Fable) Anthropic
|
|
222
|
+
// enforces separately from the unified weekly. Prefer it over the legacy
|
|
223
|
+
// top-level buckets (whose 0-100 percent misreads a genuine 1% as 100%).
|
|
224
|
+
const scopedWeekly = {};
|
|
225
|
+
let limitsSession = null;
|
|
226
|
+
let limitsWeeklyAll = null;
|
|
227
|
+
for (const l of (Array.isArray(data?.limits) ? data.limits : [])) {
|
|
228
|
+
if (l?.kind === 'weekly_scoped') {
|
|
229
|
+
const fam = l?.scope?.model?.display_name ? modelFamily(l.scope.model.display_name) : null;
|
|
230
|
+
if (fam) scopedWeekly[fam] = parseLimitEntry(l);
|
|
231
|
+
} else if (l?.kind === 'session') {
|
|
232
|
+
limitsSession = parseLimitEntry(l);
|
|
233
|
+
} else if (l?.kind === 'weekly_all') {
|
|
234
|
+
limitsWeeklyAll = parseLimitEntry(l);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
192
238
|
return {
|
|
193
|
-
fiveHour: normalizeUsageBucket(data?.five_hour),
|
|
194
|
-
sevenDay: normalizeUsageBucket(data?.seven_day),
|
|
195
|
-
|
|
239
|
+
fiveHour: limitsSession || normalizeUsageBucket(data?.five_hour),
|
|
240
|
+
sevenDay: limitsWeeklyAll || normalizeUsageBucket(data?.seven_day),
|
|
241
|
+
scopedWeekly,
|
|
196
242
|
};
|
|
197
243
|
} catch (err) {
|
|
198
244
|
return { error: err.message || String(err), status: null };
|
package/src/server.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import http from 'node:http';
|
|
2
2
|
import { writeFile, mkdir } from 'node:fs/promises';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
+
import { modelFamily } from './oauth.js';
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
const HOP_BY_HOP_HEADERS = new Set([
|
|
@@ -453,7 +454,7 @@ async function forwardRequest(
|
|
|
453
454
|
const errorBody = await readErrorBody(upstreamRes);
|
|
454
455
|
const retryAfter = parseRetryAfter(upstreamRes.headers.get('retry-after'))
|
|
455
456
|
|| parseProviderRetryAfter(errorBody, account.provider);
|
|
456
|
-
const rateLimit = classifyRateLimit(account, rateLimitHeaders, errorBody);
|
|
457
|
+
const rateLimit = classifyRateLimit(account, rateLimitHeaders, errorBody, { model: requestInfo.model, retryAfter });
|
|
457
458
|
if (rateLimit.scope === 'upstream') {
|
|
458
459
|
const parsedError = parseJsonError(errorBody);
|
|
459
460
|
const fingerprint = `429:${rateLimit.fingerprint || overloadFingerprint(errorBody, body)}`;
|
|
@@ -552,8 +553,9 @@ async function forwardRequest(
|
|
|
552
553
|
status: 429,
|
|
553
554
|
recordFailure: false,
|
|
554
555
|
fingerprint: rateLimit.scope === 'unknown' ? rateLimit.fingerprint : null,
|
|
556
|
+
modelScope: rateLimit.modelScope || null,
|
|
555
557
|
});
|
|
556
|
-
accountManager.releaseAccount(lease, { status: 429, error: 'rate_limited' });
|
|
558
|
+
accountManager.releaseAccount(lease, { status: 429, error: rateLimit.modelScope ? 'model_rate_limited' : 'rate_limited' });
|
|
557
559
|
|
|
558
560
|
if (logDir) {
|
|
559
561
|
logSections.push(`=== RESPONSE 429 — "${account.name}" rate-limited ${retryAfter}s ===\n${formatHeaders(upstreamRes.headers)}`);
|
|
@@ -1008,7 +1010,7 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
|
|
|
1008
1010
|
return `All ${n} accounts exhausted. Retry in ${retryAfter}s.`;
|
|
1009
1011
|
}
|
|
1010
1012
|
|
|
1011
|
-
export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, describeRequest };
|
|
1013
|
+
export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, describeRequest, classifyRateLimit };
|
|
1012
1014
|
|
|
1013
1015
|
async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
|
|
1014
1016
|
if (!upstreamRes.body) return '';
|
|
@@ -1073,7 +1075,7 @@ function parseJsonError(body) {
|
|
|
1073
1075
|
}
|
|
1074
1076
|
}
|
|
1075
1077
|
|
|
1076
|
-
function classifyRateLimit(account, headers, body) {
|
|
1078
|
+
function classifyRateLimit(account, headers, body, opts = {}) {
|
|
1077
1079
|
if (account.type === 'provider') return { scope: 'account', fingerprint: null };
|
|
1078
1080
|
|
|
1079
1081
|
const parsed = parseJsonError(body);
|
|
@@ -1085,18 +1087,38 @@ function classifyRateLimit(account, headers, body) {
|
|
|
1085
1087
|
const tokensRemaining = Number(headers['anthropic-ratelimit-tokens-remaining']);
|
|
1086
1088
|
const requestsRemaining = Number(headers['anthropic-ratelimit-requests-remaining']);
|
|
1087
1089
|
|
|
1090
|
+
// A PER-MODEL weekly sub-limit (e.g. Fable) rejects while the account's UNIFIED
|
|
1091
|
+
// quota is healthy. We can't rely on a captured Fable-429 body shape, so detect
|
|
1092
|
+
// by the CONTRADICTION: a weekly-class (long) reset AND both unified buckets NOT
|
|
1093
|
+
// exhausted AND the request carries a known model family. Tagging modelScope
|
|
1094
|
+
// makes the failover bench only (account, model) — not the whole account (which
|
|
1095
|
+
// still has headroom for its other models). retryAfter is in SECONDS.
|
|
1096
|
+
const fam = modelFamily(opts.model);
|
|
1097
|
+
const retryAfter = Number(opts.retryAfter);
|
|
1098
|
+
// Model-scope requires POSITIVE evidence the unified quota has headroom: at least
|
|
1099
|
+
// one unified utilization header present AND both present ones below the floor. A
|
|
1100
|
+
// rejection with NO utilization headers is treated as account-wide (safer — a
|
|
1101
|
+
// genuine account cap with stripped headers must bench the whole account, not one
|
|
1102
|
+
// model). Neither bucket may be at/above the exhaustion floor.
|
|
1103
|
+
const haveUnifiedEvidence = Number.isFinite(weekly) || Number.isFinite(fiveHour);
|
|
1104
|
+
const unifiedNotExhausted =
|
|
1105
|
+
(!Number.isFinite(weekly) || weekly < 0.985) && (!Number.isFinite(fiveHour) || fiveHour < 0.985);
|
|
1106
|
+
const modelScope =
|
|
1107
|
+
(fam && haveUnifiedEvidence && unifiedNotExhausted && Number.isFinite(retryAfter) && retryAfter >= 30 * 60)
|
|
1108
|
+
? fam : null;
|
|
1109
|
+
|
|
1088
1110
|
const quotaHeaderExhaustion =
|
|
1089
1111
|
unifiedStatus === 'rejected'
|
|
1090
1112
|
|| (Number.isFinite(fiveHour) && fiveHour >= 0.985)
|
|
1091
1113
|
|| (Number.isFinite(weekly) && weekly >= 0.985)
|
|
1092
1114
|
|| (headers['anthropic-ratelimit-tokens-remaining'] != null && tokensRemaining <= 0)
|
|
1093
1115
|
|| (headers['anthropic-ratelimit-requests-remaining'] != null && requestsRemaining <= 0);
|
|
1094
|
-
if (quotaHeaderExhaustion) return { scope: 'account', fingerprint: null };
|
|
1116
|
+
if (quotaHeaderExhaustion) return { scope: 'account', fingerprint: null, modelScope };
|
|
1095
1117
|
|
|
1096
1118
|
const quotaBodyExhaustion =
|
|
1097
1119
|
/\b(account|plan|session|weekly|quota)\b.{0,40}\b(exhausted|limit|exceeded|reached)\b/i.test(message)
|
|
1098
1120
|
|| /\busage\b.{0,40}\b(exhausted|exceeded|reached)\b/i.test(message);
|
|
1099
|
-
if (quotaBodyExhaustion) return { scope: 'account', fingerprint: null };
|
|
1121
|
+
if (quotaBodyExhaustion) return { scope: 'account', fingerprint: null, modelScope };
|
|
1100
1122
|
|
|
1101
1123
|
const explicitSharedThrottle =
|
|
1102
1124
|
message.includes('not your usage limit')
|
package/src/tui.js
CHANGED
|
@@ -986,6 +986,15 @@ export class TUI {
|
|
|
986
986
|
}
|
|
987
987
|
const weekly = weeklyPolicyText(this.am, a);
|
|
988
988
|
if (weekly) line += ` ${weekly}`;
|
|
989
|
+
// Per-model weekly caps (e.g. Fable maxed while the unified weekly still has
|
|
990
|
+
// headroom) — surfaced separately so a capped model on an otherwise-healthy
|
|
991
|
+
// account is visible, and routing away from it is explained.
|
|
992
|
+
const exhaustedFloor = this.am.scheduler?.weeklyExhaustedThreshold ?? 0.985;
|
|
993
|
+
const capped = Object.entries(q.scopedWeekly || {})
|
|
994
|
+
.filter(([, e]) => e && e.isActive !== false
|
|
995
|
+
&& (e.severity === 'critical' || (e.utilization != null && e.utilization >= exhaustedFloor)))
|
|
996
|
+
.map(([fam]) => fam.charAt(0).toUpperCase() + fam.slice(1));
|
|
997
|
+
if (capped.length) line += ` ${red(`${capped.join(',')} maxed`)}`;
|
|
989
998
|
line += ` ${dim(loadText(this._accountLoad(a)))}`;
|
|
990
999
|
return line;
|
|
991
1000
|
}
|