maxpool 1.24.80 → 1.24.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +725 -35
- package/src/claude-reset-grants.js +124 -24
- package/src/index.js +1 -0
- package/src/prober.js +162 -14
- package/src/reset-policy.js +130 -25
- package/src/reset-timing.js +27 -0
- package/src/server.js +486 -93
- package/src/tui.js +111 -28
- package/src/zai-reset-cards.js +20 -6
package/src/account-manager.js
CHANGED
|
@@ -3,6 +3,16 @@ import { computeDuplicates, credentialFp, accountIdentity, remoteAccountIdentity
|
|
|
3
3
|
import { appendEventLog } from './event-log.js';
|
|
4
4
|
import { hourFloor } from './outside-usage.js';
|
|
5
5
|
import { CapacityLedger, TANK_MIN_UTIL, mergeCycles, tankStatsFromCycles } from './capacity-ledger.js';
|
|
6
|
+
// A1: the ONE shared grant-scope answer (display, routing, claim and lift all read it).
|
|
7
|
+
import { appliesTo, forgetClaimOrg } from './claude-reset-grants.js';
|
|
8
|
+
import { POLICY, CLAUDE_WALL_WEEKLY, probeSentAt } from './reset-policy.js';
|
|
9
|
+
import {
|
|
10
|
+
LISTING_FRESH_MS, LIFT_GRACE_MS, CLAIM_STALE_LISTING_MAX_MS, CLAIM_BACKOFF_STEPS_MS,
|
|
11
|
+
CLAIM_BACKOFF_CAP_MS, REACTIVE_CLAIM_TIMEOUT_MS,
|
|
12
|
+
} from './reset-timing.js';
|
|
13
|
+
|
|
14
|
+
// Review r2 A: consecutive request-scoped faults before they count against the account.
|
|
15
|
+
const REQUEST_FAULT_ESCALATE = 3;
|
|
6
16
|
|
|
7
17
|
// Nominal window length per kind — mirrors capacity-ledger's WINDOW_MS (kept here as a
|
|
8
18
|
// local table so account-manager does not import a private constant).
|
|
@@ -92,6 +102,7 @@ function emptyQuota() {
|
|
|
92
102
|
// marker — a swallowed failing probe no longer silently freezes a stale tag.
|
|
93
103
|
// Header-driven updates do NOT stamp this (headers can't refresh scoped/provider).
|
|
94
104
|
lastProbeOkAt: null, // ms
|
|
105
|
+
lastProbeOkSentAt: null, // ms — SEND time of that probe (claim ordering, C1)
|
|
95
106
|
// Last background probe FAILURE — surfaced in the TUI/status so a persistently
|
|
96
107
|
// failing probe (e.g. the usage endpoint rate-limiting us) is VISIBLE instead of
|
|
97
108
|
// silently swallowed (which let a stale weekly keep looking fresh). Cleared on
|
|
@@ -208,7 +219,13 @@ const DEFAULT_SCHEDULER = {
|
|
|
208
219
|
// This is the 5-hour-window level at which the hold lets go (providers; Claude
|
|
209
220
|
// benches at switchThreshold anyway). Weekly reserve/soft tiers do NOT release.
|
|
210
221
|
cacheAffinityFiveHourWall: 0.97,
|
|
211
|
-
|
|
222
|
+
// RETURN HOME after a provider detour (owner 2026-11-09): when a conversation's turn
|
|
223
|
+
// went to GLM/Kimi and a later turn is routed back to Claude, it prefers the Claude
|
|
224
|
+
// account it used last — that account still holds its stored copy. Only while the copy
|
|
225
|
+
// is alive: Claude Code writes 1-hour copies (measured 2026-11-09: 272 of 272 cache
|
|
226
|
+
// writes `ephemeral_1h`; 55 of 64 detours came back within the hour, median 533 s).
|
|
227
|
+
cacheAffinityReturnWindowMs: 3_600_000,
|
|
228
|
+
concurrencyWeight: 2, // multiplies in-flight load (activeWeight+reqWeight) — dominant
|
|
212
229
|
perAccountConcurrencyTarget: 3, // D: soft per-account in-flight target; past it, capPenalty bites
|
|
213
230
|
capPenaltyWeight: 10, // steep penalty per unit of in-flight depth past D (throttle safety floor)
|
|
214
231
|
paceCostWeight: 1.5, // soft de-preference of accounts burning ahead of pace (was the ×6 term)
|
|
@@ -365,6 +382,15 @@ const PERSISTED_QUOTA_FIELDS = [
|
|
|
365
382
|
];
|
|
366
383
|
|
|
367
384
|
const CONVERSATION_AFFINITY_MAX = 5000;
|
|
385
|
+
/** Bounded LRU insert: re-insert = most recent; the oldest key drops past the cap. */
|
|
386
|
+
function lruSet(map, key, value) {
|
|
387
|
+
map.delete(key);
|
|
388
|
+
map.set(key, value);
|
|
389
|
+
while (map.size > CONVERSATION_AFFINITY_MAX) map.delete(map.keys().next().value);
|
|
390
|
+
}
|
|
391
|
+
/** An Anthropic account with its own prompt cache (allow-list: a provider or legacy
|
|
392
|
+
* pool row can forward to GLM, so "not a provider" is not proof of a Claude cache). */
|
|
393
|
+
const isClaudeAccount = (a) => a?.type === 'oauth' || a?.type === 'apikey';
|
|
368
394
|
const IDENTITY_WINDOW_MS = 15 * 60_000;
|
|
369
395
|
const IDENTITY_MAX_SAMPLES = 4000;
|
|
370
396
|
const IDENTITY_MIN_PER_SESSION = 8;
|
|
@@ -519,6 +545,46 @@ function _windowElapsedRatio(resetAt, durationMs, now) {
|
|
|
519
545
|
return Math.max(0, (durationMs - (resetAt - now)) / durationMs);
|
|
520
546
|
}
|
|
521
547
|
|
|
548
|
+
/** B1/B2 (2026-11-10): normalize a REMOTE status row's reset payload. One shared
|
|
549
|
+
* shape for syncRemoteMirrors and the TUI benched-twin overlay. An older cloud
|
|
550
|
+
* omits the new fields: a missing `eligible` reads `grants.length>0` (holdings
|
|
551
|
+
* still display, never a silent 0 — the "cloud rows show no resets" bug);
|
|
552
|
+
* missing new grant fields take the fail-closed default from A1; a missing
|
|
553
|
+
* capLift means no lift (never recomputed from the mirror's listing copy). */
|
|
554
|
+
export function normalizeRemoteResets(ra) {
|
|
555
|
+
if (!ra) return { resetCards: null, resetGrants: null, capLift: null };
|
|
556
|
+
const out = { resetCards: null, resetGrants: null, capLift: null };
|
|
557
|
+
if (ra.resetCards && typeof ra.resetCards === 'object') {
|
|
558
|
+
out.resetCards = {
|
|
559
|
+
fiveHour: Array.isArray(ra.resetCards.fiveHour) ? ra.resetCards.fiveHour : [],
|
|
560
|
+
weekly: Array.isArray(ra.resetCards.weekly) ? ra.resetCards.weekly : [],
|
|
561
|
+
checkedAt: ra.resetCards.checkedAt ?? null,
|
|
562
|
+
stale: Boolean(ra.resetCards.stale),
|
|
563
|
+
lastError: ra.resetCards.lastError ?? null,
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
if (ra.resetGrants && typeof ra.resetGrants === 'object') {
|
|
567
|
+
const grants = Array.isArray(ra.resetGrants.grants) ? ra.resetGrants.grants : [];
|
|
568
|
+
out.resetGrants = {
|
|
569
|
+
grants,
|
|
570
|
+
nextGrantId: ra.resetGrants.nextGrantId ?? null,
|
|
571
|
+
cooldownUntil: ra.resetGrants.cooldownUntil ?? null,
|
|
572
|
+
// Missing eligible (an older cloud export) → display iff grants exist.
|
|
573
|
+
eligible: ra.resetGrants.eligible !== undefined ? ra.resetGrants.eligible !== false : grants.length > 0,
|
|
574
|
+
checkedAt: ra.resetGrants.checkedAt ?? null,
|
|
575
|
+
stale: Boolean(ra.resetGrants.stale),
|
|
576
|
+
lastError: ra.resetGrants.lastError ?? null,
|
|
577
|
+
};
|
|
578
|
+
}
|
|
579
|
+
if (ra.capLift && typeof ra.capLift === 'object') {
|
|
580
|
+
// `ceiling` is the holder's computed lifted cap (null = full lift). A give-back
|
|
581
|
+
// lift MUST carry it — without it the mirror would read a full lift.
|
|
582
|
+
const ceiling = Number.isFinite(ra.capLift.ceiling) ? ra.capLift.ceiling : null;
|
|
583
|
+
out.capLift = { active: ra.capLift.active === true, since: ra.capLift.since ?? null, mode: ra.capLift.mode ?? null, ceiling };
|
|
584
|
+
}
|
|
585
|
+
return out;
|
|
586
|
+
}
|
|
587
|
+
|
|
522
588
|
export class AccountManager {
|
|
523
589
|
constructor(accounts, switchThreshold = 0.90, schedulerOptions = {}, dependencies = {}) {
|
|
524
590
|
this.scheduler = { ...DEFAULT_SCHEDULER, ...schedulerOptions };
|
|
@@ -631,6 +697,10 @@ export class AccountManager {
|
|
|
631
697
|
// Bounded LRU — oldest conversations drop first; a dropped entry only means the
|
|
632
698
|
// next turn is balanced fresh.
|
|
633
699
|
this.conversationLastAccount = new Map();
|
|
700
|
+
// _homeKey (affinityKey, else sessionKey) → { name, at, detoured } of the last CLAUDE
|
|
701
|
+
// account only (a GLM/Kimi turn never overwrites the name; it sets `detoured`), so a
|
|
702
|
+
// conversation returning from a provider detour can go home.
|
|
703
|
+
this.conversationLastClaudeAccount = new Map();
|
|
634
704
|
// Conversation-identity HEALTH (owner 2026-11-08: "make sure this cannot degrade
|
|
635
705
|
// silently"). Twice in one day a Claude Code request-shape detail (a moving
|
|
636
706
|
// cache_control marker, then a per-request billing checksum) made every turn hash
|
|
@@ -906,7 +976,11 @@ export class AccountManager {
|
|
|
906
976
|
const prevCurrentName = requestInfo.sessionKey
|
|
907
977
|
? this._sessionBinding(requestInfo.sessionKey)?.currentName
|
|
908
978
|
: null;
|
|
979
|
+
requestInfo.returnedHome = null; // set by the selection loop when it sends the turn home
|
|
909
980
|
const account = this.getActiveAccount(requestInfo, excludedIndexes);
|
|
981
|
+
// The D3 retry-same hint is good for exactly ONE acquire, honored or not — a stale
|
|
982
|
+
// hint would pull a later failover hop back onto the account it just left.
|
|
983
|
+
requestInfo.retrySameAccountName = null;
|
|
910
984
|
if (!account) return null;
|
|
911
985
|
|
|
912
986
|
const weight = Math.max(1, Number(requestInfo.weight) || 1);
|
|
@@ -924,7 +998,9 @@ export class AccountManager {
|
|
|
924
998
|
this.notePoolSend(account);
|
|
925
999
|
// Non-null only when this acquire moved the session off its prior account.
|
|
926
1000
|
const migratedFromName = (prevCurrentName && account.name !== prevCurrentName) ? prevCurrentName : null;
|
|
927
|
-
|
|
1001
|
+
const returnedHome = requestInfo.returnedHome?.name === account.name;
|
|
1002
|
+
const returnedHomeSwitched = returnedHome && requestInfo.returnedHome.switched === true;
|
|
1003
|
+
return { account, weight, startedAt: Date.now(), upstreamThrottleProbe, migratedFromName, returnedHome, returnedHomeSwitched };
|
|
928
1004
|
}
|
|
929
1005
|
|
|
930
1006
|
/** Fail-safe: snap a session's binding back to the pre-migration issuing account
|
|
@@ -1035,13 +1111,31 @@ export class AccountManager {
|
|
|
1035
1111
|
return out;
|
|
1036
1112
|
}
|
|
1037
1113
|
|
|
1114
|
+
/**
|
|
1115
|
+
* outcome.neutral — the attempt says nothing about the account (client cancel,
|
|
1116
|
+
* request-shape refusal): no counters move at all.
|
|
1117
|
+
* outcome.scope — 'account' (default) | 'request'. A request-scoped failure is
|
|
1118
|
+
* recorded (failedRequests, lastError) but never extends the
|
|
1119
|
+
* consecutiveFailures streak, so it cannot trip the hold-release
|
|
1120
|
+
* gate for the account's other conversations.
|
|
1121
|
+
*/
|
|
1038
1122
|
releaseAccount(lease, outcome = {}) {
|
|
1039
1123
|
if (!lease?.account) return;
|
|
1040
1124
|
const account = lease.account;
|
|
1041
1125
|
|
|
1126
|
+
const depthBefore = account.activeWeight;
|
|
1042
1127
|
account.inFlight = Math.max(0, account.inFlight - 1);
|
|
1043
1128
|
account.activeWeight = Math.max(0, account.activeWeight - lease.weight);
|
|
1044
1129
|
|
|
1130
|
+
// Owner's throttling constraint: make any throttle on a return-home turn countable,
|
|
1131
|
+
// and say whether the switch (vs the loop's own pick) put the turn on home.
|
|
1132
|
+
if (lease.returnedHome && (outcome.status === 429 || (outcome.status === 503 && account.remote)) && outcome.error !== 'request_shape_refused') {
|
|
1133
|
+
// A pool-wide throttle (markUpstreamThrottled) is not about home's depth — tag it so
|
|
1134
|
+
// the post-release count can leave it out.
|
|
1135
|
+
const scope = outcome.upstreamThrottled ? ' pool-wide' : '';
|
|
1136
|
+
console.log(`[Maxpool] return-home lease throttled (${outcome.status} ${outcome.error || '-'}${scope}, ${lease.returnedHomeSwitched ? 'switched' : 'natural'}): ${account.name} depth=${depthBefore}`);
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1045
1139
|
if (lease.upstreamThrottleProbe) {
|
|
1046
1140
|
if (outcome.success) {
|
|
1047
1141
|
this.clearUpstreamThrottle('successful recovery probe');
|
|
@@ -1060,6 +1154,10 @@ export class AccountManager {
|
|
|
1060
1154
|
if (outcome.success) {
|
|
1061
1155
|
account.completedRequests++;
|
|
1062
1156
|
account.consecutiveFailures = 0;
|
|
1157
|
+
// Review r2 A: any success ends a request-scoped fault run.
|
|
1158
|
+
account.requestFaultStreak = 0;
|
|
1159
|
+
account.requestFaultEscalated = false;
|
|
1160
|
+
account.softCooldownUntil = null;
|
|
1063
1161
|
account.providerRateLimitStreak = 0;
|
|
1064
1162
|
account.lastStatus = outcome.status || account.lastStatus;
|
|
1065
1163
|
account.lastResponseMs = Date.now() - lease.startedAt;
|
|
@@ -1078,7 +1176,12 @@ export class AccountManager {
|
|
|
1078
1176
|
|
|
1079
1177
|
if (outcome.error || outcome.status) {
|
|
1080
1178
|
account.failedRequests++;
|
|
1081
|
-
|
|
1179
|
+
// scope:'request' (cancel-adjacent / first-byte timeout / link blip): the fault
|
|
1180
|
+
// belongs to THIS request, not the account. It must not bump the streak that
|
|
1181
|
+
// the hold-release gate (_affinityRelease `failures-N`) reads — one such fault
|
|
1182
|
+
// used to evict every conversation held on the account (4 in 1s, 2026-11-09
|
|
1183
|
+
// 08:13Z). Account-level faults (429 / caps / auth) keep the default 'account'.
|
|
1184
|
+
if (outcome.scope !== 'request') account.consecutiveFailures++;
|
|
1082
1185
|
account.lastStatus = outcome.status || account.lastStatus;
|
|
1083
1186
|
account.lastResponseMs = Date.now() - lease.startedAt;
|
|
1084
1187
|
this._recordLoadEvent(account, lease, outcome);
|
|
@@ -1145,6 +1248,21 @@ export class AccountManager {
|
|
|
1145
1248
|
return exhausted ? { resetAt: e.resetAt || null, family: fam } : null;
|
|
1146
1249
|
}
|
|
1147
1250
|
|
|
1251
|
+
/** Review r2 A: inside a request-scoped soft cooldown (new placements only). */
|
|
1252
|
+
_softCooling(account, now = Date.now()) {
|
|
1253
|
+
return Boolean(account.softCooldownUntil && now < account.softCooldownUntil);
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
/** Does `account` already hold this request's conversation (so a soft cooldown must
|
|
1257
|
+
* not move it)? The one-shot retry-same hint, or the last account for the affinity
|
|
1258
|
+
* key (which falls back to the sessionKey, so a sticky binding is covered). A
|
|
1259
|
+
* return-home target needs no clause: homeCand is captured before the skip. */
|
|
1260
|
+
_holdsRequest(account, requestInfo = {}) {
|
|
1261
|
+
if (requestInfo.retrySameAccountName === account.name) return true;
|
|
1262
|
+
const key = this._affinityKey(requestInfo);
|
|
1263
|
+
return Boolean(key && this.conversationLastAccount.get(key) === account.name);
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1148
1266
|
_isAvailable(account, options = {}) {
|
|
1149
1267
|
if (!account) return false;
|
|
1150
1268
|
if (!account.enabled) return false;
|
|
@@ -1718,6 +1836,19 @@ export class AccountManager {
|
|
|
1718
1836
|
_effectiveCap(account, window = 'wk', now = Date.now()) {
|
|
1719
1837
|
const floor = account?.capUtilization;
|
|
1720
1838
|
if (floor == null) return null; // uncapped — unchanged
|
|
1839
|
+
// C3 CAP LIFT (D2, owner 2026-11-10): while an account holds a usable
|
|
1840
|
+
// full-refill reset (a `clears:['seven_day',...]` grant or a usable z.ai
|
|
1841
|
+
// weekly card), its cap does not apply — it runs to its weekly limit, the
|
|
1842
|
+
// pool spends the reset there, and the cap applies again after. Suspends the
|
|
1843
|
+
// cap on BOTH windows (the session bench reads the cap too) and the provider
|
|
1844
|
+
// max(ses,wk) cap (a GLM capped below 0.90 never reaches the z.ai wall).
|
|
1845
|
+
// Give-back-only grants (gives_back_weekly, no seven_day) lift to a measured
|
|
1846
|
+
// ceiling, not the full wall — see _capLiftCeiling.
|
|
1847
|
+
if (this.capLiftActive(account, now)) {
|
|
1848
|
+
const ceil = this._capLiftCeiling(account, window, now);
|
|
1849
|
+
if (ceil != null) return ceil;
|
|
1850
|
+
return null; // full lift for this window
|
|
1851
|
+
}
|
|
1721
1852
|
if (account.capMode !== 'dynamic') return floor; // fixed — byte-for-byte as before
|
|
1722
1853
|
const ceiling = account.capCeiling ?? this.switchThreshold;
|
|
1723
1854
|
if (floor >= ceiling) return floor; // never pull a high floor DOWN
|
|
@@ -1729,6 +1860,155 @@ export class AccountManager {
|
|
|
1729
1860
|
return floor + (ceiling - floor) * ramp;
|
|
1730
1861
|
}
|
|
1731
1862
|
|
|
1863
|
+
// ── C3: cap lift (D2) ─────────────────────────────────────────────────────
|
|
1864
|
+
// capLift is computed ONLY by the holder pool (pre-mortem #13): mirrors honour
|
|
1865
|
+
// the exported value and never recompute it from their copy of the listing.
|
|
1866
|
+
// State per account: account.capLift = { active, since, graceUntil, reason,
|
|
1867
|
+
// ceilingOverride? } — evaluated live by capLiftActive() from the CURRENT
|
|
1868
|
+
// listing, so a grant missing from a fresh listing ends the lift at once.
|
|
1869
|
+
|
|
1870
|
+
/** D4 (finding 3): the free weekly refill of the limit the lift lets the account
|
|
1871
|
+
* burn is < 24h away — the lift would spend a scarce reset for a few hours of
|
|
1872
|
+
* capacity the free refill brings anyway. Held unless the lifting reset itself
|
|
1873
|
+
* dies inside the same 24h (its value is lost either way). Unknown refill = no hold. */
|
|
1874
|
+
_liftRefillHeld(account, sourceEndsAt, now = Date.now()) {
|
|
1875
|
+
const q = account.quota || {};
|
|
1876
|
+
const refillAt = account.type === 'provider' ? q.providerWkReset : q.unified7dReset;
|
|
1877
|
+
if (refillAt == null || refillAt - now >= POLICY.dyingWindowMs) return false;
|
|
1878
|
+
const sourceDying = sourceEndsAt != null && sourceEndsAt - now < POLICY.dyingWindowMs;
|
|
1879
|
+
return !sourceDying;
|
|
1880
|
+
}
|
|
1881
|
+
|
|
1882
|
+
/** True while the account's usage cap is suspended by a held full-refill reset. */
|
|
1883
|
+
capLiftActive(account, now = Date.now()) {
|
|
1884
|
+
if (!account || account.enabled === false) return false;
|
|
1885
|
+
if (account.remote) {
|
|
1886
|
+
// Mirror: honour the holder pool's export only (never recompute).
|
|
1887
|
+
return account.capLift?.active === true;
|
|
1888
|
+
}
|
|
1889
|
+
const st = account.capLift;
|
|
1890
|
+
if (!st?.active) return false;
|
|
1891
|
+
// DEAD/OUT conditions end the lift regardless of grace.
|
|
1892
|
+
if (account.poolBenchedBy || account.refreshDead || account.subscriptionGone) {
|
|
1893
|
+
this.endCapLift(account.index, 'account dead/benched', now);
|
|
1894
|
+
return false;
|
|
1895
|
+
}
|
|
1896
|
+
// GRACE: listing failures hold the lift for LIFT_GRACE_MS, then end it.
|
|
1897
|
+
if (now > (st.graceUntil ?? 0)) {
|
|
1898
|
+
this.endCapLift(account.index, 'grace expired', now);
|
|
1899
|
+
return false;
|
|
1900
|
+
}
|
|
1901
|
+
// The free refill moved inside 24h while lifted: the cap returns now.
|
|
1902
|
+
if (this._liftRefillHeld(account, st.sourceEndsAt ?? null, now)) {
|
|
1903
|
+
this.endCapLift(account.index, 'free refill <24h away', now);
|
|
1904
|
+
return false;
|
|
1905
|
+
}
|
|
1906
|
+
return true;
|
|
1907
|
+
}
|
|
1908
|
+
|
|
1909
|
+
/** The lifted ceiling for a window, or null for "no cap at all".
|
|
1910
|
+
* A give-back grant with a measured drop lifts only to cap + drop (per window,
|
|
1911
|
+
* the weekly is where the give-back lands); a full seven_day grant lifts fully.
|
|
1912
|
+
* A MIRROR reads the holder's exported ceiling (it carries no giveBack); a
|
|
1913
|
+
* give-back lift whose ceiling is unknown fails CLOSED to the cap itself. */
|
|
1914
|
+
_capLiftCeiling(account, _window, _now) {
|
|
1915
|
+
const st = account.capLift;
|
|
1916
|
+
if (!st) return null;
|
|
1917
|
+
if (st.mode !== 'giveback') return null; // full lift
|
|
1918
|
+
const cap = Number(account.capUtilization) || 0;
|
|
1919
|
+
if (account.remote) return Number.isFinite(st.ceiling) ? st.ceiling : cap;
|
|
1920
|
+
const drop = Number(account.giveBack?.drop);
|
|
1921
|
+
return Number.isFinite(drop) && drop > 0 ? Math.min(1, cap + drop) : cap;
|
|
1922
|
+
}
|
|
1923
|
+
|
|
1924
|
+
/** Re-evaluate the lift from the CURRENT listing (holder pool only). Called by
|
|
1925
|
+
* applyResetGrants / applyResetCards after every successful listing, and by
|
|
1926
|
+
* endCapLift on terminal events. Idempotent; logs each change once. */
|
|
1927
|
+
refreshCapLift(accountIndex, now = Date.now(), reason = 'listing refreshed') {
|
|
1928
|
+
const account = this.accounts[accountIndex];
|
|
1929
|
+
if (!account || account.remote) return;
|
|
1930
|
+
const was = Boolean(account.capLift?.active);
|
|
1931
|
+
|
|
1932
|
+
// START conditions (all must hold).
|
|
1933
|
+
let active = false;
|
|
1934
|
+
let mode = null;
|
|
1935
|
+
let sources = []; // the resets that carry the lift (their expiry decides the refill hold)
|
|
1936
|
+
// No cap configured → nothing to lift (starting a lift on an uncapped account
|
|
1937
|
+
// would remap its ordinary weekly tiers and log noise for no benefit).
|
|
1938
|
+
const capFloor = account.capUtilization;
|
|
1939
|
+
if (capFloor != null && account.enabled !== false && !account.poolBenchedBy && !account.refreshDead && !account.subscriptionGone) {
|
|
1940
|
+
const rg = account.resetGrants;
|
|
1941
|
+
const cards = account.resetCards;
|
|
1942
|
+
const listingOk = rg && rg.stale !== true && rg.eligible !== false
|
|
1943
|
+
&& rg.checkedAt != null && (now - rg.checkedAt) <= LISTING_FRESH_MS
|
|
1944
|
+
// No cooldown, no claim backoff for the grant that would lift (C3).
|
|
1945
|
+
&& !(rg.cooldownUntil != null && rg.cooldownUntil > now);
|
|
1946
|
+
if (listingOk) {
|
|
1947
|
+
const inBackoff = g => {
|
|
1948
|
+
const until = this._claimRetryUntil?.get(`${account.name}:${g.id}`);
|
|
1949
|
+
return until != null && until > now;
|
|
1950
|
+
};
|
|
1951
|
+
const fresh = (rg.grants || []).filter(g =>
|
|
1952
|
+
g && g.usableNow !== false && !g.paused && !g.expired && !g.blocking && !g.unavailableReason
|
|
1953
|
+
&& (g.endsAt == null || g.endsAt > now) && (g.resetsLeft ?? 1) > 0 && !inBackoff(g));
|
|
1954
|
+
// appliesTo is the ONE scope answer: full = clears seven_day; give-back =
|
|
1955
|
+
// gives_back_weekly on a grant that does NOT clear seven_day.
|
|
1956
|
+
const fullGrants = fresh.filter(g => Array.isArray(g.clears) && g.clears.includes('seven_day'));
|
|
1957
|
+
const givebackGrants = fresh.filter(g => g.givesBackWeekly === true
|
|
1958
|
+
&& !(Array.isArray(g.clears) && g.clears.includes('seven_day')));
|
|
1959
|
+
if (fullGrants.length) { active = true; mode = 'full'; sources = fullGrants.map(g => g.endsAt); }
|
|
1960
|
+
else if (givebackGrants.length) {
|
|
1961
|
+
// Give-back lifts ONLY once a measured claim recorded the drop (pre-mortem #7:
|
|
1962
|
+
// a give-back grant returns what the MODEL used, not the whole weekly).
|
|
1963
|
+
const drop = Number(account.giveBack?.drop);
|
|
1964
|
+
if (Number.isFinite(drop) && drop > 0) { active = true; mode = 'giveback'; sources = givebackGrants.map(g => g.endsAt); }
|
|
1965
|
+
}
|
|
1966
|
+
}
|
|
1967
|
+
// z.ai weekly card: a full refill by the same rule (pre-mortem #9: a GLM
|
|
1968
|
+
// capped below 0.90 never reaches the z.ai wall otherwise).
|
|
1969
|
+
if (!active && account.type === 'provider' && account.provider === 'zai' && cards && cards.stale !== true
|
|
1970
|
+
&& cards.checkedAt != null && (now - cards.checkedAt) <= LISTING_FRESH_MS) {
|
|
1971
|
+
const hasWeeklyLimit = account.quota?.providerWk != null || account.quota?.providerWkReset != null;
|
|
1972
|
+
const liveWeekly = (cards.weekly || []).filter(c => c && !c.expired && (c.expiresAt == null || c.expiresAt > now));
|
|
1973
|
+
if (hasWeeklyLimit && liveWeekly.length) { active = true; mode = 'full'; sources = liveWeekly.map(c => c.expiresAt); }
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
// The soonest-expiring source is the one a claim would spend first.
|
|
1977
|
+
const known = sources.filter(t => t != null);
|
|
1978
|
+
const sourceEndsAt = known.length ? Math.min(...known) : null;
|
|
1979
|
+
// D4 (finding 3): never start or keep the lift inside 24h of the free refill.
|
|
1980
|
+
let endWhy = 'grant missing from fresh listing';
|
|
1981
|
+
if (active && this._liftRefillHeld(account, sourceEndsAt, now)) { active = false; endWhy = 'free refill <24h away'; }
|
|
1982
|
+
|
|
1983
|
+
if (active) {
|
|
1984
|
+
const prev = account.capLift;
|
|
1985
|
+
account.capLift = {
|
|
1986
|
+
active: true,
|
|
1987
|
+
since: prev?.active ? prev.since : now,
|
|
1988
|
+
graceUntil: now + LIFT_GRACE_MS,
|
|
1989
|
+
mode: mode || 'full',
|
|
1990
|
+
sourceEndsAt,
|
|
1991
|
+
reason,
|
|
1992
|
+
};
|
|
1993
|
+
if (!was) this._logCapLift(account, 'cap lifted', reason);
|
|
1994
|
+
} else if (was) {
|
|
1995
|
+
this.endCapLift(accountIndex, endWhy, now);
|
|
1996
|
+
}
|
|
1997
|
+
}
|
|
1998
|
+
|
|
1999
|
+
/** Force-end the lift (claim success/failure, grant gone, resetsLeft 0, cooldown). */
|
|
2000
|
+
endCapLift(accountIndex, why = 'lift ended', now = Date.now()) {
|
|
2001
|
+
const account = this.accounts[accountIndex];
|
|
2002
|
+
if (!account?.capLift?.active) return;
|
|
2003
|
+
account.capLift = { ...account.capLift, active: false, endedAt: now };
|
|
2004
|
+
this._logCapLift(account, 'cap restored', why);
|
|
2005
|
+
}
|
|
2006
|
+
|
|
2007
|
+
_logCapLift(account, verb, why) {
|
|
2008
|
+
console.log(`[reset-grant] ${account.name}: ${verb} (${why})`);
|
|
2009
|
+
appendEventLog(`[reset-grant] ${account.name}: ${verb} (${why})`);
|
|
2010
|
+
}
|
|
2011
|
+
|
|
1732
2012
|
/**
|
|
1733
2013
|
* When a DYNAMIC cap will have risen far enough to stop benching `utilization` on
|
|
1734
2014
|
* this window — i.e. the instant the ramp crosses it. Null when that never happens
|
|
@@ -2325,10 +2605,13 @@ export class AccountManager {
|
|
|
2325
2605
|
];
|
|
2326
2606
|
|
|
2327
2607
|
const bestUnlockMap = new Map();
|
|
2608
|
+
const home = this._returnHome(requestInfo, now);
|
|
2328
2609
|
for (const weeklyOptions of weeklyPasses) {
|
|
2329
2610
|
best = null;
|
|
2330
2611
|
bestScore = Infinity;
|
|
2331
2612
|
bestPriority = Infinity;
|
|
2613
|
+
let homeCand = null; // the home account as a candidate THIS pass, scored as holder
|
|
2614
|
+
let coolBest = null; // best soft-cooled account (review r2 A): used only if nothing else fits
|
|
2332
2615
|
|
|
2333
2616
|
for (let i = 0; i < this.accounts.length; i++) {
|
|
2334
2617
|
const idx = (this.nextIndex + i) % this.accounts.length;
|
|
@@ -2344,6 +2627,20 @@ export class AccountManager {
|
|
|
2344
2627
|
|
|
2345
2628
|
const priority = this._effectivePriority(account, requestInfo, now);
|
|
2346
2629
|
const score = this._scoreAccount(account, requestInfo, scoringCtx, critUnlock);
|
|
2630
|
+
if (home && account.name === home.name) {
|
|
2631
|
+
homeCand = { account, priority, critUnlock,
|
|
2632
|
+
score: this._scoreAccount(account, requestInfo, scoringCtx, critUnlock, home.name) };
|
|
2633
|
+
}
|
|
2634
|
+
// REQUEST-SCOPED SOFT COOLDOWN (review r2 A): after a request-scoped first-byte
|
|
2635
|
+
// timeout the account is deprioritised for NEW placements only — a cold pick
|
|
2636
|
+
// takes it last. A conversation it HOLDS (_holdsRequest) is not touched:
|
|
2637
|
+
// evicting those is exactly what a request-scoped fault must never do.
|
|
2638
|
+
if (this._softCooling(account, now) && !this._holdsRequest(account, requestInfo)) {
|
|
2639
|
+
if (!coolBest || priority < coolBest.priority || (priority === coolBest.priority && score < coolBest.score)) {
|
|
2640
|
+
coolBest = { account, priority, score, critUnlock };
|
|
2641
|
+
}
|
|
2642
|
+
continue;
|
|
2643
|
+
}
|
|
2347
2644
|
if (priority < bestPriority || (priority === bestPriority && score < bestScore)) {
|
|
2348
2645
|
bestPriority = priority;
|
|
2349
2646
|
bestScore = score;
|
|
@@ -2352,6 +2649,47 @@ export class AccountManager {
|
|
|
2352
2649
|
}
|
|
2353
2650
|
}
|
|
2354
2651
|
|
|
2652
|
+
// Deprioritised, never benched: with no other candidate, the soft-cooled one serves.
|
|
2653
|
+
if (!best && coolBest) {
|
|
2654
|
+
best = coolBest.account;
|
|
2655
|
+
bestPriority = coolBest.priority;
|
|
2656
|
+
bestScore = coolBest.score;
|
|
2657
|
+
if (coolBest.critUnlock) bestUnlockMap.set(best.name, coolBest.critUnlock); else bestUnlockMap.delete(best.name);
|
|
2658
|
+
}
|
|
2659
|
+
|
|
2660
|
+
// RETURN HOME: only once this pass already picked a Claude account — a provider
|
|
2661
|
+
// pick is never touched, so this never pulls a conversation off GLM sooner. Home
|
|
2662
|
+
// wins when it is a candidate here, in the same priority tier, and its score AS
|
|
2663
|
+
// HOLDER (release gates, zeroed steering, anti-dogpile cap) beats the pick.
|
|
2664
|
+
let homeNote = null;
|
|
2665
|
+
let homeSwitched = false;
|
|
2666
|
+
const pickedBeforeHome = best;
|
|
2667
|
+
if (best && home && isClaudeAccount(best)) {
|
|
2668
|
+
if (best.name === home.name) {
|
|
2669
|
+
homeNote = `home=${home.name}:returned`;
|
|
2670
|
+
} else if (home.stale) {
|
|
2671
|
+
homeNote = `home=${home.name}:skipped(stale-${Math.round(home.ageMs / 60_000)}m)`;
|
|
2672
|
+
} else if (homeCand && homeCand.priority <= bestPriority && homeCand.score < bestScore) {
|
|
2673
|
+
// `<=`, not `===`: a soft-cooled HOME is skipped from the best-pick above but
|
|
2674
|
+
// still recorded in homeCand — home at equal-or-better priority wins even then
|
|
2675
|
+
// (review r3 R3-5: soft-cooldown must never override return-home).
|
|
2676
|
+
best = homeCand.account;
|
|
2677
|
+
homeSwitched = true;
|
|
2678
|
+
if (homeCand.critUnlock) bestUnlockMap.set(best.name, homeCand.critUnlock); else bestUnlockMap.delete(best.name);
|
|
2679
|
+
homeNote = `home=${home.name}:returned`;
|
|
2680
|
+
} else if (homeCand && this._softCooling(homeCand.account, now)) {
|
|
2681
|
+
// R3-5: a soft-cooled home that did NOT win as holder: name the soft cooldown
|
|
2682
|
+
// (it kept home out of the best-pick) instead of an `outscored:` line computed
|
|
2683
|
+
// as if home had competed there.
|
|
2684
|
+
homeNote = `home=${home.name}:skipped(soft-cooled;${homeCand.priority > bestPriority ? 'priority-tier' : `holder-score ${homeCand.score.toFixed(1)}>=pick ${bestScore.toFixed(1)}`})`;
|
|
2685
|
+
} else {
|
|
2686
|
+
let why = 'unknown';
|
|
2687
|
+
try { why = this._moveReason(home.name, best, requestInfo, excludedIndexes, scoringCtx, weeklyOptions, bestUnlockMap.get(best.name) || null, now, true); } catch (e) { why = `error:${e.message}`; }
|
|
2688
|
+
homeNote = `home=${home.name}:skipped(${why})`;
|
|
2689
|
+
}
|
|
2690
|
+
if (best.name === home.name) requestInfo.returnedHome = { name: home.name, switched: homeSwitched };
|
|
2691
|
+
}
|
|
2692
|
+
|
|
2355
2693
|
if (best) {
|
|
2356
2694
|
const switched = best.index !== this.currentIndex;
|
|
2357
2695
|
this.currentIndex = best.index;
|
|
@@ -2380,9 +2718,11 @@ export class AccountManager {
|
|
|
2380
2718
|
const ckey = this._affinityKey(requestInfo);
|
|
2381
2719
|
const lastName = ckey ? this.conversationLastAccount.get(ckey) : null;
|
|
2382
2720
|
if (lastName && lastName !== best.name) {
|
|
2721
|
+
// why= explains why the conversation LEFT its last account — measured against
|
|
2722
|
+
// the loop's own pick, before any return-home switch.
|
|
2383
2723
|
let why = 'unknown';
|
|
2384
|
-
try { why = this._moveReason(lastName,
|
|
2385
|
-
console.log(`[Maxpool] conversation moved: ${lastName} → ${best.name} (${requestInfo.bodyBytes ?? '?'}B) why=${why}`);
|
|
2724
|
+
try { why = this._moveReason(lastName, pickedBeforeHome, requestInfo, excludedIndexes, scoringCtx, weeklyOptions, bestUnlockMap.get(pickedBeforeHome.name) || null, now); } catch (e) { why = `error:${e.message}`; }
|
|
2725
|
+
console.log(`[Maxpool] conversation moved: ${lastName} → ${best.name} (${requestInfo.bodyBytes ?? '?'}B) why=${why}${homeNote ? ` ${homeNote}` : ''}`);
|
|
2386
2726
|
}
|
|
2387
2727
|
return best;
|
|
2388
2728
|
}
|
|
@@ -3074,10 +3414,18 @@ export class AccountManager {
|
|
|
3074
3414
|
*/
|
|
3075
3415
|
_holdsUsableReset(account, now = Date.now()) {
|
|
3076
3416
|
if (!account || account.enabled === false) return false;
|
|
3417
|
+
if (account.refreshDead || account.subscriptionGone) return false; // C4: a dead account can never spend
|
|
3418
|
+
// A spent/unknown listing holds nothing (C4: only a reset C1/C3 could spend).
|
|
3077
3419
|
const grants = account.resetGrants?.grants || [];
|
|
3078
|
-
|
|
3420
|
+
const listingFresh = account.resetGrants && account.resetGrants.stale !== true;
|
|
3421
|
+
// appliesTo is the ONE scope answer (C4): only a grant that can refill a WEEKLY
|
|
3422
|
+
// limit counts — session-only and unmapped-scope grants are never claimed, so
|
|
3423
|
+
// they earn no routing discount.
|
|
3424
|
+
if (grants.some(g => g && g.usableNow !== false && !g.paused && !g.blocking && !g.unavailableReason
|
|
3425
|
+
&& (g.endsAt == null || g.endsAt > now) && (g.resetsLeft ?? 1) > 0
|
|
3426
|
+
&& appliesTo(g).some(s => s !== 'session'))) return listingFresh;
|
|
3079
3427
|
const cards = account.resetCards;
|
|
3080
|
-
if (!cards) return false;
|
|
3428
|
+
if (!cards || cards.stale === true) return false;
|
|
3081
3429
|
const live = c => c && !c.expired && (c.expiresAt == null || c.expiresAt > now);
|
|
3082
3430
|
if ((cards.fiveHour || []).some(live)) return true;
|
|
3083
3431
|
const hasWeeklyLimit = account.quota?.providerWk != null || account.quota?.providerWkReset != null;
|
|
@@ -3087,13 +3435,17 @@ export class AccountManager {
|
|
|
3087
3435
|
_resetMultiplier(account, now = Date.now()) {
|
|
3088
3436
|
const d = Number(this.scheduler.resetBoostDiscount) || 0;
|
|
3089
3437
|
if (d <= 0) return 1;
|
|
3438
|
+
// C3/C4 (pre-mortem #10): NO discount while the cap lift is active — a lifted
|
|
3439
|
+
// reset-holder must attract ORDINARY share, not be preferred from 10% upward
|
|
3440
|
+
// (which would pile traffic onto ~10 accounts and 429 them).
|
|
3441
|
+
if (this.capLiftActive(account, now)) return 1;
|
|
3090
3442
|
return this._holdsUsableReset(account, now) ? 1 - Math.min(d, 0.9) : 1;
|
|
3091
3443
|
}
|
|
3092
3444
|
|
|
3093
|
-
_scoreTerms(account, requestInfo = {}, ctx = null, criticalUnlock = null) {
|
|
3445
|
+
_scoreTerms(account, requestInfo = {}, ctx = null, criticalUnlock = null, holderName = undefined) {
|
|
3094
3446
|
const now = ctx?.now ?? Date.now();
|
|
3095
3447
|
// CACHE AFFINITY — see _cacheAffinity.
|
|
3096
|
-
const affinity = this._cacheAffinity(account, requestInfo, criticalUnlock);
|
|
3448
|
+
const affinity = this._cacheAffinity(account, requestInfo, criticalUnlock, holderName);
|
|
3097
3449
|
const reqWeight = Math.max(1, requestInfo.weight || 1);
|
|
3098
3450
|
// POOL LINK scores as the cloud's WORST-case load (owner 2026-10-29: "the team
|
|
3099
3451
|
// pool will respect the accounts within it... included in the balancing act").
|
|
@@ -3212,8 +3564,8 @@ export class AccountManager {
|
|
|
3212
3564
|
}
|
|
3213
3565
|
|
|
3214
3566
|
/** Routing cost — lower is preferred. Sum of the named terms minus the affinity bonus. */
|
|
3215
|
-
_scoreAccount(account, requestInfo = {}, ctx = null, criticalUnlock = null) {
|
|
3216
|
-
const t = this._scoreTerms(account, requestInfo, ctx, criticalUnlock);
|
|
3567
|
+
_scoreAccount(account, requestInfo = {}, ctx = null, criticalUnlock = null, holderName = undefined) {
|
|
3568
|
+
const t = this._scoreTerms(account, requestInfo, ctx, criticalUnlock, holderName);
|
|
3217
3569
|
return Math.max(0, t.concurrency + t.capPenalty + t.paceCost + t.utilizationCost + t.scopedPace + t.spread + t.ramp + t.reserveCost + t.criticalCost + t.failurePenalty - t.affinity);
|
|
3218
3570
|
}
|
|
3219
3571
|
|
|
@@ -3237,11 +3589,47 @@ export class AccountManager {
|
|
|
3237
3589
|
*/
|
|
3238
3590
|
_noteConversationAccount(requestInfo, account) {
|
|
3239
3591
|
const key = this._affinityKey(requestInfo);
|
|
3240
|
-
if (
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3592
|
+
if (key) lruSet(this.conversationLastAccount, key, account.name);
|
|
3593
|
+
// Home memory: ONE key per conversation, independent of identity health, so the
|
|
3594
|
+
// degraded <-> ok flip of _affinityKey can neither hide a home nor lend it to the
|
|
3595
|
+
// sibling conversations that share the terminal key. The entry carries its own
|
|
3596
|
+
// detour state: a Claude turn clears it, a provider turn sets it (same name and
|
|
3597
|
+
// `at`, only the LRU position moves so other traffic cannot evict a long detour).
|
|
3598
|
+
const homeKey = this._homeKey(requestInfo);
|
|
3599
|
+
if (!homeKey) return;
|
|
3600
|
+
if (isClaudeAccount(account)) {
|
|
3601
|
+
lruSet(this.conversationLastClaudeAccount, homeKey,
|
|
3602
|
+
{ name: account.name, at: requestInfo.now ?? Date.now(), detoured: false });
|
|
3603
|
+
} else {
|
|
3604
|
+
const home = this.conversationLastClaudeAccount.get(homeKey);
|
|
3605
|
+
if (home) lruSet(this.conversationLastClaudeAccount, homeKey, { ...home, detoured: true });
|
|
3606
|
+
}
|
|
3607
|
+
}
|
|
3608
|
+
|
|
3609
|
+
/** Key of the conversation's home entry: the conversation itself, never the shared
|
|
3610
|
+
* terminal key when a conversation id exists. When identity is genuinely broken
|
|
3611
|
+
* (a fresh hash every request) the home is simply never found, so the normal pick
|
|
3612
|
+
* stands — never a forced or wrong return. */
|
|
3613
|
+
_homeKey(requestInfo = {}) {
|
|
3614
|
+
return requestInfo.affinityKey || requestInfo.sessionKey || null;
|
|
3615
|
+
}
|
|
3616
|
+
|
|
3617
|
+
/**
|
|
3618
|
+
* RETURN HOME (owner 2026-11-09): the Claude account a conversation should go back to
|
|
3619
|
+
* after a provider detour, or null. A detour = a provider turn was served after the
|
|
3620
|
+
* last Claude turn (the entry's own `detoured` flag), so a conversation that never
|
|
3621
|
+
* left Claude has no home to return to. `stale` when the stored copy has outlived
|
|
3622
|
+
* cacheAffinityReturnWindowMs — going home then buys nothing.
|
|
3623
|
+
*/
|
|
3624
|
+
_returnHome(requestInfo, now = Date.now()) {
|
|
3625
|
+
const homeKey = this._homeKey(requestInfo);
|
|
3626
|
+
if (!homeKey) return null;
|
|
3627
|
+
const home = this.conversationLastClaudeAccount.get(homeKey);
|
|
3628
|
+
if (!home || !home.detoured) return null;
|
|
3629
|
+
const ageMs = now - home.at;
|
|
3630
|
+
let windowMs = Number(this.scheduler.cacheAffinityReturnWindowMs);
|
|
3631
|
+
if (!Number.isFinite(windowMs) || windowMs < 0) windowMs = DEFAULT_SCHEDULER.cacheAffinityReturnWindowMs;
|
|
3632
|
+
return { name: home.name, stale: ageMs > windowMs, ageMs };
|
|
3245
3633
|
}
|
|
3246
3634
|
|
|
3247
3635
|
/** Conversation identity: terminal + prefix hash when the server computed one;
|
|
@@ -3320,13 +3708,25 @@ export class AccountManager {
|
|
|
3320
3708
|
return null;
|
|
3321
3709
|
}
|
|
3322
3710
|
|
|
3323
|
-
|
|
3711
|
+
/** `holderName` scores `account` as if it held the conversation (return-home after a
|
|
3712
|
+
* provider detour); omitted = the conversation's real last account. */
|
|
3713
|
+
_cacheAffinity(account, requestInfo = {}, criticalUnlock = null, holderName = undefined) {
|
|
3324
3714
|
const base = Number(this.scheduler.cacheAffinityBonus) || 0;
|
|
3325
3715
|
// Sticky mode has its own binding; its score loop runs only once the session has
|
|
3326
3716
|
// decided to LEAVE a hot account — a discount there would pull it straight back.
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3717
|
+
// D3 retry-same (one acquire, set by server.js after a first-byte timeout on a big
|
|
3718
|
+
// prompt): the account that just timed out holds this prompt's warm copy, so it is
|
|
3719
|
+
// scored AS THE HOLDER — the same bonus, release gates, priority tiers, weekly
|
|
3720
|
+
// passes, fairness and anti-dogpile cap as any held conversation (red-team r1: no
|
|
3721
|
+
// bypass). Only the cooldown is skipped, upstream (server.js never cools it). It
|
|
3722
|
+
// applies in sticky mode too: pulling the turn back to that account IS the intent,
|
|
3723
|
+
// for exactly one acquire.
|
|
3724
|
+
const retrySame = requestInfo.retrySameAccountName != null && requestInfo.retrySameAccountName === account.name;
|
|
3725
|
+
if (base <= 0 || (this.scheduler.routingMode === 'sticky' && !retrySame)) return 0;
|
|
3726
|
+
if (!retrySame) {
|
|
3727
|
+
const key = this._affinityKey(requestInfo);
|
|
3728
|
+
if (!key || (holderName ?? this.conversationLastAccount.get(key)) !== account.name) return 0;
|
|
3729
|
+
}
|
|
3330
3730
|
// Scale with the CONVERSATION'S SIZE too: losing the cache costs the prefix a
|
|
3331
3731
|
// hop re-writes, and requestInfo.bodyBytes ≈ prefix + this turn. Checkpoint
|
|
3332
3732
|
// 2026-11-08 11:07Z: a flat 400 cut full re-writes only 18.9%→14.8% of turns —
|
|
@@ -3363,7 +3763,9 @@ export class AccountManager {
|
|
|
3363
3763
|
* names the reason, incl. availability causes); `outscored:` = the hold was live and
|
|
3364
3764
|
* the sibling still won, with the dominant score terms that beat the bonus.
|
|
3365
3765
|
*/
|
|
3366
|
-
_moveReason(lastName, best, requestInfo, excludedIndexes, scoringCtx, weeklyOptions, bestUnlock, now) {
|
|
3766
|
+
_moveReason(lastName, best, requestInfo, excludedIndexes, scoringCtx, weeklyOptions, bestUnlock, now, asHolder = false) {
|
|
3767
|
+
// asHolder: explain why `lastName` lost while scored AS the holder (return-home).
|
|
3768
|
+
const holder = asHolder ? lastName : undefined;
|
|
3367
3769
|
const last = this.accounts.find(a => a.name === lastName);
|
|
3368
3770
|
if (!last) return 'released:account-gone';
|
|
3369
3771
|
if (excludedIndexes.has(last.index)) return 'released:excluded';
|
|
@@ -3374,8 +3776,8 @@ export class AccountManager {
|
|
|
3374
3776
|
const gate = this._affinityRelease(last, unlock);
|
|
3375
3777
|
if (gate) return `released:${gate}`;
|
|
3376
3778
|
if (this._effectivePriority(last, requestInfo, now) > this._effectivePriority(best, requestInfo, now)) return 'released:priority-tier';
|
|
3377
|
-
if (this._cacheAffinity(last, requestInfo, unlock) <= 0) return 'released:no-hold';
|
|
3378
|
-
const lt = this._scoreTerms(last, requestInfo, scoringCtx, unlock);
|
|
3779
|
+
if (this._cacheAffinity(last, requestInfo, unlock, holder) <= 0) return 'released:no-hold';
|
|
3780
|
+
const lt = this._scoreTerms(last, requestInfo, scoringCtx, unlock, holder);
|
|
3379
3781
|
const bt = this._scoreTerms(best, requestInfo, scoringCtx, bestUnlock);
|
|
3380
3782
|
const sum = t => t.concurrency + t.capPenalty + t.paceCost + t.utilizationCost + t.scopedPace + t.spread + t.ramp + t.reserveCost + t.criticalCost + t.failurePenalty - t.affinity;
|
|
3381
3783
|
const delta = sum(lt) - sum(bt);
|
|
@@ -3731,6 +4133,13 @@ export class AccountManager {
|
|
|
3731
4133
|
// session window is its ONLY quota signal. Session availability otherwise
|
|
3732
4134
|
// belongs to _isSessionQuotaUnavailable + PROVIDER_SES_WALL.
|
|
3733
4135
|
const used = wkUsed != null ? wkUsed : (sesUsed ?? 0);
|
|
4136
|
+
// C3 CAP LIFT (provider twin): a lifted z.ai account tiers `normal` below
|
|
4137
|
+
// the wall so it can climb to the 0.90 card wall instead of being stranded
|
|
4138
|
+
// in reserve/critical by its own cap.
|
|
4139
|
+
if (this.capLiftActive(account, now)) {
|
|
4140
|
+
const ceil = this._capLiftCeiling(account, 'wk', now);
|
|
4141
|
+
if ((ceil == null && used < this.scheduler.weeklyExhaustedThreshold) || (ceil != null && used < ceil)) return 'normal';
|
|
4142
|
+
}
|
|
3734
4143
|
// USAGE CAP — checked FIRST, before every tier: a reservation is owner intent
|
|
3735
4144
|
// and outranks both the tier ladder and the upstream verdict. There is no
|
|
3736
4145
|
// upstreamAllows carve-out for providers anyway, but the ordering documents
|
|
@@ -3749,6 +4158,16 @@ export class AccountManager {
|
|
|
3749
4158
|
if (q.unified7d == null) return 'unknown';
|
|
3750
4159
|
|
|
3751
4160
|
const used = clamp01(q.unified7d);
|
|
4161
|
+
// C3 CAP LIFT (pre-mortem #10): while a general reset can be spent, the
|
|
4162
|
+
// soft/reserve/critical tiers would strand it — above 0.95 an account is
|
|
4163
|
+
// last-resort-only, so it never reaches the 0.99 claim point. Below the wall
|
|
4164
|
+
// (0.999, or the give-back ceiling cap+drop) a lifted account tiers as
|
|
4165
|
+
// `normal`, at ordinary share; above the ceiling the ordinary logic applies.
|
|
4166
|
+
if (this.capLiftActive(account, now)) {
|
|
4167
|
+
const ceil = this._capLiftCeiling(account, 'wk', now);
|
|
4168
|
+
if (ceil == null && used < this.scheduler.weeklyExhaustedThreshold) return 'normal';
|
|
4169
|
+
if (ceil != null && used < ceil) return 'normal';
|
|
4170
|
+
}
|
|
3752
4171
|
// NEVER bench on utilization alone while the upstream explicitly says the account is
|
|
3753
4172
|
// ALLOWED. Anthropic reports both a percentage and a verdict; the verdict is the
|
|
3754
4173
|
// authority, and it keeps saying "allowed"/"allowed_warning" at 100% used. This mirrors
|
|
@@ -3800,7 +4219,7 @@ export class AccountManager {
|
|
|
3800
4219
|
* Update an account's quota from a background usage probe (fetchUsage result).
|
|
3801
4220
|
* Same effect as learning quota from a live response, but for idle accounts.
|
|
3802
4221
|
*/
|
|
3803
|
-
applyUsageData(accountIndex, usage) {
|
|
4222
|
+
applyUsageData(accountIndex, usage, { sentAt = null } = {}) {
|
|
3804
4223
|
// A successful probe ends any escalation streak.
|
|
3805
4224
|
if (this.accounts[accountIndex]?.quota) this.accounts[accountIndex].quota.consecutiveProbeFailures = 0;
|
|
3806
4225
|
const account = this.accounts[accountIndex];
|
|
@@ -3877,6 +4296,18 @@ export class AccountManager {
|
|
|
3877
4296
|
}
|
|
3878
4297
|
|
|
3879
4298
|
q.lastProbeOkAt = Date.now();
|
|
4299
|
+
// C1 (finding 1): the SEND time orders this reading against a claim — a probe
|
|
4300
|
+
// sent before a claim and applied after it carries the pre-claim reading.
|
|
4301
|
+
q.lastProbeOkSentAt = Number.isFinite(sentAt) ? sentAt : q.lastProbeOkAt;
|
|
4302
|
+
// C3 give-back measurement: the FIRST successful probe after a give-back claim
|
|
4303
|
+
// records how far unified7d DROPPED — the value a give-back grant actually
|
|
4304
|
+
// returns (it only returns what the model used, never the whole weekly;
|
|
4305
|
+
// pre-mortem #7). Later lifts for this account use cap + drop as the ceiling.
|
|
4306
|
+
if (account.giveBack?.pendingAt != null && q.lastProbeOkSentAt > account.giveBack.pendingAt
|
|
4307
|
+
&& usage.sevenDay?.utilization != null) {
|
|
4308
|
+
const utilBefore = account.giveBack.utilBefore;
|
|
4309
|
+
account.giveBack = { drop: Math.max(0, utilBefore - clamp01(usage.sevenDay.utilization)), at: Date.now() };
|
|
4310
|
+
}
|
|
3880
4311
|
// A successful probe clears any recorded failure — freshness confirmed — including
|
|
3881
4312
|
// the org-403 streak (a healthy probe proves the org still accepts OAuth).
|
|
3882
4313
|
q.consecutiveOrgGone = 0;
|
|
@@ -3976,17 +4407,29 @@ export class AccountManager {
|
|
|
3976
4407
|
* the OAuth quota gates.
|
|
3977
4408
|
*/
|
|
3978
4409
|
/** Store the latest Claude reset-grant listing (prober piggyback). */
|
|
3979
|
-
applyResetGrants(accountIndex, listing) {
|
|
4410
|
+
applyResetGrants(accountIndex, listing, now = Date.now()) {
|
|
3980
4411
|
const account = this.accounts[accountIndex];
|
|
3981
4412
|
if (!account || !listing) return;
|
|
3982
|
-
if (listing.error) {
|
|
4413
|
+
if (listing.error) {
|
|
4414
|
+
// A failed check must not erase the last good count (2026-11-10: "2~119d" went
|
|
4415
|
+
// blank after one transient non-2xx, with no reset claimed). Keep the previous
|
|
4416
|
+
// listing, mark it stale, and keep checkedAt = the last GOOD read. Never good
|
|
4417
|
+
// before → stay null (nothing to show).
|
|
4418
|
+
if (account.resetGrants) {
|
|
4419
|
+
account.resetGrants = { ...account.resetGrants, stale: true, lastError: String(listing.error), lastErrorAt: now };
|
|
4420
|
+
}
|
|
4421
|
+
return;
|
|
4422
|
+
}
|
|
3983
4423
|
account.resetGrants = {
|
|
3984
4424
|
grants: listing.grants || [],
|
|
3985
4425
|
nextGrantId: listing.nextGrantId ?? null,
|
|
3986
4426
|
cooldownUntil: listing.cooldownUntil ?? null,
|
|
3987
4427
|
eligible: listing.eligible !== false,
|
|
3988
|
-
checkedAt:
|
|
4428
|
+
checkedAt: now,
|
|
4429
|
+
stale: false,
|
|
3989
4430
|
};
|
|
4431
|
+
// C3: re-evaluate the cap lift from the fresh listing.
|
|
4432
|
+
this.refreshCapLift(accountIndex, now, 'listing refreshed');
|
|
3990
4433
|
}
|
|
3991
4434
|
|
|
3992
4435
|
/** Store the latest reset-card listing for an account (prober piggyback).
|
|
@@ -4025,11 +4468,50 @@ export class AccountManager {
|
|
|
4025
4468
|
applyResetCards(accountIndex, cards) {
|
|
4026
4469
|
const account = this.accounts[accountIndex];
|
|
4027
4470
|
if (!account || !cards) return;
|
|
4028
|
-
if (cards.error) {
|
|
4471
|
+
if (cards.error) {
|
|
4472
|
+
// A4 (2026-11-10): keep-last-good, same as Claude grants. The old
|
|
4473
|
+
// `resetCards = null` blanked the count on any transient error (the z.ai
|
|
4474
|
+
// twin of the "2 → gone → back" flicker) and handed a wrong 0 to routing.
|
|
4475
|
+
// Never good before → stay null.
|
|
4476
|
+
if (account.resetCards) {
|
|
4477
|
+
account.resetCards = { ...account.resetCards, stale: true, lastError: String(cards.error), lastErrorAt: Date.now() };
|
|
4478
|
+
}
|
|
4479
|
+
return;
|
|
4480
|
+
}
|
|
4481
|
+
// DRIFT = a 200 the parser can't have meant (pre-mortem #17): expected arrays
|
|
4482
|
+
// missing, or raw entries present but none kept — treated EXACTLY like an
|
|
4483
|
+
// error: keep the last good cards (marked stale) and leave the cap lift on
|
|
4484
|
+
// its grace window (finding 11: re-evaluating the lift on a drifted listing
|
|
4485
|
+
// ended it at once instead of after the grace).
|
|
4486
|
+
const drift = !Array.isArray(cards.fiveHour) || !Array.isArray(cards.weekly);
|
|
4487
|
+
if (drift) {
|
|
4488
|
+
if (account.resetCards) {
|
|
4489
|
+
account.resetCards = { ...account.resetCards, stale: true, lastError: 'listing drift (unexpected payload shape)', lastErrorAt: Date.now() };
|
|
4490
|
+
}
|
|
4491
|
+
return;
|
|
4492
|
+
}
|
|
4029
4493
|
account.resetCards = {
|
|
4030
|
-
fiveHour: cards.fiveHour
|
|
4031
|
-
weekly: cards.weekly
|
|
4494
|
+
fiveHour: cards.fiveHour,
|
|
4495
|
+
weekly: cards.weekly,
|
|
4032
4496
|
checkedAt: Date.now(),
|
|
4497
|
+
stale: false,
|
|
4498
|
+
lastError: null,
|
|
4499
|
+
};
|
|
4500
|
+
// C3: a z.ai weekly card can start the lift too.
|
|
4501
|
+
this.refreshCapLift(accountIndex, Date.now(), 'cards refreshed');
|
|
4502
|
+
}
|
|
4503
|
+
|
|
4504
|
+
/** Remove one card recordId from the stored list after a SUCCESSFUL redeem
|
|
4505
|
+
* (A4): the follow-up read can fail, and a redeemed card that lingers keeps
|
|
4506
|
+
* feeding _holdsUsableReset a discount for capacity that no longer exists. */
|
|
4507
|
+
removeResetCard(accountIndex, recordId) {
|
|
4508
|
+
const account = this.accounts[accountIndex];
|
|
4509
|
+
if (!account?.resetCards || recordId == null) return;
|
|
4510
|
+
const rm = c => c.recordId !== recordId;
|
|
4511
|
+
account.resetCards = {
|
|
4512
|
+
...account.resetCards,
|
|
4513
|
+
fiveHour: (account.resetCards.fiveHour || []).filter(rm),
|
|
4514
|
+
weekly: (account.resetCards.weekly || []).filter(rm),
|
|
4033
4515
|
};
|
|
4034
4516
|
}
|
|
4035
4517
|
|
|
@@ -4618,10 +5100,35 @@ export class AccountManager {
|
|
|
4618
5100
|
console.log(`[Maxpool] Account "${account.name}" disabled after HTTP ${status} (${reason})`);
|
|
4619
5101
|
}
|
|
4620
5102
|
|
|
4621
|
-
markTransientFailure(accountIndex, reason = 'transient_error', { network = false } = {}) {
|
|
5103
|
+
markTransientFailure(accountIndex, reason = 'transient_error', { network = false, scope = 'account' } = {}) {
|
|
4622
5104
|
const account = this.accounts[accountIndex];
|
|
4623
5105
|
if (!account) return;
|
|
4624
5106
|
|
|
5107
|
+
// REQUEST-SCOPED (review r2 A): a first-byte/idle timeout on ONE request says nothing
|
|
5108
|
+
// about the account's other conversations. Deprioritise it for new placements only
|
|
5109
|
+
// (softCooldownUntil — held conversations keep it), and count consecutive request-
|
|
5110
|
+
// scoped faults. REQUEST_FAULT_ESCALATE in a row (no success between) is no longer
|
|
5111
|
+
// "one slow request": escalate to the account-scoped shape below (real cooldown +
|
|
5112
|
+
// failure streak → held conversations released), logged once per run.
|
|
5113
|
+
if (network && scope === 'request') {
|
|
5114
|
+
account.failedRequests++;
|
|
5115
|
+
account.lastError = reason;
|
|
5116
|
+
account.lastErrorAt = Date.now();
|
|
5117
|
+
account.requestFaultStreak = (account.requestFaultStreak || 0) + 1;
|
|
5118
|
+
if (account.requestFaultStreak < REQUEST_FAULT_ESCALATE) {
|
|
5119
|
+
account.softCooldownUntil = Date.now() + this.scheduler.networkCooldownMs;
|
|
5120
|
+
console.log(`[Maxpool] Account "${account.name}" deprioritised for new placements for ${Math.ceil(this.scheduler.networkCooldownMs / 1000)}s after ${reason} (request-scoped ${account.requestFaultStreak}/${REQUEST_FAULT_ESCALATE}; held conversations stay)`);
|
|
5121
|
+
return;
|
|
5122
|
+
}
|
|
5123
|
+
account.consecutiveFailures++;
|
|
5124
|
+
account.cooldownUntil = Date.now() + this.scheduler.networkCooldownMs;
|
|
5125
|
+
if (!account.requestFaultEscalated) {
|
|
5126
|
+
account.requestFaultEscalated = true;
|
|
5127
|
+
console.log(`[Maxpool] Account "${account.name}": ${account.requestFaultStreak} request-scoped faults in a row (${reason}) — treating as an ACCOUNT fault: cooling down ${Math.ceil(this.scheduler.networkCooldownMs / 1000)}s, held conversations released`);
|
|
5128
|
+
}
|
|
5129
|
+
return;
|
|
5130
|
+
}
|
|
5131
|
+
|
|
4625
5132
|
if (network) {
|
|
4626
5133
|
// A network-class failure (lost connectivity / token-refresh `fetch failed`)
|
|
4627
5134
|
// is a FLEET-WIDE condition, not this account's fault — every account fails at
|
|
@@ -4849,6 +5356,9 @@ export class AccountManager {
|
|
|
4849
5356
|
account.refreshDead = false; // fresh tokens from re-auth revive a dead-refresh account
|
|
4850
5357
|
account.subscriptionGone = false; // a working re-auth means the org accepts OAuth again
|
|
4851
5358
|
if (account.status === 'error') account.status = 'active';
|
|
5359
|
+
// A re-login may be a DIFFERENT org under the same name: forget the cached
|
|
5360
|
+
// claim org and the claim backoff (finding 10).
|
|
5361
|
+
this._forgetClaimState(account.name);
|
|
4852
5362
|
console.log(`[Maxpool] Updated tokens for account "${account.name}"`);
|
|
4853
5363
|
this._onTokenRefresh?.(accountIndex, {
|
|
4854
5364
|
accessToken,
|
|
@@ -5113,6 +5623,9 @@ export class AccountManager {
|
|
|
5113
5623
|
const removed = this.accounts[index];
|
|
5114
5624
|
if (removed.inFlight > 0) return false;
|
|
5115
5625
|
const removedName = removed.name;
|
|
5626
|
+
// A later account under the same name must start clean (finding 10). Mirrors
|
|
5627
|
+
// never claimed, so only local rows carry claim state.
|
|
5628
|
+
if (!removed.remote) this._forgetClaimState(removedName);
|
|
5116
5629
|
this.accounts.splice(index, 1);
|
|
5117
5630
|
this.accounts.forEach((a, i) => a.index = i);
|
|
5118
5631
|
if (this.currentIndex >= this.accounts.length) {
|
|
@@ -5330,8 +5843,15 @@ export class AccountManager {
|
|
|
5330
5843
|
a.capCeiling = ra.capCeiling ?? a.capCeiling;
|
|
5331
5844
|
a.capRampStart = ra.capRampStart ?? a.capRampStart;
|
|
5332
5845
|
a.addedAt = null; // H3: mirrors never count as "warming"
|
|
5333
|
-
|
|
5334
|
-
|
|
5846
|
+
// B1/B2 (2026-11-10): one shared normalizeRemoteResets for the mirror row AND
|
|
5847
|
+
// the benched-twin overlay — an OLDER cloud omits `eligible`/new fields; a
|
|
5848
|
+
// missing `eligible` reads grants.length>0 so its holdings still display.
|
|
5849
|
+
// The lift is NEVER recomputed here (C3/pre-mortem #13): the exported
|
|
5850
|
+
// capLift flag is honoured as-is, missing = no lift.
|
|
5851
|
+
const rr = normalizeRemoteResets(ra);
|
|
5852
|
+
a.resetCards = rr.resetCards;
|
|
5853
|
+
a.resetGrants = rr.resetGrants;
|
|
5854
|
+
a.capLift = rr.capLift;
|
|
5335
5855
|
a.remoteMonthlyPriceUSD = Number.isFinite(ra.monthlyPriceUSD) ? ra.monthlyPriceUSD : null;
|
|
5336
5856
|
a.remoteTank = ra.tank || null;
|
|
5337
5857
|
a.remoteKeyFp = ra.keyFingerprint || null;
|
|
@@ -5360,6 +5880,154 @@ export class AccountManager {
|
|
|
5360
5880
|
return Boolean(account && account.poolBenchedBy);
|
|
5361
5881
|
}
|
|
5362
5882
|
|
|
5883
|
+
// ── C2: the reactive claim (a 429 on the serving lease claims AT ONCE) ──────
|
|
5884
|
+
// Runs only on the serving lease's LOCAL account holding the credential — never
|
|
5885
|
+
// a mirror row (its 429 is the CLOUD's; the cloud claims on its own side) and
|
|
5886
|
+
// never a relayed 429 (pre-mortem #21: the wrong pool would double-spend).
|
|
5887
|
+
// The grant comes from the STORED listing: stale is allowed up to 6h with
|
|
5888
|
+
// endsAt>now (a fresh listing is exactly what's missing at the wall — the usage
|
|
5889
|
+
// endpoint 429s). The claim uses the CURRENT access token; a 401 defers to the
|
|
5890
|
+
// sweep (no refresh on this path).
|
|
5891
|
+
|
|
5892
|
+
/** Pick the grant for a reactive claim. weekly=true → a general grant; a family
|
|
5893
|
+
* name → that model's grant, only if the probe MEASURED the bucket at the wall
|
|
5894
|
+
* (a reactive 429's own 1.0 mark is not measurement). Returns the grant or null. */
|
|
5895
|
+
_reactiveClaimGrant(account, kind, now = Date.now()) {
|
|
5896
|
+
if (!account?.resetGrants || account.resetGrants.eligible === false) return null;
|
|
5897
|
+
const checkedAt = account.resetGrants.checkedAt;
|
|
5898
|
+
if (checkedAt == null || now - checkedAt > CLAIM_STALE_LISTING_MAX_MS) return null; // stored listing ≤ 6h
|
|
5899
|
+
const grants = (account.resetGrants.grants || [])
|
|
5900
|
+
.filter(g => g && g.usableNow !== false && !g.paused && !g.expired && !g.blocking && !g.unavailableReason
|
|
5901
|
+
&& (g.endsAt == null || g.endsAt > now) && (g.resetsLeft ?? 1) > 0)
|
|
5902
|
+
.sort((a, b) => (a.endsAt ?? Infinity) - (b.endsAt ?? Infinity));
|
|
5903
|
+
if (!grants.length) return null;
|
|
5904
|
+
if (kind === 'weekly') return grants.find(g => appliesTo(g).includes('general')) || null;
|
|
5905
|
+
// Model-scoped: the probe must have MEASURED that family's bucket at the wall.
|
|
5906
|
+
const e = account.quota?.scopedWeekly?.[kind];
|
|
5907
|
+
const measured = e && e.reactive !== true && e.utilization != null && e.utilization >= CLAUDE_WALL_WEEKLY;
|
|
5908
|
+
return measured ? (grants.find(g => appliesTo(g).includes(kind)) || null) : null;
|
|
5909
|
+
}
|
|
5910
|
+
|
|
5911
|
+
/** A claim SUCCEEDED: spend it in the stored listing at once (finding 1). Until
|
|
5912
|
+
* the next listing (≤ 5 min, or hours when the endpoint 429s) the stored copy
|
|
5913
|
+
* otherwise still offers the spent reset to the reactive path, the lift and
|
|
5914
|
+
* the routing discount. resetsLeft drops by one; at 0 the grant leaves. */
|
|
5915
|
+
noteGrantClaimed(account, grantId) {
|
|
5916
|
+
const rg = account?.resetGrants;
|
|
5917
|
+
if (!rg || !Array.isArray(rg.grants)) return;
|
|
5918
|
+
const grants = [];
|
|
5919
|
+
for (const g of rg.grants) {
|
|
5920
|
+
if (g?.id !== grantId) { grants.push(g); continue; }
|
|
5921
|
+
const left = Math.floor(g.resetsLeft ?? 1) - 1;
|
|
5922
|
+
if (left >= 1) grants.push({ ...g, resetsLeft: left });
|
|
5923
|
+
}
|
|
5924
|
+
account.resetGrants = { ...rg, grants };
|
|
5925
|
+
}
|
|
5926
|
+
|
|
5927
|
+
/** Forget everything claim-related keyed by this account name: the cached org
|
|
5928
|
+
* and the per-grant backoff (finding 10 — a same-name re-login to another org
|
|
5929
|
+
* must not POST to the old org or inherit its backoff). */
|
|
5930
|
+
_forgetClaimState(accountName) {
|
|
5931
|
+
forgetClaimOrg(accountName);
|
|
5932
|
+
const prefix = `${accountName}:`;
|
|
5933
|
+
for (const m of [this._claimTries, this._claimRetryUntil]) {
|
|
5934
|
+
if (!m) continue;
|
|
5935
|
+
for (const k of [...m.keys()]) if (k.startsWith(prefix)) m.delete(k);
|
|
5936
|
+
}
|
|
5937
|
+
}
|
|
5938
|
+
|
|
5939
|
+
/** C6: the ONE claim lock per account, shared by the prober and the reactive
|
|
5940
|
+
* path. Returns true when the lock was ACQUIRED (caller must release). */
|
|
5941
|
+
acquireClaimLock(accountName) {
|
|
5942
|
+
if (!this._claimLocks) this._claimLocks = new Set();
|
|
5943
|
+
if (this._claimLocks.has(accountName)) return false;
|
|
5944
|
+
this._claimLocks.add(accountName);
|
|
5945
|
+
return true;
|
|
5946
|
+
}
|
|
5947
|
+
|
|
5948
|
+
releaseClaimLock(accountName) {
|
|
5949
|
+
this._claimLocks?.delete(accountName);
|
|
5950
|
+
}
|
|
5951
|
+
|
|
5952
|
+
/** C6: record a claim FAILURE — backoff per (account, grantId) 15m/30m/1h, cap 6h.
|
|
5953
|
+
* Shared state with the prober path (same maps). Returns the wait ms. */
|
|
5954
|
+
noteClaimFailure(accountName, grantId, now = Date.now()) {
|
|
5955
|
+
const key = `${accountName}:${grantId}`;
|
|
5956
|
+
if (!this._claimTries) this._claimTries = new Map();
|
|
5957
|
+
if (!this._claimRetryUntil) this._claimRetryUntil = new Map();
|
|
5958
|
+
const tries = (this._claimTries.get(key) || 0) + 1;
|
|
5959
|
+
this._claimTries.set(key, tries);
|
|
5960
|
+
const steps = CLAIM_BACKOFF_STEPS_MS;
|
|
5961
|
+
const waitMs = Math.min(CLAIM_BACKOFF_CAP_MS, steps[Math.min(tries - 1, steps.length - 1)]);
|
|
5962
|
+
this._claimRetryUntil.set(key, now + waitMs);
|
|
5963
|
+
return { tries, waitMs };
|
|
5964
|
+
}
|
|
5965
|
+
|
|
5966
|
+
/** C6: a SUCCESSFUL claim clears the backoff for that grant. */
|
|
5967
|
+
noteClaimSuccess(accountName, grantId) {
|
|
5968
|
+
this._claimRetryUntil?.set(`${accountName}:${grantId}`, 0);
|
|
5969
|
+
this._claimTries?.set(`${accountName}:${grantId}`, 0);
|
|
5970
|
+
}
|
|
5971
|
+
|
|
5972
|
+
/** Try the reactive claim for a weekly-class 429 (7d header ≥ 0.999) or a scoped
|
|
5973
|
+
* 429 (model family). Returns true when a claim was attempted. NEVER called for
|
|
5974
|
+
* a mirror row or a benched/disabled account. Shares the C6 lock and backoff
|
|
5975
|
+
* state with the prober path. The server calls it fire-and-forget; the claim
|
|
5976
|
+
* itself gives up after REACTIVE_CLAIM_TIMEOUT_MS so the lock is never held for
|
|
5977
|
+
* minutes. A 401 (token rejected) defers to the next sweep — it refreshes the
|
|
5978
|
+
* token first — and is NOT counted as a failed claim (no backoff). */
|
|
5979
|
+
async tryReactiveClaim(account, { kind, weeklyUtil: _weeklyUtil = null }, now = Date.now()) {
|
|
5980
|
+
if (!account) return false;
|
|
5981
|
+
if (account.remote || account.poolBenchedBy || account.enabled === false) return false;
|
|
5982
|
+
if (!account.credential || account.type !== 'oauth') return false;
|
|
5983
|
+
// C6 (finding 1): ONE claim per wall — after a claim, only a probe SENT after
|
|
5984
|
+
// it may show the wall again. Repeated 429s on the same wall never re-claim.
|
|
5985
|
+
const lastClaimAt = account.lastClaimAt ?? null;
|
|
5986
|
+
if (lastClaimAt != null && !((probeSentAt(account.quota) ?? -Infinity) > lastClaimAt)) return false;
|
|
5987
|
+
// No grant (or a scoped 429 without a probe-measured bucket) → the next sweep
|
|
5988
|
+
// decides on its own fresh reading.
|
|
5989
|
+
const grant = this._reactiveClaimGrant(account, kind, now);
|
|
5990
|
+
if (!grant) return false;
|
|
5991
|
+
// C6: shared backoff per (account, grantId).
|
|
5992
|
+
const key = `${account.name}:${grant.id}`;
|
|
5993
|
+
const until = this._claimRetryUntil?.get(key);
|
|
5994
|
+
if (until && until > now) return false;
|
|
5995
|
+
if (!this.acquireClaimLock(account.name)) return false;
|
|
5996
|
+
try {
|
|
5997
|
+
const { claimResetGrant } = await import('./claude-reset-grants.js');
|
|
5998
|
+
const r = await claimResetGrant(account.credential, grant.id, {
|
|
5999
|
+
accountKey: account.name, signal: AbortSignal.timeout(REACTIVE_CLAIM_TIMEOUT_MS),
|
|
6000
|
+
});
|
|
6001
|
+
if (!r?.ok && r?.status === 401) {
|
|
6002
|
+
console.log(`[reset-grant] ${account.name}: reactive claim deferred to the next sweep (token rejected)`);
|
|
6003
|
+
return true; // attempted; no backoff, no lift change
|
|
6004
|
+
}
|
|
6005
|
+
if (r?.ok) {
|
|
6006
|
+
this.noteClaimSuccess(account.name, grant.id);
|
|
6007
|
+
account.lastClaimAt = Date.now();
|
|
6008
|
+
this.noteGrantClaimed(account, grant.id);
|
|
6009
|
+
// Clear the wall this claim refills.
|
|
6010
|
+
if (kind !== 'weekly' && account.quota?.scopedWeekly?.[kind]) delete account.quota.scopedWeekly[kind];
|
|
6011
|
+
if (account.status === 'throttled') {
|
|
6012
|
+
account.status = 'active'; account.rateLimitedUntil = null;
|
|
6013
|
+
if (account.lastError === 'rate_limited') { account.lastError = null; account.lastErrorAt = null; }
|
|
6014
|
+
}
|
|
6015
|
+
this.endCapLift(account.index, 'reactive claim succeeded');
|
|
6016
|
+
console.log(`[reset-grant] ${account.name}: grant ${grant.id} claimed reactively (${kind === 'weekly' ? 'weekly 429 at the wall' : `${kind} scoped 429`})`);
|
|
6017
|
+
return true;
|
|
6018
|
+
}
|
|
6019
|
+
const { tries, waitMs } = this.noteClaimFailure(account.name, grant.id, now);
|
|
6020
|
+
console.log(`[reset-grant] ${account.name}: reactive claim FAILED ${grant.id} (try ${tries}, next attempt in ${Math.round(waitMs / 60000)}min): ${r?.error}`);
|
|
6021
|
+
this.endCapLift(account.index, 'reactive claim failed');
|
|
6022
|
+
return true; // attempted
|
|
6023
|
+
} catch (e) {
|
|
6024
|
+
console.log(`[reset-grant] ${account.name}: reactive claim error ${e.message}`);
|
|
6025
|
+
return true;
|
|
6026
|
+
} finally {
|
|
6027
|
+
this.releaseClaimLock(account.name);
|
|
6028
|
+
}
|
|
6029
|
+
}
|
|
6030
|
+
|
|
5363
6031
|
getStatus() {
|
|
5364
6032
|
const now = Date.now();
|
|
5365
6033
|
return {
|
|
@@ -5396,8 +6064,30 @@ export class AccountManager {
|
|
|
5396
6064
|
remote: Boolean(a.remote),
|
|
5397
6065
|
// Reset cards / grants (display + cross-pool routing preference). Card ids,
|
|
5398
6066
|
// counts and expiry only — no credentials; the redeem stays with the holder.
|
|
5399
|
-
|
|
5400
|
-
|
|
6067
|
+
// B1 (2026-11-10): carries eligible/stale/lastError/checkedAt/capLift/giveBack
|
|
6068
|
+
// so a linked pool (and its TUI) can display and honour them; a DISABLED
|
|
6069
|
+
// account exports null resets (pre-mortem #20: the API must not report
|
|
6070
|
+
// resets for accounts the pool can't use).
|
|
6071
|
+
resetCards: a.enabled === false || !a.resetCards ? null : {
|
|
6072
|
+
fiveHour: a.resetCards.fiveHour, weekly: a.resetCards.weekly,
|
|
6073
|
+
checkedAt: a.resetCards.checkedAt, stale: Boolean(a.resetCards.stale), lastError: a.resetCards.lastError ?? null,
|
|
6074
|
+
},
|
|
6075
|
+
resetGrants: a.enabled === false || !a.resetGrants ? null : {
|
|
6076
|
+
grants: a.resetGrants.grants, cooldownUntil: a.resetGrants.cooldownUntil,
|
|
6077
|
+
checkedAt: a.resetGrants.checkedAt, stale: Boolean(a.resetGrants.stale), lastError: a.resetGrants.lastError ?? null,
|
|
6078
|
+
eligible: a.resetGrants.eligible !== false,
|
|
6079
|
+
},
|
|
6080
|
+
// C3: the holder pool's cap-lift verdict. Mirrors honour this as-is;
|
|
6081
|
+
// missing = no lift. Never recomputed on the mirror side.
|
|
6082
|
+
// `ceiling` = the holder's lifted cap (null = full lift): a give-back lift
|
|
6083
|
+
// exported without it read as a FULL lift on every mirror (finding 2).
|
|
6084
|
+
// The EVALUATED verdict (grace, refill proximity, dead/benched), never the
|
|
6085
|
+
// raw stored flag — a lift that capLiftActive would end must not reach mirrors.
|
|
6086
|
+
capLift: (a.remote ? a.capLift?.active === true : this.capLiftActive(a, now))
|
|
6087
|
+
? { active: true, since: a.capLift.since ?? null, mode: a.capLift.mode ?? null,
|
|
6088
|
+
ceiling: a.remote ? (a.capLift.ceiling ?? null) : this._capLiftCeiling(a, 'wk', now) }
|
|
6089
|
+
: { active: false },
|
|
6090
|
+
giveBack: a.giveBack ?? null,
|
|
5401
6091
|
// Capacity-page inputs for linked pools (a mirror has no credential and no
|
|
5402
6092
|
// local cycles): this pool's plan price and measured tanks, per window.
|
|
5403
6093
|
monthlyPriceUSD: a.remote ? null : this.monthlyPriceFor(a),
|