maxpool 1.24.80 → 1.24.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +208 -21
- package/src/server.js +424 -93
package/package.json
CHANGED
package/src/account-manager.js
CHANGED
|
@@ -4,6 +4,9 @@ import { appendEventLog } from './event-log.js';
|
|
|
4
4
|
import { hourFloor } from './outside-usage.js';
|
|
5
5
|
import { CapacityLedger, TANK_MIN_UTIL, mergeCycles, tankStatsFromCycles } from './capacity-ledger.js';
|
|
6
6
|
|
|
7
|
+
// Review r2 A: consecutive request-scoped faults before they count against the account.
|
|
8
|
+
const REQUEST_FAULT_ESCALATE = 3;
|
|
9
|
+
|
|
7
10
|
// Nominal window length per kind — mirrors capacity-ledger's WINDOW_MS (kept here as a
|
|
8
11
|
// local table so account-manager does not import a private constant).
|
|
9
12
|
const WINDOW_MS_BY_KIND = { ses: 5 * 3600_000, wk: 7 * 86400_000 };
|
|
@@ -208,7 +211,13 @@ const DEFAULT_SCHEDULER = {
|
|
|
208
211
|
// This is the 5-hour-window level at which the hold lets go (providers; Claude
|
|
209
212
|
// benches at switchThreshold anyway). Weekly reserve/soft tiers do NOT release.
|
|
210
213
|
cacheAffinityFiveHourWall: 0.97,
|
|
211
|
-
|
|
214
|
+
// RETURN HOME after a provider detour (owner 2026-11-09): when a conversation's turn
|
|
215
|
+
// went to GLM/Kimi and a later turn is routed back to Claude, it prefers the Claude
|
|
216
|
+
// account it used last — that account still holds its stored copy. Only while the copy
|
|
217
|
+
// is alive: Claude Code writes 1-hour copies (measured 2026-11-09: 272 of 272 cache
|
|
218
|
+
// writes `ephemeral_1h`; 55 of 64 detours came back within the hour, median 533 s).
|
|
219
|
+
cacheAffinityReturnWindowMs: 3_600_000,
|
|
220
|
+
concurrencyWeight: 2, // multiplies in-flight load (activeWeight+reqWeight) — dominant
|
|
212
221
|
perAccountConcurrencyTarget: 3, // D: soft per-account in-flight target; past it, capPenalty bites
|
|
213
222
|
capPenaltyWeight: 10, // steep penalty per unit of in-flight depth past D (throttle safety floor)
|
|
214
223
|
paceCostWeight: 1.5, // soft de-preference of accounts burning ahead of pace (was the ×6 term)
|
|
@@ -365,6 +374,15 @@ const PERSISTED_QUOTA_FIELDS = [
|
|
|
365
374
|
];
|
|
366
375
|
|
|
367
376
|
const CONVERSATION_AFFINITY_MAX = 5000;
|
|
377
|
+
/** Bounded LRU insert: re-insert = most recent; the oldest key drops past the cap. */
|
|
378
|
+
function lruSet(map, key, value) {
|
|
379
|
+
map.delete(key);
|
|
380
|
+
map.set(key, value);
|
|
381
|
+
while (map.size > CONVERSATION_AFFINITY_MAX) map.delete(map.keys().next().value);
|
|
382
|
+
}
|
|
383
|
+
/** An Anthropic account with its own prompt cache (allow-list: a provider or legacy
|
|
384
|
+
* pool row can forward to GLM, so "not a provider" is not proof of a Claude cache). */
|
|
385
|
+
const isClaudeAccount = (a) => a?.type === 'oauth' || a?.type === 'apikey';
|
|
368
386
|
const IDENTITY_WINDOW_MS = 15 * 60_000;
|
|
369
387
|
const IDENTITY_MAX_SAMPLES = 4000;
|
|
370
388
|
const IDENTITY_MIN_PER_SESSION = 8;
|
|
@@ -631,6 +649,9 @@ export class AccountManager {
|
|
|
631
649
|
// Bounded LRU — oldest conversations drop first; a dropped entry only means the
|
|
632
650
|
// next turn is balanced fresh.
|
|
633
651
|
this.conversationLastAccount = new Map();
|
|
652
|
+
// Same key → { name, at } of the last CLAUDE account only (a GLM/Kimi turn never
|
|
653
|
+
// overwrites it), so a conversation returning from a provider detour can go home.
|
|
654
|
+
this.conversationLastClaudeAccount = new Map();
|
|
634
655
|
// Conversation-identity HEALTH (owner 2026-11-08: "make sure this cannot degrade
|
|
635
656
|
// silently"). Twice in one day a Claude Code request-shape detail (a moving
|
|
636
657
|
// cache_control marker, then a per-request billing checksum) made every turn hash
|
|
@@ -906,7 +927,11 @@ export class AccountManager {
|
|
|
906
927
|
const prevCurrentName = requestInfo.sessionKey
|
|
907
928
|
? this._sessionBinding(requestInfo.sessionKey)?.currentName
|
|
908
929
|
: null;
|
|
930
|
+
requestInfo.returnedHome = null; // set by the selection loop when it sends the turn home
|
|
909
931
|
const account = this.getActiveAccount(requestInfo, excludedIndexes);
|
|
932
|
+
// The D3 retry-same hint is good for exactly ONE acquire, honored or not — a stale
|
|
933
|
+
// hint would pull a later failover hop back onto the account it just left.
|
|
934
|
+
requestInfo.retrySameAccountName = null;
|
|
910
935
|
if (!account) return null;
|
|
911
936
|
|
|
912
937
|
const weight = Math.max(1, Number(requestInfo.weight) || 1);
|
|
@@ -924,7 +949,9 @@ export class AccountManager {
|
|
|
924
949
|
this.notePoolSend(account);
|
|
925
950
|
// Non-null only when this acquire moved the session off its prior account.
|
|
926
951
|
const migratedFromName = (prevCurrentName && account.name !== prevCurrentName) ? prevCurrentName : null;
|
|
927
|
-
|
|
952
|
+
const returnedHome = requestInfo.returnedHome?.name === account.name;
|
|
953
|
+
const returnedHomeSwitched = returnedHome && requestInfo.returnedHome.switched === true;
|
|
954
|
+
return { account, weight, startedAt: Date.now(), upstreamThrottleProbe, migratedFromName, returnedHome, returnedHomeSwitched };
|
|
928
955
|
}
|
|
929
956
|
|
|
930
957
|
/** Fail-safe: snap a session's binding back to the pre-migration issuing account
|
|
@@ -1035,13 +1062,31 @@ export class AccountManager {
|
|
|
1035
1062
|
return out;
|
|
1036
1063
|
}
|
|
1037
1064
|
|
|
1065
|
+
/**
|
|
1066
|
+
* outcome.neutral — the attempt says nothing about the account (client cancel,
|
|
1067
|
+
* request-shape refusal): no counters move at all.
|
|
1068
|
+
* outcome.scope — 'account' (default) | 'request'. A request-scoped failure is
|
|
1069
|
+
* recorded (failedRequests, lastError) but never extends the
|
|
1070
|
+
* consecutiveFailures streak, so it cannot trip the hold-release
|
|
1071
|
+
* gate for the account's other conversations.
|
|
1072
|
+
*/
|
|
1038
1073
|
releaseAccount(lease, outcome = {}) {
|
|
1039
1074
|
if (!lease?.account) return;
|
|
1040
1075
|
const account = lease.account;
|
|
1041
1076
|
|
|
1077
|
+
const depthBefore = account.activeWeight;
|
|
1042
1078
|
account.inFlight = Math.max(0, account.inFlight - 1);
|
|
1043
1079
|
account.activeWeight = Math.max(0, account.activeWeight - lease.weight);
|
|
1044
1080
|
|
|
1081
|
+
// Owner's throttling constraint: make any throttle on a return-home turn countable,
|
|
1082
|
+
// and say whether the switch (vs the loop's own pick) put the turn on home.
|
|
1083
|
+
if (lease.returnedHome && (outcome.status === 429 || (outcome.status === 503 && account.remote)) && outcome.error !== 'request_shape_refused') {
|
|
1084
|
+
// A pool-wide throttle (markUpstreamThrottled) is not about home's depth — tag it so
|
|
1085
|
+
// the post-release count can leave it out.
|
|
1086
|
+
const scope = outcome.upstreamThrottled ? ' pool-wide' : '';
|
|
1087
|
+
console.log(`[Maxpool] return-home lease throttled (${outcome.status} ${outcome.error || '-'}${scope}, ${lease.returnedHomeSwitched ? 'switched' : 'natural'}): ${account.name} depth=${depthBefore}`);
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1045
1090
|
if (lease.upstreamThrottleProbe) {
|
|
1046
1091
|
if (outcome.success) {
|
|
1047
1092
|
this.clearUpstreamThrottle('successful recovery probe');
|
|
@@ -1060,6 +1105,10 @@ export class AccountManager {
|
|
|
1060
1105
|
if (outcome.success) {
|
|
1061
1106
|
account.completedRequests++;
|
|
1062
1107
|
account.consecutiveFailures = 0;
|
|
1108
|
+
// Review r2 A: any success ends a request-scoped fault run.
|
|
1109
|
+
account.requestFaultStreak = 0;
|
|
1110
|
+
account.requestFaultEscalated = false;
|
|
1111
|
+
account.softCooldownUntil = null;
|
|
1063
1112
|
account.providerRateLimitStreak = 0;
|
|
1064
1113
|
account.lastStatus = outcome.status || account.lastStatus;
|
|
1065
1114
|
account.lastResponseMs = Date.now() - lease.startedAt;
|
|
@@ -1078,7 +1127,12 @@ export class AccountManager {
|
|
|
1078
1127
|
|
|
1079
1128
|
if (outcome.error || outcome.status) {
|
|
1080
1129
|
account.failedRequests++;
|
|
1081
|
-
|
|
1130
|
+
// scope:'request' (cancel-adjacent / first-byte timeout / link blip): the fault
|
|
1131
|
+
// belongs to THIS request, not the account. It must not bump the streak that
|
|
1132
|
+
// the hold-release gate (_affinityRelease `failures-N`) reads — one such fault
|
|
1133
|
+
// used to evict every conversation held on the account (4 in 1s, 2026-11-09
|
|
1134
|
+
// 08:13Z). Account-level faults (429 / caps / auth) keep the default 'account'.
|
|
1135
|
+
if (outcome.scope !== 'request') account.consecutiveFailures++;
|
|
1082
1136
|
account.lastStatus = outcome.status || account.lastStatus;
|
|
1083
1137
|
account.lastResponseMs = Date.now() - lease.startedAt;
|
|
1084
1138
|
this._recordLoadEvent(account, lease, outcome);
|
|
@@ -1145,6 +1199,21 @@ export class AccountManager {
|
|
|
1145
1199
|
return exhausted ? { resetAt: e.resetAt || null, family: fam } : null;
|
|
1146
1200
|
}
|
|
1147
1201
|
|
|
1202
|
+
/** Review r2 A: inside a request-scoped soft cooldown (new placements only). */
|
|
1203
|
+
_softCooling(account, now = Date.now()) {
|
|
1204
|
+
return Boolean(account.softCooldownUntil && now < account.softCooldownUntil);
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
/** Does `account` already hold this request's conversation (so a soft cooldown must
|
|
1208
|
+
* not move it)? The one-shot retry-same hint, or the last account for the affinity
|
|
1209
|
+
* key (which falls back to the sessionKey, so a sticky binding is covered). A
|
|
1210
|
+
* return-home target needs no clause: homeCand is captured before the skip. */
|
|
1211
|
+
_holdsRequest(account, requestInfo = {}) {
|
|
1212
|
+
if (requestInfo.retrySameAccountName === account.name) return true;
|
|
1213
|
+
const key = this._affinityKey(requestInfo);
|
|
1214
|
+
return Boolean(key && this.conversationLastAccount.get(key) === account.name);
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1148
1217
|
_isAvailable(account, options = {}) {
|
|
1149
1218
|
if (!account) return false;
|
|
1150
1219
|
if (!account.enabled) return false;
|
|
@@ -2325,10 +2394,13 @@ export class AccountManager {
|
|
|
2325
2394
|
];
|
|
2326
2395
|
|
|
2327
2396
|
const bestUnlockMap = new Map();
|
|
2397
|
+
const home = this._returnHome(requestInfo, now);
|
|
2328
2398
|
for (const weeklyOptions of weeklyPasses) {
|
|
2329
2399
|
best = null;
|
|
2330
2400
|
bestScore = Infinity;
|
|
2331
2401
|
bestPriority = Infinity;
|
|
2402
|
+
let homeCand = null; // the home account as a candidate THIS pass, scored as holder
|
|
2403
|
+
let coolBest = null; // best soft-cooled account (review r2 A): used only if nothing else fits
|
|
2332
2404
|
|
|
2333
2405
|
for (let i = 0; i < this.accounts.length; i++) {
|
|
2334
2406
|
const idx = (this.nextIndex + i) % this.accounts.length;
|
|
@@ -2344,6 +2416,20 @@ export class AccountManager {
|
|
|
2344
2416
|
|
|
2345
2417
|
const priority = this._effectivePriority(account, requestInfo, now);
|
|
2346
2418
|
const score = this._scoreAccount(account, requestInfo, scoringCtx, critUnlock);
|
|
2419
|
+
if (home && account.name === home.name) {
|
|
2420
|
+
homeCand = { account, priority, critUnlock,
|
|
2421
|
+
score: this._scoreAccount(account, requestInfo, scoringCtx, critUnlock, home.name) };
|
|
2422
|
+
}
|
|
2423
|
+
// REQUEST-SCOPED SOFT COOLDOWN (review r2 A): after a request-scoped first-byte
|
|
2424
|
+
// timeout the account is deprioritised for NEW placements only — a cold pick
|
|
2425
|
+
// takes it last. A conversation it HOLDS (_holdsRequest) is not touched:
|
|
2426
|
+
// evicting those is exactly what a request-scoped fault must never do.
|
|
2427
|
+
if (this._softCooling(account, now) && !this._holdsRequest(account, requestInfo)) {
|
|
2428
|
+
if (!coolBest || priority < coolBest.priority || (priority === coolBest.priority && score < coolBest.score)) {
|
|
2429
|
+
coolBest = { account, priority, score, critUnlock };
|
|
2430
|
+
}
|
|
2431
|
+
continue;
|
|
2432
|
+
}
|
|
2347
2433
|
if (priority < bestPriority || (priority === bestPriority && score < bestScore)) {
|
|
2348
2434
|
bestPriority = priority;
|
|
2349
2435
|
bestScore = score;
|
|
@@ -2352,6 +2438,47 @@ export class AccountManager {
|
|
|
2352
2438
|
}
|
|
2353
2439
|
}
|
|
2354
2440
|
|
|
2441
|
+
// Deprioritised, never benched: with no other candidate, the soft-cooled one serves.
|
|
2442
|
+
if (!best && coolBest) {
|
|
2443
|
+
best = coolBest.account;
|
|
2444
|
+
bestPriority = coolBest.priority;
|
|
2445
|
+
bestScore = coolBest.score;
|
|
2446
|
+
if (coolBest.critUnlock) bestUnlockMap.set(best.name, coolBest.critUnlock); else bestUnlockMap.delete(best.name);
|
|
2447
|
+
}
|
|
2448
|
+
|
|
2449
|
+
// RETURN HOME: only once this pass already picked a Claude account — a provider
|
|
2450
|
+
// pick is never touched, so this never pulls a conversation off GLM sooner. Home
|
|
2451
|
+
// wins when it is a candidate here, in the same priority tier, and its score AS
|
|
2452
|
+
// HOLDER (release gates, zeroed steering, anti-dogpile cap) beats the pick.
|
|
2453
|
+
let homeNote = null;
|
|
2454
|
+
let homeSwitched = false;
|
|
2455
|
+
const pickedBeforeHome = best;
|
|
2456
|
+
if (best && home && isClaudeAccount(best)) {
|
|
2457
|
+
if (best.name === home.name) {
|
|
2458
|
+
homeNote = `home=${home.name}:returned`;
|
|
2459
|
+
} else if (home.stale) {
|
|
2460
|
+
homeNote = `home=${home.name}:skipped(stale-${Math.round(home.ageMs / 60_000)}m)`;
|
|
2461
|
+
} else if (homeCand && homeCand.priority <= bestPriority && homeCand.score < bestScore) {
|
|
2462
|
+
// `<=`, not `===`: a soft-cooled HOME is skipped from the best-pick above but
|
|
2463
|
+
// still recorded in homeCand — home at equal-or-better priority wins even then
|
|
2464
|
+
// (review r3 R3-5: soft-cooldown must never override return-home).
|
|
2465
|
+
best = homeCand.account;
|
|
2466
|
+
homeSwitched = true;
|
|
2467
|
+
if (homeCand.critUnlock) bestUnlockMap.set(best.name, homeCand.critUnlock); else bestUnlockMap.delete(best.name);
|
|
2468
|
+
homeNote = `home=${home.name}:returned`;
|
|
2469
|
+
} else if (homeCand && this._softCooling(homeCand.account, now)) {
|
|
2470
|
+
// R3-5: a soft-cooled home that did NOT win as holder: name the soft cooldown
|
|
2471
|
+
// (it kept home out of the best-pick) instead of an `outscored:` line computed
|
|
2472
|
+
// as if home had competed there.
|
|
2473
|
+
homeNote = `home=${home.name}:skipped(soft-cooled;${homeCand.priority > bestPriority ? 'priority-tier' : `holder-score ${homeCand.score.toFixed(1)}>=pick ${bestScore.toFixed(1)}`})`;
|
|
2474
|
+
} else {
|
|
2475
|
+
let why = 'unknown';
|
|
2476
|
+
try { why = this._moveReason(home.name, best, requestInfo, excludedIndexes, scoringCtx, weeklyOptions, bestUnlockMap.get(best.name) || null, now, true); } catch (e) { why = `error:${e.message}`; }
|
|
2477
|
+
homeNote = `home=${home.name}:skipped(${why})`;
|
|
2478
|
+
}
|
|
2479
|
+
if (best.name === home.name) requestInfo.returnedHome = { name: home.name, switched: homeSwitched };
|
|
2480
|
+
}
|
|
2481
|
+
|
|
2355
2482
|
if (best) {
|
|
2356
2483
|
const switched = best.index !== this.currentIndex;
|
|
2357
2484
|
this.currentIndex = best.index;
|
|
@@ -2380,9 +2507,11 @@ export class AccountManager {
|
|
|
2380
2507
|
const ckey = this._affinityKey(requestInfo);
|
|
2381
2508
|
const lastName = ckey ? this.conversationLastAccount.get(ckey) : null;
|
|
2382
2509
|
if (lastName && lastName !== best.name) {
|
|
2510
|
+
// why= explains why the conversation LEFT its last account — measured against
|
|
2511
|
+
// the loop's own pick, before any return-home switch.
|
|
2383
2512
|
let why = 'unknown';
|
|
2384
|
-
try { why = this._moveReason(lastName,
|
|
2385
|
-
console.log(`[Maxpool] conversation moved: ${lastName} → ${best.name} (${requestInfo.bodyBytes ?? '?'}B) why=${why}`);
|
|
2513
|
+
try { why = this._moveReason(lastName, pickedBeforeHome, requestInfo, excludedIndexes, scoringCtx, weeklyOptions, bestUnlockMap.get(pickedBeforeHome.name) || null, now); } catch (e) { why = `error:${e.message}`; }
|
|
2514
|
+
console.log(`[Maxpool] conversation moved: ${lastName} → ${best.name} (${requestInfo.bodyBytes ?? '?'}B) why=${why}${homeNote ? ` ${homeNote}` : ''}`);
|
|
2386
2515
|
}
|
|
2387
2516
|
return best;
|
|
2388
2517
|
}
|
|
@@ -3090,10 +3219,10 @@ export class AccountManager {
|
|
|
3090
3219
|
return this._holdsUsableReset(account, now) ? 1 - Math.min(d, 0.9) : 1;
|
|
3091
3220
|
}
|
|
3092
3221
|
|
|
3093
|
-
_scoreTerms(account, requestInfo = {}, ctx = null, criticalUnlock = null) {
|
|
3222
|
+
_scoreTerms(account, requestInfo = {}, ctx = null, criticalUnlock = null, holderName = undefined) {
|
|
3094
3223
|
const now = ctx?.now ?? Date.now();
|
|
3095
3224
|
// CACHE AFFINITY — see _cacheAffinity.
|
|
3096
|
-
const affinity = this._cacheAffinity(account, requestInfo, criticalUnlock);
|
|
3225
|
+
const affinity = this._cacheAffinity(account, requestInfo, criticalUnlock, holderName);
|
|
3097
3226
|
const reqWeight = Math.max(1, requestInfo.weight || 1);
|
|
3098
3227
|
// POOL LINK scores as the cloud's WORST-case load (owner 2026-10-29: "the team
|
|
3099
3228
|
// pool will respect the accounts within it... included in the balancing act").
|
|
@@ -3212,8 +3341,8 @@ export class AccountManager {
|
|
|
3212
3341
|
}
|
|
3213
3342
|
|
|
3214
3343
|
/** Routing cost — lower is preferred. Sum of the named terms minus the affinity bonus. */
|
|
3215
|
-
_scoreAccount(account, requestInfo = {}, ctx = null, criticalUnlock = null) {
|
|
3216
|
-
const t = this._scoreTerms(account, requestInfo, ctx, criticalUnlock);
|
|
3344
|
+
_scoreAccount(account, requestInfo = {}, ctx = null, criticalUnlock = null, holderName = undefined) {
|
|
3345
|
+
const t = this._scoreTerms(account, requestInfo, ctx, criticalUnlock, holderName);
|
|
3217
3346
|
return Math.max(0, t.concurrency + t.capPenalty + t.paceCost + t.utilizationCost + t.scopedPace + t.spread + t.ramp + t.reserveCost + t.criticalCost + t.failurePenalty - t.affinity);
|
|
3218
3347
|
}
|
|
3219
3348
|
|
|
@@ -3238,10 +3367,29 @@ export class AccountManager {
|
|
|
3238
3367
|
_noteConversationAccount(requestInfo, account) {
|
|
3239
3368
|
const key = this._affinityKey(requestInfo);
|
|
3240
3369
|
if (!key) return;
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3370
|
+
lruSet(this.conversationLastAccount, key, account.name);
|
|
3371
|
+
if (isClaudeAccount(account)) {
|
|
3372
|
+
lruSet(this.conversationLastClaudeAccount, key, { name: account.name, at: requestInfo.now ?? Date.now() });
|
|
3373
|
+
}
|
|
3374
|
+
}
|
|
3375
|
+
|
|
3376
|
+
/**
|
|
3377
|
+
* RETURN HOME (owner 2026-11-09): the Claude account a conversation should go back to
|
|
3378
|
+
* after a provider detour, or null. A detour = the last account differs from the last
|
|
3379
|
+
* CLAUDE account (the Claude memory only ever holds Claude names). That includes the
|
|
3380
|
+
* case where the last-account entry was evicted but the Claude entry survived: going
|
|
3381
|
+
* home then restores the evicted hold, which is harmless. `stale` when the stored copy
|
|
3382
|
+
* has outlived cacheAffinityReturnWindowMs — going home then buys nothing.
|
|
3383
|
+
*/
|
|
3384
|
+
_returnHome(requestInfo, now = Date.now()) {
|
|
3385
|
+
const key = this._affinityKey(requestInfo);
|
|
3386
|
+
if (!key) return null;
|
|
3387
|
+
const home = this.conversationLastClaudeAccount.get(key);
|
|
3388
|
+
if (!home || home.name === this.conversationLastAccount.get(key)) return null;
|
|
3389
|
+
const ageMs = now - home.at;
|
|
3390
|
+
let windowMs = Number(this.scheduler.cacheAffinityReturnWindowMs);
|
|
3391
|
+
if (!Number.isFinite(windowMs) || windowMs < 0) windowMs = DEFAULT_SCHEDULER.cacheAffinityReturnWindowMs;
|
|
3392
|
+
return { name: home.name, stale: ageMs > windowMs, ageMs };
|
|
3245
3393
|
}
|
|
3246
3394
|
|
|
3247
3395
|
/** Conversation identity: terminal + prefix hash when the server computed one;
|
|
@@ -3320,13 +3468,25 @@ export class AccountManager {
|
|
|
3320
3468
|
return null;
|
|
3321
3469
|
}
|
|
3322
3470
|
|
|
3323
|
-
|
|
3471
|
+
/** `holderName` scores `account` as if it held the conversation (return-home after a
|
|
3472
|
+
* provider detour); omitted = the conversation's real last account. */
|
|
3473
|
+
_cacheAffinity(account, requestInfo = {}, criticalUnlock = null, holderName = undefined) {
|
|
3324
3474
|
const base = Number(this.scheduler.cacheAffinityBonus) || 0;
|
|
3325
3475
|
// Sticky mode has its own binding; its score loop runs only once the session has
|
|
3326
3476
|
// decided to LEAVE a hot account — a discount there would pull it straight back.
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3477
|
+
// D3 retry-same (one acquire, set by server.js after a first-byte timeout on a big
|
|
3478
|
+
// prompt): the account that just timed out holds this prompt's warm copy, so it is
|
|
3479
|
+
// scored AS THE HOLDER — the same bonus, release gates, priority tiers, weekly
|
|
3480
|
+
// passes, fairness and anti-dogpile cap as any held conversation (red-team r1: no
|
|
3481
|
+
// bypass). Only the cooldown is skipped, upstream (server.js never cools it). It
|
|
3482
|
+
// applies in sticky mode too: pulling the turn back to that account IS the intent,
|
|
3483
|
+
// for exactly one acquire.
|
|
3484
|
+
const retrySame = requestInfo.retrySameAccountName != null && requestInfo.retrySameAccountName === account.name;
|
|
3485
|
+
if (base <= 0 || (this.scheduler.routingMode === 'sticky' && !retrySame)) return 0;
|
|
3486
|
+
if (!retrySame) {
|
|
3487
|
+
const key = this._affinityKey(requestInfo);
|
|
3488
|
+
if (!key || (holderName ?? this.conversationLastAccount.get(key)) !== account.name) return 0;
|
|
3489
|
+
}
|
|
3330
3490
|
// Scale with the CONVERSATION'S SIZE too: losing the cache costs the prefix a
|
|
3331
3491
|
// hop re-writes, and requestInfo.bodyBytes ≈ prefix + this turn. Checkpoint
|
|
3332
3492
|
// 2026-11-08 11:07Z: a flat 400 cut full re-writes only 18.9%→14.8% of turns —
|
|
@@ -3363,7 +3523,9 @@ export class AccountManager {
|
|
|
3363
3523
|
* names the reason, incl. availability causes); `outscored:` = the hold was live and
|
|
3364
3524
|
* the sibling still won, with the dominant score terms that beat the bonus.
|
|
3365
3525
|
*/
|
|
3366
|
-
_moveReason(lastName, best, requestInfo, excludedIndexes, scoringCtx, weeklyOptions, bestUnlock, now) {
|
|
3526
|
+
_moveReason(lastName, best, requestInfo, excludedIndexes, scoringCtx, weeklyOptions, bestUnlock, now, asHolder = false) {
|
|
3527
|
+
// asHolder: explain why `lastName` lost while scored AS the holder (return-home).
|
|
3528
|
+
const holder = asHolder ? lastName : undefined;
|
|
3367
3529
|
const last = this.accounts.find(a => a.name === lastName);
|
|
3368
3530
|
if (!last) return 'released:account-gone';
|
|
3369
3531
|
if (excludedIndexes.has(last.index)) return 'released:excluded';
|
|
@@ -3374,8 +3536,8 @@ export class AccountManager {
|
|
|
3374
3536
|
const gate = this._affinityRelease(last, unlock);
|
|
3375
3537
|
if (gate) return `released:${gate}`;
|
|
3376
3538
|
if (this._effectivePriority(last, requestInfo, now) > this._effectivePriority(best, requestInfo, now)) return 'released:priority-tier';
|
|
3377
|
-
if (this._cacheAffinity(last, requestInfo, unlock) <= 0) return 'released:no-hold';
|
|
3378
|
-
const lt = this._scoreTerms(last, requestInfo, scoringCtx, unlock);
|
|
3539
|
+
if (this._cacheAffinity(last, requestInfo, unlock, holder) <= 0) return 'released:no-hold';
|
|
3540
|
+
const lt = this._scoreTerms(last, requestInfo, scoringCtx, unlock, holder);
|
|
3379
3541
|
const bt = this._scoreTerms(best, requestInfo, scoringCtx, bestUnlock);
|
|
3380
3542
|
const sum = t => t.concurrency + t.capPenalty + t.paceCost + t.utilizationCost + t.scopedPace + t.spread + t.ramp + t.reserveCost + t.criticalCost + t.failurePenalty - t.affinity;
|
|
3381
3543
|
const delta = sum(lt) - sum(bt);
|
|
@@ -4618,10 +4780,35 @@ export class AccountManager {
|
|
|
4618
4780
|
console.log(`[Maxpool] Account "${account.name}" disabled after HTTP ${status} (${reason})`);
|
|
4619
4781
|
}
|
|
4620
4782
|
|
|
4621
|
-
markTransientFailure(accountIndex, reason = 'transient_error', { network = false } = {}) {
|
|
4783
|
+
markTransientFailure(accountIndex, reason = 'transient_error', { network = false, scope = 'account' } = {}) {
|
|
4622
4784
|
const account = this.accounts[accountIndex];
|
|
4623
4785
|
if (!account) return;
|
|
4624
4786
|
|
|
4787
|
+
// REQUEST-SCOPED (review r2 A): a first-byte/idle timeout on ONE request says nothing
|
|
4788
|
+
// about the account's other conversations. Deprioritise it for new placements only
|
|
4789
|
+
// (softCooldownUntil — held conversations keep it), and count consecutive request-
|
|
4790
|
+
// scoped faults. REQUEST_FAULT_ESCALATE in a row (no success between) is no longer
|
|
4791
|
+
// "one slow request": escalate to the account-scoped shape below (real cooldown +
|
|
4792
|
+
// failure streak → held conversations released), logged once per run.
|
|
4793
|
+
if (network && scope === 'request') {
|
|
4794
|
+
account.failedRequests++;
|
|
4795
|
+
account.lastError = reason;
|
|
4796
|
+
account.lastErrorAt = Date.now();
|
|
4797
|
+
account.requestFaultStreak = (account.requestFaultStreak || 0) + 1;
|
|
4798
|
+
if (account.requestFaultStreak < REQUEST_FAULT_ESCALATE) {
|
|
4799
|
+
account.softCooldownUntil = Date.now() + this.scheduler.networkCooldownMs;
|
|
4800
|
+
console.log(`[Maxpool] Account "${account.name}" deprioritised for new placements for ${Math.ceil(this.scheduler.networkCooldownMs / 1000)}s after ${reason} (request-scoped ${account.requestFaultStreak}/${REQUEST_FAULT_ESCALATE}; held conversations stay)`);
|
|
4801
|
+
return;
|
|
4802
|
+
}
|
|
4803
|
+
account.consecutiveFailures++;
|
|
4804
|
+
account.cooldownUntil = Date.now() + this.scheduler.networkCooldownMs;
|
|
4805
|
+
if (!account.requestFaultEscalated) {
|
|
4806
|
+
account.requestFaultEscalated = true;
|
|
4807
|
+
console.log(`[Maxpool] Account "${account.name}": ${account.requestFaultStreak} request-scoped faults in a row (${reason}) — treating as an ACCOUNT fault: cooling down ${Math.ceil(this.scheduler.networkCooldownMs / 1000)}s, held conversations released`);
|
|
4808
|
+
}
|
|
4809
|
+
return;
|
|
4810
|
+
}
|
|
4811
|
+
|
|
4625
4812
|
if (network) {
|
|
4626
4813
|
// A network-class failure (lost connectivity / token-refresh `fetch failed`)
|
|
4627
4814
|
// is a FLEET-WIDE condition, not this account's fault — every account fails at
|