maxpool 1.24.95 → 1.24.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -0
- package/package.json +1 -1
- package/src/account-manager.js +456 -29
- package/src/index.js +57 -4
- package/src/prober.js +11 -2
- package/src/server.js +266 -25
- package/src/tui.js +232 -61
- package/src/updater.js +7 -2
- package/src/usage-label.js +50 -0
package/src/account-manager.js
CHANGED
|
@@ -225,6 +225,11 @@ const DEFAULT_SCHEDULER = {
|
|
|
225
225
|
// is alive: Claude Code writes 1-hour copies (measured 2026-11-09: 272 of 272 cache
|
|
226
226
|
// writes `ephemeral_1h`; 55 of 64 detours came back within the hour, median 533 s).
|
|
227
227
|
cacheAffinityReturnWindowMs: 3_600_000,
|
|
228
|
+
// STICKY BUSY WAIT (owner 2026-11-15): how long, in total, a warm Claude conversation
|
|
229
|
+
// waits for its own cloud account to stop being busy before it moves (a move re-writes
|
|
230
|
+
// its whole prompt cache). Edited live from the TUI routing screen (`b`); env
|
|
231
|
+
// MAXPOOL_STICKY_BUSY_BUDGET_MS overrides it when set. 0 = never wait.
|
|
232
|
+
stickyBusyBudgetMs: 60_000,
|
|
228
233
|
concurrencyWeight: 2, // multiplies in-flight load (activeWeight+reqWeight) — dominant
|
|
229
234
|
perAccountConcurrencyTarget: 3, // D: soft per-account in-flight target; past it, capPenalty bites
|
|
230
235
|
capPenaltyWeight: 10, // steep penalty per unit of in-flight depth past D (throttle safety floor)
|
|
@@ -342,6 +347,16 @@ const DEFAULT_SCHEDULER = {
|
|
|
342
347
|
providers: {},
|
|
343
348
|
};
|
|
344
349
|
const LOAD_EVENT_MAX_AGE_MS = 60 * 60 * 1000;
|
|
350
|
+
// Absolute weekly headroom TIE-BREAK for new placements (2026-11-15): at equal load a
|
|
351
|
+
// fresh-weekly account beats a 60%-weekly one. ~0.75 points at 100% used vs ~70 per
|
|
352
|
+
// in-flight unit, so it only ever decides near-ties; held conversations are unaffected
|
|
353
|
+
// (their steering terms are zeroed).
|
|
354
|
+
const ABS_UTIL_WEIGHT = 0.25;
|
|
355
|
+
// Pinned-busy back-off bounds for a cloud row (markPinBusy).
|
|
356
|
+
const PIN_BUSY_MIN_MS = 1000;
|
|
357
|
+
const PIN_BUSY_MAX_MS = 5000;
|
|
358
|
+
// Edge-pool client outcomes ring (noteClientOutcome).
|
|
359
|
+
const CLIENT_OUTCOME_MAX_AGE_MS = 60 * 60 * 1000;
|
|
345
360
|
// Consecutive failed recovery probes before we stop trusting the SHARED breaker and
|
|
346
361
|
// fall back to per-account handling. Bounds any future "poisoned probe" from becoming
|
|
347
362
|
// an indefinite fleet-wide outage (2026-07-25 incident).
|
|
@@ -381,6 +396,67 @@ const PERSISTED_QUOTA_FIELDS = [
|
|
|
381
396
|
'providerQuotaSource', 'lastProbeOkAt', 'noPlan',
|
|
382
397
|
];
|
|
383
398
|
|
|
399
|
+
/** The sticky busy-wait budget actually in force, and where it comes from — ONE resolver
|
|
400
|
+
* for the server (per request, live) and the TUI setting (review 2026-11-16, 5c: the
|
|
401
|
+
* screen used to show the saved setting while an env override silently won). */
|
|
402
|
+
export function resolveStickyBusyBudget(scheduler, env = process.env) {
|
|
403
|
+
const raw = env.MAXPOOL_STICKY_BUSY_BUDGET_MS;
|
|
404
|
+
if (raw != null && raw !== '' && Number.isFinite(Number(raw)) && Number(raw) >= 0) {
|
|
405
|
+
return { ms: Number(raw), fromEnv: true };
|
|
406
|
+
}
|
|
407
|
+
const v = Number(scheduler?.stickyBusyBudgetMs);
|
|
408
|
+
return { ms: Number.isFinite(v) && v >= 0 ? v : 60_000, fromEnv: false };
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/** Figures only the EDGE pool (the one the client talks to) can know for a request it
|
|
412
|
+
* forwarded to a cloud row — see _mirrorEfficiencyAcc. */
|
|
413
|
+
const EDGE_EFFICIENCY_FIELDS = ['bounced', 'refused', 'waited', 'waitMs', 'waitThenMoved', 'moved', 'newConversation'];
|
|
414
|
+
|
|
415
|
+
const efficiencyWindowMs = (windowKey) => (windowKey === 'last1h' ? 60 * 60_000 : 15 * 60_000);
|
|
416
|
+
|
|
417
|
+
function emptyEfficiency() {
|
|
418
|
+
return { served: 0, in: 0, out: 0, cr: 0, cc: 0, moved: 0,
|
|
419
|
+
movedByCause: { busy: 0, wall: 0, balance: 0, link: 0, other: 0 },
|
|
420
|
+
newConversation: 0, bounced: 0, refused: 0, waited: 0, waitMs: 0, waitThenMoved: 0, e2e: [] };
|
|
421
|
+
}
|
|
422
|
+
/** Add one exported/summarized efficiency block into an accumulator (missing fields = 0). */
|
|
423
|
+
function addEfficiency(acc, e) {
|
|
424
|
+
for (const k of ['served', 'in', 'out', 'cr', 'cc', 'moved', 'newConversation', 'bounced', 'refused', 'waited', 'waitMs', 'waitThenMoved']) {
|
|
425
|
+
acc[k] += Number(e[k]) || 0;
|
|
426
|
+
}
|
|
427
|
+
for (const c of Object.keys(acc.movedByCause)) acc.movedByCause[c] += Number(e.movedByCause?.[c]) || 0;
|
|
428
|
+
// Latency samples exist for this pool's own rows only (percentiles do not add).
|
|
429
|
+
if (Array.isArray(e.e2e)) acc.e2e.push(...e.e2e);
|
|
430
|
+
}
|
|
431
|
+
const pct = (arr, p) => {
|
|
432
|
+
if (!arr.length) return null;
|
|
433
|
+
const s = [...arr].sort((x, y) => x - y);
|
|
434
|
+
return s[Math.min(s.length - 1, Math.floor(p * s.length))];
|
|
435
|
+
};
|
|
436
|
+
/** Derived figures: hit = cr/(cr+cc+in) (GLM has no cc → r/(r+in)); missTokens = in+cc. */
|
|
437
|
+
function finishEfficiency(acc, provider) {
|
|
438
|
+
const prompt = acc.cr + acc.cc + acc.in;
|
|
439
|
+
const { e2e, ...sums } = acc;
|
|
440
|
+
return {
|
|
441
|
+
...sums,
|
|
442
|
+
provider,
|
|
443
|
+
hit: prompt > 0 ? acc.cr / prompt : null,
|
|
444
|
+
missTokens: acc.in + acc.cc,
|
|
445
|
+
promptTokens: prompt,
|
|
446
|
+
lowVolume: acc.served < 20 || prompt < 100_000,
|
|
447
|
+
e2eP50: pct(e2e, 0.5),
|
|
448
|
+
e2eP95: pct(e2e, 0.95),
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** Combine efficiency blocks (summaries or exports) into one, with derived figures —
|
|
453
|
+
* `maxpool status` uses it to add linked pools' exported sums to the local ones. */
|
|
454
|
+
export function combineEfficiency(blocks, provider) {
|
|
455
|
+
const acc = emptyEfficiency();
|
|
456
|
+
for (const b of blocks) if (b) addEfficiency(acc, b);
|
|
457
|
+
return finishEfficiency(acc, provider);
|
|
458
|
+
}
|
|
459
|
+
|
|
384
460
|
const CONVERSATION_AFFINITY_MAX = 5000;
|
|
385
461
|
/** Bounded LRU insert: re-insert = most recent; the oldest key drops past the cap. */
|
|
386
462
|
function lruSet(map, key, value) {
|
|
@@ -698,6 +774,11 @@ export class AccountManager {
|
|
|
698
774
|
// Bounded LRU — oldest conversations drop first; a dropped entry only means the
|
|
699
775
|
// next turn is balanced fresh.
|
|
700
776
|
this.conversationLastAccount = new Map();
|
|
777
|
+
// When that account last SERVED the conversation (first real upstream byte) —
|
|
778
|
+
// holdsWarmConversation reads it. Same LRU bound, in-memory like the map above.
|
|
779
|
+
this.conversationLastServedAt = new Map();
|
|
780
|
+
// Final client outcomes of POST /v1/messages at THIS (edge) pool, 1 h ring.
|
|
781
|
+
this.clientOutcomes = [];
|
|
701
782
|
// _homeKey (affinityKey, else sessionKey) → { name, at, detoured } of the last CLAUDE
|
|
702
783
|
// account only (a GLM/Kimi turn never overwrites the name; it sets `detoured`), so a
|
|
703
784
|
// conversation returning from a provider detour can go home.
|
|
@@ -952,6 +1033,15 @@ export class AccountManager {
|
|
|
952
1033
|
return weeklyPasses.some(options => this.accounts.some(account => {
|
|
953
1034
|
if (excludedIndexes.has(account.index)) return false;
|
|
954
1035
|
if (!this._matchesRequest(account, profile, requestInfo)) return false;
|
|
1036
|
+
// A row that JUST refused busy (pinBusyUntil, clamped 1-5 s) is no route for a NEW
|
|
1037
|
+
// placement — else the queue pacer re-tries an all-busy pool in a sub-second spin.
|
|
1038
|
+
// Its holder may still use it, unless this request already gave up waiting on it
|
|
1039
|
+
// (_busyExcluded). The DEPTH half of _pinBusyForNew (30 s-poll snapshot, one slot
|
|
1040
|
+
// short of the limit) is deliberately NOT consulted here: the cloud still accepts
|
|
1041
|
+
// at limit-1, so that row is a route — depth belongs to placement RANKING only
|
|
1042
|
+
// (_selectNext's coolBest), matching what the acquire path would actually do.
|
|
1043
|
+
if (this._recentlyRefusedBusy(account, now)
|
|
1044
|
+
&& (!this._holdsRequest(account, requestInfo) || requestInfo._busyExcluded?.has(account.name))) return false;
|
|
955
1045
|
return this._isAvailable(account, { ...options, model: requestInfo.model, now });
|
|
956
1046
|
}));
|
|
957
1047
|
}
|
|
@@ -1001,7 +1091,7 @@ export class AccountManager {
|
|
|
1001
1091
|
const migratedFromName = (prevCurrentName && account.name !== prevCurrentName) ? prevCurrentName : null;
|
|
1002
1092
|
const returnedHome = requestInfo.returnedHome?.name === account.name;
|
|
1003
1093
|
const returnedHomeSwitched = returnedHome && requestInfo.returnedHome.switched === true;
|
|
1004
|
-
return { account, weight, startedAt: Date.now(), upstreamThrottleProbe, migratedFromName, returnedHome, returnedHomeSwitched };
|
|
1094
|
+
return { account, weight, startedAt: Date.now(), upstreamThrottleProbe, migratedFromName, returnedHome, returnedHomeSwitched, requestInfo };
|
|
1005
1095
|
}
|
|
1006
1096
|
|
|
1007
1097
|
/** Fail-safe: snap a session's binding back to the pre-migration issuing account
|
|
@@ -1160,6 +1250,13 @@ export class AccountManager {
|
|
|
1160
1250
|
return;
|
|
1161
1251
|
}
|
|
1162
1252
|
|
|
1253
|
+
// A pinned refusal (busy / unavailable / throttled cloud row) never reached the
|
|
1254
|
+
// client: one BOUNCED load event, no failure counters, no streak, no error text.
|
|
1255
|
+
if (outcome.bounced) {
|
|
1256
|
+
this._recordLoadEvent(account, lease, { ...outcome, success: false, bounced: true });
|
|
1257
|
+
return;
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1163
1260
|
// A per-model weekly cap is NOT an account failure — the account is healthy for
|
|
1164
1261
|
// every other model. Don't poison its failure counters / scoring penalty (mirror
|
|
1165
1262
|
// the network-blip carve-out); the scoped bench is already set in markRateLimited.
|
|
@@ -1183,7 +1280,14 @@ export class AccountManager {
|
|
|
1183
1280
|
account.lastError = null;
|
|
1184
1281
|
account.lastErrorAt = null;
|
|
1185
1282
|
}
|
|
1186
|
-
|
|
1283
|
+
// Backstop for callers that never saw a first byte (unit callers): a 2xx served
|
|
1284
|
+
// the conversation. Idempotent — the first-byte note already ran on the real paths.
|
|
1285
|
+
if (lease.requestInfo && !(outcome.status >= 300)) this.noteConversationServed(lease.requestInfo, account);
|
|
1286
|
+
// Efficiency fields derived from the served request; a field the caller passed
|
|
1287
|
+
// explicitly on the outcome wins.
|
|
1288
|
+
const tel = this._servedTelemetry(lease.requestInfo, account);
|
|
1289
|
+
for (const k of Object.keys(tel)) if (outcome[k] != null) tel[k] = outcome[k];
|
|
1290
|
+
this._recordLoadEvent(account, lease, { ...outcome, ...tel, success: true });
|
|
1187
1291
|
account.lastSuccessAt = Date.now();
|
|
1188
1292
|
return;
|
|
1189
1293
|
}
|
|
@@ -1214,10 +1318,149 @@ export class AccountManager {
|
|
|
1214
1318
|
success: Boolean(outcome.success),
|
|
1215
1319
|
linkFault: Boolean(outcome.linkFault),
|
|
1216
1320
|
status: outcome.status || null,
|
|
1321
|
+
// Efficiency telemetry (2026-11-15) — optional, read by _efficiencySummary only.
|
|
1322
|
+
bounced: Boolean(outcome.bounced),
|
|
1323
|
+
usage: outcome.usage || null,
|
|
1324
|
+
moved: Boolean(outcome.moved),
|
|
1325
|
+
newConversation: Boolean(outcome.newConversation),
|
|
1326
|
+
moveCause: outcome.moveCause || null,
|
|
1327
|
+
busyWaitMs: outcome.busyWaitMs || 0,
|
|
1328
|
+
e2eFirstByteMs: Number.isFinite(outcome.e2eFirstByteMs) ? outcome.e2eFirstByteMs : null,
|
|
1329
|
+
refusal: outcome.refusal || null,
|
|
1330
|
+
// _efficiencyAcc skips these: a count_tokens call is not a conversation turn.
|
|
1331
|
+
countTokens: Boolean(outcome.countTokens),
|
|
1217
1332
|
});
|
|
1218
1333
|
this._pruneLoadEvents(account, now);
|
|
1219
1334
|
}
|
|
1220
1335
|
|
|
1336
|
+
/** Efficiency fields of a SERVED event, read off the request that was served. */
|
|
1337
|
+
_servedTelemetry(ri, account) {
|
|
1338
|
+
if (!ri) return {};
|
|
1339
|
+
// A PINNED request was routed by the linked (edge) pool: the edge books its move,
|
|
1340
|
+
// cause, wait and client-felt latency on its own mirror row. Booking them here too
|
|
1341
|
+
// would double-count the move (as 'balance' — this pool never knew the cause) and
|
|
1342
|
+
// add a latency sample that starts AFTER the edge's wait. Served + tokens stay here.
|
|
1343
|
+
const pinned = ri.pinAccount != null;
|
|
1344
|
+
const moved = !pinned && Boolean(ri._servedMoved) && ri._notedServed === account.name;
|
|
1345
|
+
return {
|
|
1346
|
+
countTokens: Boolean(ri.isCountTokens),
|
|
1347
|
+
usage: ri.isCountTokens ? null : {
|
|
1348
|
+
in: ri._capacityInput || 0, out: ri._capacityOutput || 0,
|
|
1349
|
+
cr: ri._capacityCacheRead || 0, cc: ri._capacityCacheCreation || 0,
|
|
1350
|
+
},
|
|
1351
|
+
moved,
|
|
1352
|
+
moveCause: moved ? (ri._moveCause || (ri._hop > 0 ? 'other' : 'balance')) : null,
|
|
1353
|
+
newConversation: !pinned && Boolean(ri._newConversation) && ri._notedServed === account.name,
|
|
1354
|
+
|
|
1355
|
+
busyWaitMs: ri._busyWaitMs || 0,
|
|
1356
|
+
e2eFirstByteMs: !pinned && ri._servedAt && ri.startedAt ? Math.max(0, ri._servedAt - ri.startedAt) : null,
|
|
1357
|
+
};
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
/** Cache / move / bounce figures for one account over a window (getStatus + TUI only —
|
|
1361
|
+
* never on the scoring path). Additive sums plus derived hit share. */
|
|
1362
|
+
_efficiencySummary(account, windowMs, now = Date.now()) {
|
|
1363
|
+
return finishEfficiency(this._efficiencyAcc(account, windowMs, now), isClaudeAccount(account) ? 'claude' : 'glm');
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
_efficiencyAcc(account, windowMs, now = Date.now()) {
|
|
1367
|
+
this._pruneLoadEvents(account, now);
|
|
1368
|
+
const since = now - windowMs;
|
|
1369
|
+
const out = emptyEfficiency();
|
|
1370
|
+
for (const e of account.loadEvents || []) {
|
|
1371
|
+
if (e.at < since || e.countTokens) continue;
|
|
1372
|
+
if (e.bounced) {
|
|
1373
|
+
out.bounced++;
|
|
1374
|
+
if (e.refusal && e.refusal !== 'busy') out.refused++;
|
|
1375
|
+
continue;
|
|
1376
|
+
}
|
|
1377
|
+
if (!e.success) continue;
|
|
1378
|
+
out.served++;
|
|
1379
|
+
if (e.usage) { out.in += e.usage.in; out.out += e.usage.out; out.cr += e.usage.cr; out.cc += e.usage.cc; }
|
|
1380
|
+
if (e.moved) {
|
|
1381
|
+
out.moved++;
|
|
1382
|
+
const c = Object.hasOwn(out.movedByCause, e.moveCause) ? e.moveCause : 'other';
|
|
1383
|
+
out.movedByCause[c]++;
|
|
1384
|
+
if (e.busyWaitMs > 0) out.waitThenMoved++;
|
|
1385
|
+
}
|
|
1386
|
+
if (e.newConversation) out.newConversation++;
|
|
1387
|
+
if (e.busyWaitMs > 0) { out.waited++; out.waitMs += e.busyWaitMs; }
|
|
1388
|
+
if (e.e2eFirstByteMs != null) out.e2e.push(e.e2eFirstByteMs);
|
|
1389
|
+
}
|
|
1390
|
+
return out;
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
/** One cloud MIRROR row's figures, each field taken from the only pool that can know it
|
|
1394
|
+
* (review 2026-11-16, cluster 1):
|
|
1395
|
+
* - served / tokens / cache hit: the SERVING pool's export (it saw every client of that
|
|
1396
|
+
* account, teammates included);
|
|
1397
|
+
* - bounced / refused / waited / waitMs / waitThenMoved / moved / movedByCause /
|
|
1398
|
+
* newConversation, and first-byte latency: THIS pool's own events on the row. The
|
|
1399
|
+
* cloud books a pin refusal before any lease, never sees the edge's wait or its move
|
|
1400
|
+
* cause, and starts its clock after the edge's wait — and it no longer counts pinned
|
|
1401
|
+
* requests as its own moves (_servedTelemetry), so nothing here is counted twice.
|
|
1402
|
+
* An older cloud with no export: this pool's own events alone (it counts nothing). */
|
|
1403
|
+
_mirrorEfficiencyAcc(account, windowKey, windowMs, now = Date.now()) {
|
|
1404
|
+
const edge = this._efficiencyAcc(account, windowMs, now);
|
|
1405
|
+
const exp = account._remoteLoad?.[windowKey]?.efficiency;
|
|
1406
|
+
if (!exp) return edge;
|
|
1407
|
+
const acc = emptyEfficiency();
|
|
1408
|
+
for (const k of ['served', 'in', 'out', 'cr', 'cc']) acc[k] = Number(exp[k]) || 0;
|
|
1409
|
+
for (const k of EDGE_EFFICIENCY_FIELDS) acc[k] = (Number(exp[k]) || 0) + edge[k];
|
|
1410
|
+
for (const c of Object.keys(acc.movedByCause)) {
|
|
1411
|
+
acc.movedByCause[c] = (Number(exp.movedByCause?.[c]) || 0) + edge.movedByCause[c];
|
|
1412
|
+
}
|
|
1413
|
+
acc.e2e = edge.e2e;
|
|
1414
|
+
return acc;
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
/** The per-row summary the TUI and `maxpool status` show: a mirror row merges the
|
|
1418
|
+
* serving pool's export with this pool's edge-only fields; a local row is its own. */
|
|
1419
|
+
rowEfficiencySummary(account, windowKey = 'last15m', now = Date.now()) {
|
|
1420
|
+
const windowMs = efficiencyWindowMs(windowKey);
|
|
1421
|
+
const acc = account.remote ? this._mirrorEfficiencyAcc(account, windowKey, windowMs, now) : this._efficiencyAcc(account, windowMs, now);
|
|
1422
|
+
return finishEfficiency(acc, isClaudeAccount(account) ? 'claude' : 'glm');
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
/** Overall efficiency, per provider, for the TUI / `maxpool status`: this pool's own
|
|
1426
|
+
* rows plus each cloud mirror row (_mirrorEfficiencyAcc: cache/served from the cloud's
|
|
1427
|
+
* export, bounces/waits/moves/latency from this pool's own events — each counted once).
|
|
1428
|
+
* First-byte percentiles cover this pool's CLIENTS, mirror-served turns included, so
|
|
1429
|
+
* an edge busy wait shows up in p95. */
|
|
1430
|
+
efficiencyOverall(windowKey = 'last15m', now = Date.now(), { mirrors = true } = {}) {
|
|
1431
|
+
const windowMs = efficiencyWindowMs(windowKey);
|
|
1432
|
+
const by = { claude: emptyEfficiency(), glm: emptyEfficiency() };
|
|
1433
|
+
for (const a of this.accounts) {
|
|
1434
|
+
if (a.enabled === false || (a.remote && !mirrors)) continue;
|
|
1435
|
+
const tag = isClaudeAccount(a) ? 'claude' : 'glm';
|
|
1436
|
+
const e = a.remote ? this._mirrorEfficiencyAcc(a, windowKey, windowMs, now) : this._efficiencyAcc(a, windowMs, now);
|
|
1437
|
+
addEfficiency(by[tag], e);
|
|
1438
|
+
}
|
|
1439
|
+
const e2e = [...by.claude.e2e, ...by.glm.e2e];
|
|
1440
|
+
return { claude: finishEfficiency(by.claude, 'claude'), glm: finishEfficiency(by.glm, 'glm'),
|
|
1441
|
+
e2eP50: pct(e2e, 0.5), e2eP95: pct(e2e, 0.95) };
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
/** Final client outcome of one request at THIS pool. Edge-only: a request pinned to
|
|
1445
|
+
* us by a linked pool is that pool's client, never ours (its refusals would read as
|
|
1446
|
+
* our client errors). POST /v1/messages only, count_tokens skipped. */
|
|
1447
|
+
noteClientOutcome(requestInfo, status, now = Date.now()) {
|
|
1448
|
+
if (!requestInfo || requestInfo.pinAccount != null || requestInfo.isCountTokens) return;
|
|
1449
|
+
// The client cancelled (Esc / Ctrl-C) — no error was served to anyone.
|
|
1450
|
+
if (requestInfo._clientCancelled) return;
|
|
1451
|
+
if (requestInfo.method !== 'POST' || !/^\/v1\/messages(\?|$)/.test(requestInfo.path || '')) return;
|
|
1452
|
+
this.clientOutcomes.push({ at: now, status: Number(status) || 0 });
|
|
1453
|
+
const cutoff = now - CLIENT_OUTCOME_MAX_AGE_MS;
|
|
1454
|
+
while (this.clientOutcomes.length && this.clientOutcomes[0].at < cutoff) this.clientOutcomes.shift();
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
clientOutcomeSummary(windowMs = 15 * 60_000, now = Date.now()) {
|
|
1458
|
+
const ev = this.clientOutcomes.filter(e => e.at >= now - windowMs);
|
|
1459
|
+
const poolErrors = ev.filter(e => e.status >= 500 || e.status === 429).length;
|
|
1460
|
+
const requestErrors = ev.filter(e => e.status >= 400 && e.status < 500 && e.status !== 429).length;
|
|
1461
|
+
return { total: ev.length, poolErrors, requestErrors };
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1221
1464
|
_pruneLoadEvents(account, now = Date.now()) {
|
|
1222
1465
|
if (!account.loadEvents?.length) return;
|
|
1223
1466
|
const cutoff = now - LOAD_EVENT_MAX_AGE_MS;
|
|
@@ -1229,7 +1472,11 @@ export class AccountManager {
|
|
|
1229
1472
|
_loadSummary(account, windowMs, now = Date.now()) {
|
|
1230
1473
|
this._pruneLoadEvents(account, now);
|
|
1231
1474
|
const since = now - windowMs;
|
|
1232
|
-
const
|
|
1475
|
+
const all = (account.loadEvents || []).filter(e => e.at >= since);
|
|
1476
|
+
// A BOUNCE (pinned refusal, 2026-11-15) never served anything: out of every
|
|
1477
|
+
// denominator — requests, weight (spread share), avgMs — and out of `failed`.
|
|
1478
|
+
const bounced = all.filter(e => e.bounced).length;
|
|
1479
|
+
const events = all.filter(e => !e.bounced);
|
|
1233
1480
|
const requests = events.length;
|
|
1234
1481
|
// "failed" is what the operator reads as the ACCOUNT misbehaving; a link fault is
|
|
1235
1482
|
// the pool's own tunnel to the team pool and gets its own field (2026-11-12).
|
|
@@ -1244,6 +1491,7 @@ export class AccountManager {
|
|
|
1244
1491
|
requests,
|
|
1245
1492
|
failed,
|
|
1246
1493
|
linkFailed,
|
|
1494
|
+
bounced,
|
|
1247
1495
|
weight,
|
|
1248
1496
|
avgMs: timed.length ? Math.round(durationMs / timed.length) : null,
|
|
1249
1497
|
};
|
|
@@ -2680,7 +2928,7 @@ export class AccountManager {
|
|
|
2680
2928
|
// timeout the account is deprioritised for NEW placements only — a cold pick
|
|
2681
2929
|
// takes it last. A conversation it HOLDS (_holdsRequest) is not touched:
|
|
2682
2930
|
// evicting those is exactly what a request-scoped fault must never do.
|
|
2683
|
-
if (this._softCooling(account, now) && !this._holdsRequest(account, requestInfo)) {
|
|
2931
|
+
if ((this._softCooling(account, now) || this._pinBusyForNew(account, now)) && !this._holdsRequest(account, requestInfo)) {
|
|
2684
2932
|
if (!coolBest || priority < coolBest.priority || (priority === coolBest.priority && score < coolBest.score)) {
|
|
2685
2933
|
coolBest = { account, priority, score, critUnlock };
|
|
2686
2934
|
}
|
|
@@ -2952,23 +3200,103 @@ export class AccountManager {
|
|
|
2952
3200
|
return profiles.includes(profile);
|
|
2953
3201
|
}
|
|
2954
3202
|
|
|
3203
|
+
/** The in-flight depth at which a PINNED request is refused busy (the cloud's only
|
|
3204
|
+
* hard per-account cap for pinned traffic). One helper for the gate and the export. */
|
|
3205
|
+
_pinBusyLimit(_account) {
|
|
3206
|
+
return 2 * this.scheduler.perAccountConcurrencyTarget;
|
|
3207
|
+
}
|
|
3208
|
+
|
|
3209
|
+
/** A cloud row NEW placements should skip right now: it just refused busy
|
|
3210
|
+
* (pinBusyUntil), or its last known depth is one short of the cloud's pin limit.
|
|
3211
|
+
* Placement-scoped only — never in _isAvailable, so a holder still reaches it. */
|
|
3212
|
+
_pinBusyForNew(a, now = Date.now()) {
|
|
3213
|
+
if (this._recentlyRefusedBusy(a, now)) return true;
|
|
3214
|
+
const depth = this._remoteDepthEstimate(a, now);
|
|
3215
|
+
return Boolean(a.remote && Number.isFinite(a._remotePinLimit) && depth != null
|
|
3216
|
+
&& depth >= a._remotePinLimit - 1);
|
|
3217
|
+
}
|
|
3218
|
+
|
|
3219
|
+
/** A cloud row's in-flight depth NOW, as best this pool can tell (review 2026-11-16,
|
|
3220
|
+
* 2c). The poll snapshot (every ~30 s) counted our own leases at poll time; swap those
|
|
3221
|
+
* for our current leases. A refusal's depth header is fresher but describes one
|
|
3222
|
+
* instant, so it counts only within the busy window it came with (pinBusyUntil). */
|
|
3223
|
+
_remoteDepthEstimate(a, now = Date.now()) {
|
|
3224
|
+
if (Number.isFinite(a._refusalDepth) && a._refusalDepthUntil > now) return a._refusalDepth;
|
|
3225
|
+
if (!Number.isFinite(a._remoteInflightCount)) return null;
|
|
3226
|
+
return Math.max(0, a._remoteInflightCount - (a._remoteOwnAtSync || 0)) + (a.inFlight || 0);
|
|
3227
|
+
}
|
|
3228
|
+
|
|
3229
|
+
/** Only the TIME-CAPPED half of the busy signal: the row refused busy within its
|
|
3230
|
+
* 1-5 s pinBusyUntil window. Route-existence checks use this alone; the depth
|
|
3231
|
+
* snapshot (30 s poll) ranks placement but never denies a route. */
|
|
3232
|
+
_recentlyRefusedBusy(a, now = Date.now()) {
|
|
3233
|
+
return Boolean(a.pinBusyUntil && now < a.pinBusyUntil);
|
|
3234
|
+
}
|
|
3235
|
+
|
|
3236
|
+
/** A busy refusal on this row: repel new placements for `ms` (clamped 1-5 s). A
|
|
3237
|
+
* success does NOT clear it — the refusal said the row is at its pin limit. */
|
|
3238
|
+
markPinBusy(index, ms) {
|
|
3239
|
+
const a = this.accounts[index];
|
|
3240
|
+
if (!a) return;
|
|
3241
|
+
const d = Math.min(PIN_BUSY_MAX_MS, Math.max(PIN_BUSY_MIN_MS, Number(ms) || PIN_BUSY_MIN_MS));
|
|
3242
|
+
a.pinBusyUntil = Math.max(a.pinBusyUntil || 0, Date.now() + d);
|
|
3243
|
+
}
|
|
3244
|
+
|
|
3245
|
+
/** A throttled linked pool (its ANTHROPIC egress is held): cool every Claude-egress
|
|
3246
|
+
* mirror row of that link at once, so the request moves off the link in ONE hop.
|
|
3247
|
+
* Provider (GLM/Kimi) mirrors are NEVER cooled by a Claude throttle — the cloud
|
|
3248
|
+
* only refuses 'throttled' for account.type !== 'provider' (see _matchesRequest),
|
|
3249
|
+
* so those rows can still serve, and cooling them would black out GLM/'all'
|
|
3250
|
+
* routing for up to 30 s. */
|
|
3251
|
+
markLinkMirrorsCooldown(fromAccount, ms) {
|
|
3252
|
+
const linkName = fromAccount?.remote?.linkName;
|
|
3253
|
+
if (!linkName) return [];
|
|
3254
|
+
const until = Date.now() + Math.max(0, Number(ms) || 0);
|
|
3255
|
+
const cooled = this.accounts.filter(a => a.remote && a.remote.linkName === linkName && a.type !== 'provider');
|
|
3256
|
+
for (const a of cooled) a.cooldownUntil = Math.max(a.cooldownUntil || 0, until);
|
|
3257
|
+
return cooled;
|
|
3258
|
+
}
|
|
3259
|
+
|
|
3260
|
+
/** Fresher depth for a cloud row from a refusal's x-maxpool-pin-* headers. */
|
|
3261
|
+
noteRemoteDepth(name, inflight, limit) {
|
|
3262
|
+
const a = this.accounts.find(x => x.remote && x.name === name);
|
|
3263
|
+
if (!a) return;
|
|
3264
|
+
// Held only for the busy window it came with — never overwrites the poll snapshot,
|
|
3265
|
+
// so a refusal can't keep the row repelled until the next poll.
|
|
3266
|
+
if (Number.isFinite(inflight)) {
|
|
3267
|
+
a._refusalDepth = inflight;
|
|
3268
|
+
a._refusalDepthUntil = Date.now() + PIN_BUSY_MAX_MS;
|
|
3269
|
+
}
|
|
3270
|
+
if (Number.isFinite(limit)) a._remotePinLimit ??= limit;
|
|
3271
|
+
}
|
|
3272
|
+
|
|
2955
3273
|
_matchesRequest(account, profile, requestInfo = {}) {
|
|
2956
3274
|
if (this.admissionPaused) return false;
|
|
2957
3275
|
// A pinned request (from a linked pool) may ONLY run on the named account, and
|
|
2958
3276
|
// never on a mirror row (no chained forwarding between pools).
|
|
2959
3277
|
if (requestInfo.pinAccount != null) {
|
|
2960
3278
|
if (account.remote || account.name !== requestInfo.pinAccount) return false;
|
|
3279
|
+
// P3: a pinned request isn't competing in this pool's queue — it skips the
|
|
3280
|
+
// queue-fairness gate (that alone 503'd every pinned Claude request). The
|
|
3281
|
+
// upstream throttle STAYS: it describes this pool's real egress to Anthropic.
|
|
3282
|
+
if (!this._isRequestCompatible(account, profile, requestInfo)) return false;
|
|
3283
|
+
if (account.type !== 'provider' && this._isUpstreamThrottleBlocking()) {
|
|
3284
|
+
requestInfo.pinRefusal = 'throttled';
|
|
3285
|
+
return false;
|
|
3286
|
+
}
|
|
2961
3287
|
// P1: the pin bypasses this pool's balancer, so keep its anti-dogpile floor
|
|
2962
3288
|
// here — refuse past 2x the soft per-account target; the caller fails over.
|
|
2963
|
-
|
|
3289
|
+
// 'busy' tells the edge "servable once a slot frees — wait for it", so it is said
|
|
3290
|
+
// ONLY for an account that is otherwise available (review 2026-11-16, 2d). An
|
|
3291
|
+
// unavailable one (rate-limited, cooling, exhausted) matches here and is rejected
|
|
3292
|
+
// by the selection loop's _isAvailable, so the refusal reads 'unavailable' and the
|
|
3293
|
+
// edge moves at once instead of waiting out its budget on it.
|
|
3294
|
+
if ((account.inFlight || 0) >= this._pinBusyLimit(account)
|
|
3295
|
+
&& this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true, model: requestInfo.model })) {
|
|
2964
3296
|
requestInfo.pinRefusedBusy = true;
|
|
3297
|
+
requestInfo.pinRefusal = 'busy';
|
|
2965
3298
|
return false;
|
|
2966
3299
|
}
|
|
2967
|
-
// P3: a pinned request isn't competing in this pool's queue — it skips the
|
|
2968
|
-
// queue-fairness gate (that alone 503'd every pinned Claude request). The
|
|
2969
|
-
// upstream throttle STAYS: it describes this pool's real egress to Anthropic.
|
|
2970
|
-
if (!this._isRequestCompatible(account, profile, requestInfo)) return false;
|
|
2971
|
-
if (account.type !== 'provider' && this._isUpstreamThrottleBlocking()) return false;
|
|
2972
3300
|
return true;
|
|
2973
3301
|
}
|
|
2974
3302
|
// Duplicate guard: a local account whose identity also lives in a team pool
|
|
@@ -3465,8 +3793,10 @@ export class AccountManager {
|
|
|
3465
3793
|
* roomy peers (a peer serving only 'all' takes none of its 'claude' requests).
|
|
3466
3794
|
* A self that serves no profile has no confirmed room (0).
|
|
3467
3795
|
* - routable NOW: enabled, not throttled/error/exhausted, not pool-benched
|
|
3468
|
-
* (
|
|
3469
|
-
* not auth/subscription-dead.
|
|
3796
|
+
* (a benched local copy is counted through its cloud mirror, never twice),
|
|
3797
|
+
* not auth/subscription-dead. A cloud MIRROR row counts like a local one,
|
|
3798
|
+
* judged on the cloud's exported state; its freshness is the cloud's own
|
|
3799
|
+
* quota.lastProbeOkAt (an older cloud that sends none = unknown = not counted).
|
|
3470
3800
|
* - weekly util (unified7d) leaves ≥ POLICY.holdPeerWeeklyRoom to the peer's
|
|
3471
3801
|
* weekly ceiling — its effective usage cap, else 1.0 (uncapped: below the 0.65
|
|
3472
3802
|
* `soft` tier the router already spreads load at). Red-team #4: room is
|
|
@@ -3475,8 +3805,9 @@ export class AccountManager {
|
|
|
3475
3805
|
* session bench (_sessionBenchThreshold: its 5h cap, else the 0.90 switch
|
|
3476
3806
|
* threshold) — a session-benched account absorbs nothing.
|
|
3477
3807
|
* - a DIFFERENT identity (accountUuid/credential fingerprint) — not this same
|
|
3478
|
-
* account under a second name; peers sharing one
|
|
3479
|
-
* names for one subscription) count ONCE
|
|
3808
|
+
* account under a second name; peers sharing one capacity group (two local
|
|
3809
|
+
* names, or a mirror and its org-less twin, for one subscription) count ONCE
|
|
3810
|
+
* (red-team #6).
|
|
3480
3811
|
* - every ACTIVE model bucket the peer reports also has ≥ holdPeerWeeklyRoom
|
|
3481
3812
|
* left (red-team #1) — the walled account serves no model at all, so a peer
|
|
3482
3813
|
* whose Fable is exhausted/full (or 429-marked, or unreadable) would leave
|
|
@@ -3500,10 +3831,16 @@ export class AccountManager {
|
|
|
3500
3831
|
const id = this.capacityIdentity(p);
|
|
3501
3832
|
if (id == null || selfGroup.has(p)) continue;
|
|
3502
3833
|
if (p.type !== 'oauth' || p.enabled === false) continue;
|
|
3503
|
-
|
|
3834
|
+
// A benched local copy is skipped; its identity is counted through the cloud
|
|
3835
|
+
// MIRROR of the same account (one peer per capacity group, below).
|
|
3836
|
+
if (p.poolBenchedBy) continue;
|
|
3504
3837
|
if (p.refreshDead || p.subscriptionGone) continue;
|
|
3505
3838
|
if (p.status === 'throttled' || p.status === 'error' || p.status === 'exhausted') continue;
|
|
3506
3839
|
if (p.cooldownUntil && p.cooldownUntil > now) continue;
|
|
3840
|
+
// A cloud MIRROR carries the cloud's own exported state (status, refreshDead,
|
|
3841
|
+
// quota incl. the cloud's lastProbeOkAt, cap, cap-lift) — judged below by the
|
|
3842
|
+
// same gates as a local row. Its cooldown is the cloud's, copied separately.
|
|
3843
|
+
if (p.remote && p.remoteCooldownUntil && p.remoteCooldownUntil > now) continue;
|
|
3507
3844
|
const q = p.quota || {};
|
|
3508
3845
|
// Probe-fresh only: a peer whose reading predates this sweep's decision is
|
|
3509
3846
|
// unknown capacity, and unknown capacity must not hold a claim.
|
|
@@ -3514,11 +3851,13 @@ export class AccountManager {
|
|
|
3514
3851
|
if (!roomLeft(q.unified5h, this._sessionBenchThreshold(p, now), POLICY.holdPeerSessionRoom)) continue;
|
|
3515
3852
|
const buckets = Object.values(q.scopedWeekly || {}).filter(e => e && e.isActive !== false);
|
|
3516
3853
|
if (buckets.some(e => !roomLeft(e.utilization, 1, POLICY.holdPeerWeeklyRoom))) continue;
|
|
3517
|
-
|
|
3854
|
+
// One peer per capacity GROUP (lowest member index as its key): an org-less twin
|
|
3855
|
+
// of the same login, or a mirror + a second mirror of it, is one subscription.
|
|
3856
|
+
roomy.push({ p, key: Math.min(...this.capacityGroupOf(p).map(x => x.index)) });
|
|
3518
3857
|
}
|
|
3519
3858
|
if (!selfProfiles.length) return 0;
|
|
3520
3859
|
return Math.min(...selfProfiles.map(pr =>
|
|
3521
|
-
new Set(roomy.filter(({ p }) => this._matchesProfile(p, pr)).map(({
|
|
3860
|
+
new Set(roomy.filter(({ p }) => this._matchesProfile(p, pr)).map(({ key }) => key)).size));
|
|
3522
3861
|
}
|
|
3523
3862
|
|
|
3524
3863
|
// ── R7 hold register (red-team r2 #1/#2) ──────────────────────────────────
|
|
@@ -3554,7 +3893,15 @@ export class AccountManager {
|
|
|
3554
3893
|
/** The walls a hold must budget for: every OTHER account holding now, plus
|
|
3555
3894
|
* `self` (the wall being decided). Feeds wallClaimHeld's holdersInPool. */
|
|
3556
3895
|
countHoldingWalls(self, now = Date.now()) {
|
|
3557
|
-
|
|
3896
|
+
// Cloud mirrors are peers in countPeersWithRoom, so the cloud's OWN holds draw on the
|
|
3897
|
+
// same spare capacity and count as holders here (review 2026-11-16, cluster 6) —
|
|
3898
|
+
// else two pools each budget the same roomy cloud accounts. A mirror of self's own
|
|
3899
|
+
// capacity group is self, never a second holder. An older cloud exports no
|
|
3900
|
+
// wallHolding, so it adds none (the previous behaviour).
|
|
3901
|
+
const selfGroup = new Set(this.capacityGroupOf(self).map(x => x.index));
|
|
3902
|
+
const remoteHolders = this.accounts.filter(a => a !== self && a.remote && a.enabled !== false
|
|
3903
|
+
&& a._remoteWallHolding === true && !selfGroup.has(a.index)).length;
|
|
3904
|
+
return 1 + this.accounts.filter(a => a !== self && this.isHoldingWall(a, now)).length + remoteHolders;
|
|
3558
3905
|
}
|
|
3559
3906
|
|
|
3560
3907
|
/**
|
|
@@ -3606,7 +3953,10 @@ export class AccountManager {
|
|
|
3606
3953
|
// A cloud mirror's depth includes OTHER teammates' live load on that same cloud
|
|
3607
3954
|
// account (mirrored from the link poll), so every routing term below — the
|
|
3608
3955
|
// unchanged balancing logic — sees it as busy as it really is.
|
|
3609
|
-
|
|
3956
|
+
// Other teammates' load in WEIGHT units (the cloud's activeWeight minus ours, 2026-11-15);
|
|
3957
|
+
// an older cloud that omits activeWeight falls back to its in-flight count.
|
|
3958
|
+
const remoteLoadW = Number.isFinite(account._remoteOtherWeight) ? account._remoteOtherWeight : (account._remoteOtherLoad || 0);
|
|
3959
|
+
const inflight = account.activeWeight + reqWeight + remoteLoadW;
|
|
3610
3960
|
const refillMult = this._fastRefillMultiplier(account);
|
|
3611
3961
|
|
|
3612
3962
|
// DOMINANT term: in-flight concurrency. Short-term throttling is driven by
|
|
@@ -3669,7 +4019,13 @@ export class AccountManager {
|
|
|
3669
4019
|
// benching, but wrong for load balancing: an account at 80% should be clearly less
|
|
3670
4020
|
// attractive than one at 10% even if both are "on pace". Measured 2026-08-10: cc at
|
|
3671
4021
|
// 80% scored 52.30 vs glm at 10% at 52.15 — a 0.15 gap drowned by round-robin.
|
|
3672
|
-
|
|
4022
|
+
// + the absolute weekly headroom tie-break (ABS_UTIL_WEIGHT), behind the same
|
|
4023
|
+
// weekly-aware flag so the 5h-only mode stays byte-identical.
|
|
4024
|
+
// A FULL lift refilled the week: like every other weekly term, the tie-break must
|
|
4025
|
+
// not price it on its pre-refill usage (else a lifted 0.97 is worst-of-pool again).
|
|
4026
|
+
const absUsed = this.scheduler.weeklyAwareScoring !== false && !this._weeklyRefilledByLift(account, now)
|
|
4027
|
+
? this._absWeeklyUsed(account) : 0;
|
|
4028
|
+
const utilizationCost = (this._rawUtilization(account) + ABS_UTIL_WEIGHT * absUsed) * this.scheduler.utilizationWeight * refillMult * resetMult;
|
|
3673
4029
|
|
|
3674
4030
|
// Per-model weekly de-preference: an account whose scoped weekly for THIS
|
|
3675
4031
|
// request's model (e.g. Fable) is high-but-not-exhausted is a poor pick for
|
|
@@ -3683,7 +4039,8 @@ export class AccountManager {
|
|
|
3683
4039
|
|
|
3684
4040
|
const fleetRecentWeight = ctx?.fleetRecentWeight ?? 0;
|
|
3685
4041
|
const recentWeight = this._loadSummary(account, this.scheduler.spreadWindowMs, now).weight;
|
|
3686
|
-
|
|
4042
|
+
// Clamped: a provider is outside the denominator, so its raw share can exceed 1.
|
|
4043
|
+
const share = fleetRecentWeight > 0 ? Math.min(1, recentWeight / fleetRecentWeight) : 0;
|
|
3687
4044
|
// SPREAD also carries the fast-refill discount (2026-08-29, owner option 1). The
|
|
3688
4045
|
// spread term is the equaliser that pulls every account back to fleet parity — at
|
|
3689
4046
|
// converged equal share it is ~10x larger than the utilization+pace gap the
|
|
@@ -3740,8 +4097,9 @@ export class AccountManager {
|
|
|
3740
4097
|
* Applies in every scored mode (balance, prefer-*); never in sticky (see below).
|
|
3741
4098
|
*/
|
|
3742
4099
|
_noteConversationAccount(requestInfo, account) {
|
|
3743
|
-
|
|
3744
|
-
|
|
4100
|
+
// conversationLastAccount is written at the first real upstream byte
|
|
4101
|
+
// (noteConversationServed), never here: a refused/failed hop must not re-point the
|
|
4102
|
+
// conversation (2026-11-15).
|
|
3745
4103
|
// Home memory: ONE key per conversation, independent of identity health, so the
|
|
3746
4104
|
// degraded <-> ok flip of _affinityKey can neither hide a home nor lend it to the
|
|
3747
4105
|
// sibling conversations that share the terminal key. The entry carries its own
|
|
@@ -3758,6 +4116,39 @@ export class AccountManager {
|
|
|
3758
4116
|
}
|
|
3759
4117
|
}
|
|
3760
4118
|
|
|
4119
|
+
/** The conversation was SERVED by `account` (first real upstream byte). Idempotent
|
|
4120
|
+
* per request. Records whether this turn moved the conversation (for telemetry). */
|
|
4121
|
+
noteConversationServed(requestInfo, account) {
|
|
4122
|
+
if (!requestInfo || !account || requestInfo._notedServed === account.name) return;
|
|
4123
|
+
// count_tokens carries no prompt cache: it must not re-point the conversation's
|
|
4124
|
+
// account memory (holdsWarmConversation / move telemetry read it).
|
|
4125
|
+
if (requestInfo.isCountTokens) return;
|
|
4126
|
+
requestInfo._notedServed = account.name;
|
|
4127
|
+
requestInfo._servedAt = Date.now();
|
|
4128
|
+
const key = this._affinityKey(requestInfo);
|
|
4129
|
+
if (!key) return;
|
|
4130
|
+
const prev = this.conversationLastAccount.get(key);
|
|
4131
|
+
requestInfo._newConversation = prev == null;
|
|
4132
|
+
requestInfo._servedMoved = prev != null && prev !== account.name;
|
|
4133
|
+
lruSet(this.conversationLastAccount, key, account.name);
|
|
4134
|
+
lruSet(this.conversationLastServedAt, key, Date.now());
|
|
4135
|
+
}
|
|
4136
|
+
|
|
4137
|
+
/** Does `account` hold this conversation's WARM Claude cache — it served the last
|
|
4138
|
+
* turn, within the cache window? (The sticky busy wait is only worth it then.) */
|
|
4139
|
+
holdsWarmConversation(account, requestInfo = {}) {
|
|
4140
|
+
if (!isClaudeAccount(account)) return false;
|
|
4141
|
+
const key = this._affinityKey(requestInfo);
|
|
4142
|
+
if (!key || this.conversationLastAccount.get(key) !== account.name) return false;
|
|
4143
|
+
return Date.now() - (this.conversationLastServedAt.get(key) || 0) <= this._cacheAffinityWindowMs();
|
|
4144
|
+
}
|
|
4145
|
+
|
|
4146
|
+
/** cacheAffinityReturnWindowMs, falling back to the default when unset/invalid. */
|
|
4147
|
+
_cacheAffinityWindowMs() {
|
|
4148
|
+
const w = Number(this.scheduler.cacheAffinityReturnWindowMs);
|
|
4149
|
+
return Number.isFinite(w) && w >= 0 ? w : DEFAULT_SCHEDULER.cacheAffinityReturnWindowMs;
|
|
4150
|
+
}
|
|
4151
|
+
|
|
3761
4152
|
/** Key of the conversation's home entry: the conversation itself, never the shared
|
|
3762
4153
|
* terminal key when a conversation id exists. When identity is genuinely broken
|
|
3763
4154
|
* (a fresh hash every request) the home is simply never found, so the normal pick
|
|
@@ -3779,9 +4170,7 @@ export class AccountManager {
|
|
|
3779
4170
|
const home = this.conversationLastClaudeAccount.get(homeKey);
|
|
3780
4171
|
if (!home || !home.detoured) return null;
|
|
3781
4172
|
const ageMs = now - home.at;
|
|
3782
|
-
|
|
3783
|
-
if (!Number.isFinite(windowMs) || windowMs < 0) windowMs = DEFAULT_SCHEDULER.cacheAffinityReturnWindowMs;
|
|
3784
|
-
return { name: home.name, stale: ageMs > windowMs, ageMs };
|
|
4173
|
+
return { name: home.name, stale: ageMs > this._cacheAffinityWindowMs(), ageMs };
|
|
3785
4174
|
}
|
|
3786
4175
|
|
|
3787
4176
|
/** Conversation identity: terminal + prefix hash when the server computed one;
|
|
@@ -4046,6 +4435,13 @@ export class AccountManager {
|
|
|
4046
4435
|
return util;
|
|
4047
4436
|
}
|
|
4048
4437
|
|
|
4438
|
+
/** Absolute weekly used fraction (0..1), not pace-adjusted — the tie-break input. */
|
|
4439
|
+
_absWeeklyUsed(account) {
|
|
4440
|
+
const q = account?.quota;
|
|
4441
|
+
if (!q) return 0;
|
|
4442
|
+
return Math.max(q.unified7d != null ? clamp01(q.unified7d) : 0, q.providerWk != null ? clamp01(q.providerWk) : 0);
|
|
4443
|
+
}
|
|
4444
|
+
|
|
4049
4445
|
_windowScarcity(util, resetMs, windowLen, now = Date.now()) {
|
|
4050
4446
|
const used = clamp01(util);
|
|
4051
4447
|
if (!resetMs || resetMs <= now) return used; // unknown / just-reset → face value
|
|
@@ -6046,6 +6442,9 @@ export class AccountManager {
|
|
|
6046
6442
|
a.quota = { ...emptyQuota(), ...(ra.quota || {}) };
|
|
6047
6443
|
a.status = ra.status || 'active';
|
|
6048
6444
|
a.refreshDead = Boolean(ra.refreshDead);
|
|
6445
|
+
// The CLOUD's cooldown (additive export; an older cloud omits it = none). Kept off
|
|
6446
|
+
// `cooldownUntil`, which is this process's own routing gate for the mirror row.
|
|
6447
|
+
{ const cd = ra.cooldownUntil ? Date.parse(ra.cooldownUntil) : NaN; a.remoteCooldownUntil = Number.isFinite(cd) ? cd : null; }
|
|
6049
6448
|
a.lastError = ra.lastError || null;
|
|
6050
6449
|
a.rateLimitedUntil = ra.rateLimitedUntil ? Date.parse(ra.rateLimitedUntil) : null;
|
|
6051
6450
|
a.capUtilization = ra.capUtilization ?? null;
|
|
@@ -6070,7 +6469,16 @@ export class AccountManager {
|
|
|
6070
6469
|
// Other teammates' live load on this cloud account (remote inFlight minus our
|
|
6071
6470
|
// own leases) — so the balancer sees the account as busy as it really is.
|
|
6072
6471
|
a._remoteOtherLoad = Math.max(0, (ra.inFlight || 0) - (a.inFlight || 0));
|
|
6472
|
+
// The same in WEIGHT units (NaN → fallback when an older cloud omits activeWeight).
|
|
6473
|
+
a._remoteOtherWeight = Number.isFinite(ra.activeWeight) ? Math.max(0, ra.activeWeight - (a.activeWeight || 0)) : NaN;
|
|
6474
|
+
a._remoteInflightCount = Number.isFinite(ra.inFlight) ? ra.inFlight : null;
|
|
6475
|
+
// Our own leases on this row at poll time (already inside ra.inFlight): the depth
|
|
6476
|
+
// estimate swaps them for our CURRENT leases, so our finished requests stop
|
|
6477
|
+
// counting against the row until the next poll (review 2026-11-16, 2c).
|
|
6478
|
+
a._remoteOwnAtSync = a.inFlight || 0;
|
|
6479
|
+
a._remotePinLimit = Number.isFinite(ra.pinBusyAt) ? ra.pinBusyAt : null;
|
|
6073
6480
|
a._remoteLoad = ra.load || null;
|
|
6481
|
+
a._remoteWallHolding = ra.wallHolding === true;
|
|
6074
6482
|
a.probing = false;
|
|
6075
6483
|
a.enabled = true;
|
|
6076
6484
|
}
|
|
@@ -6303,6 +6711,11 @@ export class AccountManager {
|
|
|
6303
6711
|
since: this.identityHealth.since,
|
|
6304
6712
|
},
|
|
6305
6713
|
currentAccount: this.accounts[this.currentIndex]?.name,
|
|
6714
|
+
// Overall cache / move / bounce / wait figures — the SAME merge the TUI Cache line
|
|
6715
|
+
// uses (mirror rows: cache + served from the serving pool's export, edge-only
|
|
6716
|
+
// fields from this pool's own events), plus edge client outcomes.
|
|
6717
|
+
efficiency: { last15m: this.efficiencyOverall('last15m', now), last1h: this.efficiencyOverall('last1h', now) },
|
|
6718
|
+
clientOutcomes: { last15m: this.clientOutcomeSummary(15 * 60_000, now), last1h: this.clientOutcomeSummary(60 * 60_000, now) },
|
|
6306
6719
|
switchThreshold: this.switchThreshold,
|
|
6307
6720
|
routing: {
|
|
6308
6721
|
mode: this.routingMode,
|
|
@@ -6386,15 +6799,29 @@ export class AccountManager {
|
|
|
6386
6799
|
inFlight: a.inFlight,
|
|
6387
6800
|
activeWeight: a.activeWeight,
|
|
6388
6801
|
},
|
|
6389
|
-
last15m: this._loadSummary(a, 15 * 60 * 1000, now),
|
|
6390
|
-
last1h: this._loadSummary(a, 60 * 60 * 1000, now),
|
|
6802
|
+
last15m: { ...this._loadSummary(a, 15 * 60 * 1000, now), efficiency: this.rowEfficiencySummary(a, 'last15m', now) },
|
|
6803
|
+
last1h: { ...this._loadSummary(a, 60 * 60 * 1000, now), efficiency: this.rowEfficiencySummary(a, 'last1h', now) },
|
|
6391
6804
|
},
|
|
6805
|
+
// The cloud's pin limit for this row (linked pools skip it one short of this).
|
|
6806
|
+
pinBusyAt: a.remote ? null : this._pinBusyLimit(a),
|
|
6807
|
+
// A cloud row's cache figures as the SERVING pool exported them (team-wide on
|
|
6808
|
+
// that account); null on an older cloud or a local row.
|
|
6809
|
+
teamEfficiency: a.remote ? {
|
|
6810
|
+
last15m: a._remoteLoad?.last15m?.efficiency ?? null,
|
|
6811
|
+
last1h: a._remoteLoad?.last1h?.efficiency ?? null,
|
|
6812
|
+
} : null,
|
|
6392
6813
|
lastError: a.lastError,
|
|
6393
6814
|
lastErrorAt: a.lastErrorAt ? new Date(a.lastErrorAt).toISOString() : null,
|
|
6394
6815
|
cooldownUntil: Math.max(a.cooldownUntil || 0, a.provisionalUpstreamUntil || 0)
|
|
6395
6816
|
? new Date(Math.max(a.cooldownUntil || 0, a.provisionalUpstreamUntil || 0)).toISOString()
|
|
6396
6817
|
: null,
|
|
6397
|
-
|
|
6818
|
+
// A BENCHED duplicate is never probed or refreshed locally, so its own quota is
|
|
6819
|
+
// stale by design — don't export it as current. The cloud pool owns the live
|
|
6820
|
+
// numbers (the TUI overlays them via remoteRowFor); consumers read this marker.
|
|
6821
|
+
quota: a.poolBenchedBy ? null : { ...a.quota },
|
|
6822
|
+
quotaSource: a.poolBenchedBy ? 'cloud-mirror' : 'local',
|
|
6823
|
+
// A linked pool counts this row's hold against the spare capacity it shares.
|
|
6824
|
+
wallHolding: a.remote ? false : this.isHoldingWall(a, now),
|
|
6398
6825
|
outsideUsage: a.outsideUsage ? { ...a.outsideUsage } : null,
|
|
6399
6826
|
weekly: {
|
|
6400
6827
|
state: this._weeklyState(a),
|