maxpool 1.24.94 → 1.24.96
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -1
- package/package.json +1 -1
- package/src/account-manager.js +491 -39
- package/src/cloud-login.js +1 -0
- package/src/index.js +96 -34
- package/src/oauth.js +79 -0
- package/src/pool-link.js +60 -11
- package/src/prober.js +11 -2
- package/src/server.js +313 -33
- package/src/tui.js +424 -112
- package/src/usage-label.js +50 -0
package/src/account-manager.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { refreshAccessToken, isTokenExpiringSoon, modelFamily, tokenFingerprint } from './oauth.js';
|
|
1
|
+
import { refreshAccessToken, isTokenExpiringSoon, modelFamily, tokenFingerprint, sameUser, sameAccountIdentity } from './oauth.js';
|
|
2
2
|
import { computeDuplicates, credentialFp, accountIdentity, remoteAccountIdentity } from './pool-link.js';
|
|
3
3
|
import { appendEventLog } from './event-log.js';
|
|
4
4
|
import { hourFloor } from './outside-usage.js';
|
|
@@ -225,6 +225,11 @@ const DEFAULT_SCHEDULER = {
|
|
|
225
225
|
// is alive: Claude Code writes 1-hour copies (measured 2026-11-09: 272 of 272 cache
|
|
226
226
|
// writes `ephemeral_1h`; 55 of 64 detours came back within the hour, median 533 s).
|
|
227
227
|
cacheAffinityReturnWindowMs: 3_600_000,
|
|
228
|
+
// STICKY BUSY WAIT (owner 2026-11-15): how long, in total, a warm Claude conversation
|
|
229
|
+
// waits for its own cloud account to stop being busy before it moves (a move re-writes
|
|
230
|
+
// its whole prompt cache). Edited live from the TUI routing screen (`b`); env
|
|
231
|
+
// MAXPOOL_STICKY_BUSY_BUDGET_MS overrides it when set. 0 = never wait.
|
|
232
|
+
stickyBusyBudgetMs: 60_000,
|
|
228
233
|
concurrencyWeight: 2, // multiplies in-flight load (activeWeight+reqWeight) — dominant
|
|
229
234
|
perAccountConcurrencyTarget: 3, // D: soft per-account in-flight target; past it, capPenalty bites
|
|
230
235
|
capPenaltyWeight: 10, // steep penalty per unit of in-flight depth past D (throttle safety floor)
|
|
@@ -342,6 +347,16 @@ const DEFAULT_SCHEDULER = {
|
|
|
342
347
|
providers: {},
|
|
343
348
|
};
|
|
344
349
|
const LOAD_EVENT_MAX_AGE_MS = 60 * 60 * 1000;
|
|
350
|
+
// Absolute weekly headroom TIE-BREAK for new placements (2026-11-15): at equal load a
|
|
351
|
+
// fresh-weekly account beats a 60%-weekly one. ~0.75 points at 100% used vs ~70 per
|
|
352
|
+
// in-flight unit, so it only ever decides near-ties; held conversations are unaffected
|
|
353
|
+
// (their steering terms are zeroed).
|
|
354
|
+
const ABS_UTIL_WEIGHT = 0.25;
|
|
355
|
+
// Pinned-busy back-off bounds for a cloud row (markPinBusy).
|
|
356
|
+
const PIN_BUSY_MIN_MS = 1000;
|
|
357
|
+
const PIN_BUSY_MAX_MS = 5000;
|
|
358
|
+
// Edge-pool client outcomes ring (noteClientOutcome).
|
|
359
|
+
const CLIENT_OUTCOME_MAX_AGE_MS = 60 * 60 * 1000;
|
|
345
360
|
// Consecutive failed recovery probes before we stop trusting the SHARED breaker and
|
|
346
361
|
// fall back to per-account handling. Bounds any future "poisoned probe" from becoming
|
|
347
362
|
// an indefinite fleet-wide outage (2026-07-25 incident).
|
|
@@ -381,6 +396,67 @@ const PERSISTED_QUOTA_FIELDS = [
|
|
|
381
396
|
'providerQuotaSource', 'lastProbeOkAt', 'noPlan',
|
|
382
397
|
];
|
|
383
398
|
|
|
399
|
+
/** The sticky busy-wait budget actually in force, and where it comes from — ONE resolver
|
|
400
|
+
* for the server (per request, live) and the TUI setting (review 2026-11-16, 5c: the
|
|
401
|
+
* screen used to show the saved setting while an env override silently won). */
|
|
402
|
+
export function resolveStickyBusyBudget(scheduler, env = process.env) {
|
|
403
|
+
const raw = env.MAXPOOL_STICKY_BUSY_BUDGET_MS;
|
|
404
|
+
if (raw != null && raw !== '' && Number.isFinite(Number(raw)) && Number(raw) >= 0) {
|
|
405
|
+
return { ms: Number(raw), fromEnv: true };
|
|
406
|
+
}
|
|
407
|
+
const v = Number(scheduler?.stickyBusyBudgetMs);
|
|
408
|
+
return { ms: Number.isFinite(v) && v >= 0 ? v : 60_000, fromEnv: false };
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/** Figures only the EDGE pool (the one the client talks to) can know for a request it
|
|
412
|
+
* forwarded to a cloud row — see _mirrorEfficiencyAcc. */
|
|
413
|
+
const EDGE_EFFICIENCY_FIELDS = ['bounced', 'refused', 'waited', 'waitMs', 'waitThenMoved', 'moved', 'newConversation'];
|
|
414
|
+
|
|
415
|
+
const efficiencyWindowMs = (windowKey) => (windowKey === 'last1h' ? 60 * 60_000 : 15 * 60_000);
|
|
416
|
+
|
|
417
|
+
function emptyEfficiency() {
|
|
418
|
+
return { served: 0, in: 0, out: 0, cr: 0, cc: 0, moved: 0,
|
|
419
|
+
movedByCause: { busy: 0, wall: 0, balance: 0, link: 0, other: 0 },
|
|
420
|
+
newConversation: 0, bounced: 0, refused: 0, waited: 0, waitMs: 0, waitThenMoved: 0, e2e: [] };
|
|
421
|
+
}
|
|
422
|
+
/** Add one exported/summarized efficiency block into an accumulator (missing fields = 0). */
|
|
423
|
+
function addEfficiency(acc, e) {
|
|
424
|
+
for (const k of ['served', 'in', 'out', 'cr', 'cc', 'moved', 'newConversation', 'bounced', 'refused', 'waited', 'waitMs', 'waitThenMoved']) {
|
|
425
|
+
acc[k] += Number(e[k]) || 0;
|
|
426
|
+
}
|
|
427
|
+
for (const c of Object.keys(acc.movedByCause)) acc.movedByCause[c] += Number(e.movedByCause?.[c]) || 0;
|
|
428
|
+
// Latency samples exist for this pool's own rows only (percentiles do not add).
|
|
429
|
+
if (Array.isArray(e.e2e)) acc.e2e.push(...e.e2e);
|
|
430
|
+
}
|
|
431
|
+
const pct = (arr, p) => {
|
|
432
|
+
if (!arr.length) return null;
|
|
433
|
+
const s = [...arr].sort((x, y) => x - y);
|
|
434
|
+
return s[Math.min(s.length - 1, Math.floor(p * s.length))];
|
|
435
|
+
};
|
|
436
|
+
/** Derived figures: hit = cr/(cr+cc+in) (GLM has no cc → r/(r+in)); missTokens = in+cc. */
|
|
437
|
+
function finishEfficiency(acc, provider) {
|
|
438
|
+
const prompt = acc.cr + acc.cc + acc.in;
|
|
439
|
+
const { e2e, ...sums } = acc;
|
|
440
|
+
return {
|
|
441
|
+
...sums,
|
|
442
|
+
provider,
|
|
443
|
+
hit: prompt > 0 ? acc.cr / prompt : null,
|
|
444
|
+
missTokens: acc.in + acc.cc,
|
|
445
|
+
promptTokens: prompt,
|
|
446
|
+
lowVolume: acc.served < 20 || prompt < 100_000,
|
|
447
|
+
e2eP50: pct(e2e, 0.5),
|
|
448
|
+
e2eP95: pct(e2e, 0.95),
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** Combine efficiency blocks (summaries or exports) into one, with derived figures —
|
|
453
|
+
* `maxpool status` uses it to add linked pools' exported sums to the local ones. */
|
|
454
|
+
export function combineEfficiency(blocks, provider) {
|
|
455
|
+
const acc = emptyEfficiency();
|
|
456
|
+
for (const b of blocks) if (b) addEfficiency(acc, b);
|
|
457
|
+
return finishEfficiency(acc, provider);
|
|
458
|
+
}
|
|
459
|
+
|
|
384
460
|
const CONVERSATION_AFFINITY_MAX = 5000;
|
|
385
461
|
/** Bounded LRU insert: re-insert = most recent; the oldest key drops past the cap. */
|
|
386
462
|
function lruSet(map, key, value) {
|
|
@@ -625,6 +701,7 @@ export class AccountManager {
|
|
|
625
701
|
type: acct.type,
|
|
626
702
|
provider: acct.provider || (acct.type === 'provider' ? 'provider' : 'anthropic'),
|
|
627
703
|
accountUuid: acct.accountUuid || null,
|
|
704
|
+
organizationUuid: acct.organizationUuid || null,
|
|
628
705
|
credential: acct.accessToken || acct.authToken || acct.apiKey,
|
|
629
706
|
upstream: acct.upstream || null,
|
|
630
707
|
authHeader: acct.authHeader || null,
|
|
@@ -697,6 +774,11 @@ export class AccountManager {
|
|
|
697
774
|
// Bounded LRU — oldest conversations drop first; a dropped entry only means the
|
|
698
775
|
// next turn is balanced fresh.
|
|
699
776
|
this.conversationLastAccount = new Map();
|
|
777
|
+
// When that account last SERVED the conversation (first real upstream byte) —
|
|
778
|
+
// holdsWarmConversation reads it. Same LRU bound, in-memory like the map above.
|
|
779
|
+
this.conversationLastServedAt = new Map();
|
|
780
|
+
// Final client outcomes of POST /v1/messages at THIS (edge) pool, 1 h ring.
|
|
781
|
+
this.clientOutcomes = [];
|
|
700
782
|
// _homeKey (affinityKey, else sessionKey) → { name, at, detoured } of the last CLAUDE
|
|
701
783
|
// account only (a GLM/Kimi turn never overwrites the name; it sets `detoured`), so a
|
|
702
784
|
// conversation returning from a provider detour can go home.
|
|
@@ -951,6 +1033,15 @@ export class AccountManager {
|
|
|
951
1033
|
return weeklyPasses.some(options => this.accounts.some(account => {
|
|
952
1034
|
if (excludedIndexes.has(account.index)) return false;
|
|
953
1035
|
if (!this._matchesRequest(account, profile, requestInfo)) return false;
|
|
1036
|
+
// A row that JUST refused busy (pinBusyUntil, clamped 1-5 s) is no route for a NEW
|
|
1037
|
+
// placement — else the queue pacer re-tries an all-busy pool in a sub-second spin.
|
|
1038
|
+
// Its holder may still use it, unless this request already gave up waiting on it
|
|
1039
|
+
// (_busyExcluded). The DEPTH half of _pinBusyForNew (30 s-poll snapshot, one slot
|
|
1040
|
+
// short of the limit) is deliberately NOT consulted here: the cloud still accepts
|
|
1041
|
+
// at limit-1, so that row is a route — depth belongs to placement RANKING only
|
|
1042
|
+
// (_selectNext's coolBest), matching what the acquire path would actually do.
|
|
1043
|
+
if (this._recentlyRefusedBusy(account, now)
|
|
1044
|
+
&& (!this._holdsRequest(account, requestInfo) || requestInfo._busyExcluded?.has(account.name))) return false;
|
|
954
1045
|
return this._isAvailable(account, { ...options, model: requestInfo.model, now });
|
|
955
1046
|
}));
|
|
956
1047
|
}
|
|
@@ -1000,7 +1091,7 @@ export class AccountManager {
|
|
|
1000
1091
|
const migratedFromName = (prevCurrentName && account.name !== prevCurrentName) ? prevCurrentName : null;
|
|
1001
1092
|
const returnedHome = requestInfo.returnedHome?.name === account.name;
|
|
1002
1093
|
const returnedHomeSwitched = returnedHome && requestInfo.returnedHome.switched === true;
|
|
1003
|
-
return { account, weight, startedAt: Date.now(), upstreamThrottleProbe, migratedFromName, returnedHome, returnedHomeSwitched };
|
|
1094
|
+
return { account, weight, startedAt: Date.now(), upstreamThrottleProbe, migratedFromName, returnedHome, returnedHomeSwitched, requestInfo };
|
|
1004
1095
|
}
|
|
1005
1096
|
|
|
1006
1097
|
/** Fail-safe: snap a session's binding back to the pre-migration issuing account
|
|
@@ -1159,6 +1250,13 @@ export class AccountManager {
|
|
|
1159
1250
|
return;
|
|
1160
1251
|
}
|
|
1161
1252
|
|
|
1253
|
+
// A pinned refusal (busy / unavailable / throttled cloud row) never reached the
|
|
1254
|
+
// client: one BOUNCED load event, no failure counters, no streak, no error text.
|
|
1255
|
+
if (outcome.bounced) {
|
|
1256
|
+
this._recordLoadEvent(account, lease, { ...outcome, success: false, bounced: true });
|
|
1257
|
+
return;
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1162
1260
|
// A per-model weekly cap is NOT an account failure — the account is healthy for
|
|
1163
1261
|
// every other model. Don't poison its failure counters / scoring penalty (mirror
|
|
1164
1262
|
// the network-blip carve-out); the scoped bench is already set in markRateLimited.
|
|
@@ -1182,7 +1280,14 @@ export class AccountManager {
|
|
|
1182
1280
|
account.lastError = null;
|
|
1183
1281
|
account.lastErrorAt = null;
|
|
1184
1282
|
}
|
|
1185
|
-
|
|
1283
|
+
// Backstop for callers that never saw a first byte (unit callers): a 2xx served
|
|
1284
|
+
// the conversation. Idempotent — the first-byte note already ran on the real paths.
|
|
1285
|
+
if (lease.requestInfo && !(outcome.status >= 300)) this.noteConversationServed(lease.requestInfo, account);
|
|
1286
|
+
// Efficiency fields derived from the served request; a field the caller passed
|
|
1287
|
+
// explicitly on the outcome wins.
|
|
1288
|
+
const tel = this._servedTelemetry(lease.requestInfo, account);
|
|
1289
|
+
for (const k of Object.keys(tel)) if (outcome[k] != null) tel[k] = outcome[k];
|
|
1290
|
+
this._recordLoadEvent(account, lease, { ...outcome, ...tel, success: true });
|
|
1186
1291
|
account.lastSuccessAt = Date.now();
|
|
1187
1292
|
return;
|
|
1188
1293
|
}
|
|
@@ -1213,10 +1318,149 @@ export class AccountManager {
|
|
|
1213
1318
|
success: Boolean(outcome.success),
|
|
1214
1319
|
linkFault: Boolean(outcome.linkFault),
|
|
1215
1320
|
status: outcome.status || null,
|
|
1321
|
+
// Efficiency telemetry (2026-11-15) — optional, read by _efficiencySummary only.
|
|
1322
|
+
bounced: Boolean(outcome.bounced),
|
|
1323
|
+
usage: outcome.usage || null,
|
|
1324
|
+
moved: Boolean(outcome.moved),
|
|
1325
|
+
newConversation: Boolean(outcome.newConversation),
|
|
1326
|
+
moveCause: outcome.moveCause || null,
|
|
1327
|
+
busyWaitMs: outcome.busyWaitMs || 0,
|
|
1328
|
+
e2eFirstByteMs: Number.isFinite(outcome.e2eFirstByteMs) ? outcome.e2eFirstByteMs : null,
|
|
1329
|
+
refusal: outcome.refusal || null,
|
|
1330
|
+
// _efficiencyAcc skips these: a count_tokens call is not a conversation turn.
|
|
1331
|
+
countTokens: Boolean(outcome.countTokens),
|
|
1216
1332
|
});
|
|
1217
1333
|
this._pruneLoadEvents(account, now);
|
|
1218
1334
|
}
|
|
1219
1335
|
|
|
1336
|
+
/** Efficiency fields of a SERVED event, read off the request that was served. */
|
|
1337
|
+
_servedTelemetry(ri, account) {
|
|
1338
|
+
if (!ri) return {};
|
|
1339
|
+
// A PINNED request was routed by the linked (edge) pool: the edge books its move,
|
|
1340
|
+
// cause, wait and client-felt latency on its own mirror row. Booking them here too
|
|
1341
|
+
// would double-count the move (as 'balance' — this pool never knew the cause) and
|
|
1342
|
+
// add a latency sample that starts AFTER the edge's wait. Served + tokens stay here.
|
|
1343
|
+
const pinned = ri.pinAccount != null;
|
|
1344
|
+
const moved = !pinned && Boolean(ri._servedMoved) && ri._notedServed === account.name;
|
|
1345
|
+
return {
|
|
1346
|
+
countTokens: Boolean(ri.isCountTokens),
|
|
1347
|
+
usage: ri.isCountTokens ? null : {
|
|
1348
|
+
in: ri._capacityInput || 0, out: ri._capacityOutput || 0,
|
|
1349
|
+
cr: ri._capacityCacheRead || 0, cc: ri._capacityCacheCreation || 0,
|
|
1350
|
+
},
|
|
1351
|
+
moved,
|
|
1352
|
+
moveCause: moved ? (ri._moveCause || (ri._hop > 0 ? 'other' : 'balance')) : null,
|
|
1353
|
+
newConversation: !pinned && Boolean(ri._newConversation) && ri._notedServed === account.name,
|
|
1354
|
+
|
|
1355
|
+
busyWaitMs: ri._busyWaitMs || 0,
|
|
1356
|
+
e2eFirstByteMs: !pinned && ri._servedAt && ri.startedAt ? Math.max(0, ri._servedAt - ri.startedAt) : null,
|
|
1357
|
+
};
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
/** Cache / move / bounce figures for one account over a window (getStatus + TUI only —
|
|
1361
|
+
* never on the scoring path). Additive sums plus derived hit share. */
|
|
1362
|
+
_efficiencySummary(account, windowMs, now = Date.now()) {
|
|
1363
|
+
return finishEfficiency(this._efficiencyAcc(account, windowMs, now), isClaudeAccount(account) ? 'claude' : 'glm');
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
_efficiencyAcc(account, windowMs, now = Date.now()) {
|
|
1367
|
+
this._pruneLoadEvents(account, now);
|
|
1368
|
+
const since = now - windowMs;
|
|
1369
|
+
const out = emptyEfficiency();
|
|
1370
|
+
for (const e of account.loadEvents || []) {
|
|
1371
|
+
if (e.at < since || e.countTokens) continue;
|
|
1372
|
+
if (e.bounced) {
|
|
1373
|
+
out.bounced++;
|
|
1374
|
+
if (e.refusal && e.refusal !== 'busy') out.refused++;
|
|
1375
|
+
continue;
|
|
1376
|
+
}
|
|
1377
|
+
if (!e.success) continue;
|
|
1378
|
+
out.served++;
|
|
1379
|
+
if (e.usage) { out.in += e.usage.in; out.out += e.usage.out; out.cr += e.usage.cr; out.cc += e.usage.cc; }
|
|
1380
|
+
if (e.moved) {
|
|
1381
|
+
out.moved++;
|
|
1382
|
+
const c = Object.hasOwn(out.movedByCause, e.moveCause) ? e.moveCause : 'other';
|
|
1383
|
+
out.movedByCause[c]++;
|
|
1384
|
+
if (e.busyWaitMs > 0) out.waitThenMoved++;
|
|
1385
|
+
}
|
|
1386
|
+
if (e.newConversation) out.newConversation++;
|
|
1387
|
+
if (e.busyWaitMs > 0) { out.waited++; out.waitMs += e.busyWaitMs; }
|
|
1388
|
+
if (e.e2eFirstByteMs != null) out.e2e.push(e.e2eFirstByteMs);
|
|
1389
|
+
}
|
|
1390
|
+
return out;
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
/** One cloud MIRROR row's figures, each field taken from the only pool that can know it
|
|
1394
|
+
* (review 2026-11-16, cluster 1):
|
|
1395
|
+
* - served / tokens / cache hit: the SERVING pool's export (it saw every client of that
|
|
1396
|
+
* account, teammates included);
|
|
1397
|
+
* - bounced / refused / waited / waitMs / waitThenMoved / moved / movedByCause /
|
|
1398
|
+
* newConversation, and first-byte latency: THIS pool's own events on the row. The
|
|
1399
|
+
* cloud books a pin refusal before any lease, never sees the edge's wait or its move
|
|
1400
|
+
* cause, and starts its clock after the edge's wait — and it no longer counts pinned
|
|
1401
|
+
* requests as its own moves (_servedTelemetry), so nothing here is counted twice.
|
|
1402
|
+
* An older cloud with no export: this pool's own events alone (it counts nothing). */
|
|
1403
|
+
_mirrorEfficiencyAcc(account, windowKey, windowMs, now = Date.now()) {
|
|
1404
|
+
const edge = this._efficiencyAcc(account, windowMs, now);
|
|
1405
|
+
const exp = account._remoteLoad?.[windowKey]?.efficiency;
|
|
1406
|
+
if (!exp) return edge;
|
|
1407
|
+
const acc = emptyEfficiency();
|
|
1408
|
+
for (const k of ['served', 'in', 'out', 'cr', 'cc']) acc[k] = Number(exp[k]) || 0;
|
|
1409
|
+
for (const k of EDGE_EFFICIENCY_FIELDS) acc[k] = (Number(exp[k]) || 0) + edge[k];
|
|
1410
|
+
for (const c of Object.keys(acc.movedByCause)) {
|
|
1411
|
+
acc.movedByCause[c] = (Number(exp.movedByCause?.[c]) || 0) + edge.movedByCause[c];
|
|
1412
|
+
}
|
|
1413
|
+
acc.e2e = edge.e2e;
|
|
1414
|
+
return acc;
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
/** The per-row summary the TUI and `maxpool status` show: a mirror row merges the
|
|
1418
|
+
* serving pool's export with this pool's edge-only fields; a local row is its own. */
|
|
1419
|
+
rowEfficiencySummary(account, windowKey = 'last15m', now = Date.now()) {
|
|
1420
|
+
const windowMs = efficiencyWindowMs(windowKey);
|
|
1421
|
+
const acc = account.remote ? this._mirrorEfficiencyAcc(account, windowKey, windowMs, now) : this._efficiencyAcc(account, windowMs, now);
|
|
1422
|
+
return finishEfficiency(acc, isClaudeAccount(account) ? 'claude' : 'glm');
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
/** Overall efficiency, per provider, for the TUI / `maxpool status`: this pool's own
|
|
1426
|
+
* rows plus each cloud mirror row (_mirrorEfficiencyAcc: cache/served from the cloud's
|
|
1427
|
+
* export, bounces/waits/moves/latency from this pool's own events — each counted once).
|
|
1428
|
+
* First-byte percentiles cover this pool's CLIENTS, mirror-served turns included, so
|
|
1429
|
+
* an edge busy wait shows up in p95. */
|
|
1430
|
+
efficiencyOverall(windowKey = 'last15m', now = Date.now(), { mirrors = true } = {}) {
|
|
1431
|
+
const windowMs = efficiencyWindowMs(windowKey);
|
|
1432
|
+
const by = { claude: emptyEfficiency(), glm: emptyEfficiency() };
|
|
1433
|
+
for (const a of this.accounts) {
|
|
1434
|
+
if (a.enabled === false || (a.remote && !mirrors)) continue;
|
|
1435
|
+
const tag = isClaudeAccount(a) ? 'claude' : 'glm';
|
|
1436
|
+
const e = a.remote ? this._mirrorEfficiencyAcc(a, windowKey, windowMs, now) : this._efficiencyAcc(a, windowMs, now);
|
|
1437
|
+
addEfficiency(by[tag], e);
|
|
1438
|
+
}
|
|
1439
|
+
const e2e = [...by.claude.e2e, ...by.glm.e2e];
|
|
1440
|
+
return { claude: finishEfficiency(by.claude, 'claude'), glm: finishEfficiency(by.glm, 'glm'),
|
|
1441
|
+
e2eP50: pct(e2e, 0.5), e2eP95: pct(e2e, 0.95) };
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
/** Final client outcome of one request at THIS pool. Edge-only: a request pinned to
|
|
1445
|
+
* us by a linked pool is that pool's client, never ours (its refusals would read as
|
|
1446
|
+
* our client errors). POST /v1/messages only, count_tokens skipped. */
|
|
1447
|
+
noteClientOutcome(requestInfo, status, now = Date.now()) {
|
|
1448
|
+
if (!requestInfo || requestInfo.pinAccount != null || requestInfo.isCountTokens) return;
|
|
1449
|
+
// The client cancelled (Esc / Ctrl-C) — no error was served to anyone.
|
|
1450
|
+
if (requestInfo._clientCancelled) return;
|
|
1451
|
+
if (requestInfo.method !== 'POST' || !/^\/v1\/messages(\?|$)/.test(requestInfo.path || '')) return;
|
|
1452
|
+
this.clientOutcomes.push({ at: now, status: Number(status) || 0 });
|
|
1453
|
+
const cutoff = now - CLIENT_OUTCOME_MAX_AGE_MS;
|
|
1454
|
+
while (this.clientOutcomes.length && this.clientOutcomes[0].at < cutoff) this.clientOutcomes.shift();
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
clientOutcomeSummary(windowMs = 15 * 60_000, now = Date.now()) {
|
|
1458
|
+
const ev = this.clientOutcomes.filter(e => e.at >= now - windowMs);
|
|
1459
|
+
const poolErrors = ev.filter(e => e.status >= 500 || e.status === 429).length;
|
|
1460
|
+
const requestErrors = ev.filter(e => e.status >= 400 && e.status < 500 && e.status !== 429).length;
|
|
1461
|
+
return { total: ev.length, poolErrors, requestErrors };
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1220
1464
|
_pruneLoadEvents(account, now = Date.now()) {
|
|
1221
1465
|
if (!account.loadEvents?.length) return;
|
|
1222
1466
|
const cutoff = now - LOAD_EVENT_MAX_AGE_MS;
|
|
@@ -1228,7 +1472,11 @@ export class AccountManager {
|
|
|
1228
1472
|
_loadSummary(account, windowMs, now = Date.now()) {
|
|
1229
1473
|
this._pruneLoadEvents(account, now);
|
|
1230
1474
|
const since = now - windowMs;
|
|
1231
|
-
const
|
|
1475
|
+
const all = (account.loadEvents || []).filter(e => e.at >= since);
|
|
1476
|
+
// A BOUNCE (pinned refusal, 2026-11-15) never served anything: out of every
|
|
1477
|
+
// denominator — requests, weight (spread share), avgMs — and out of `failed`.
|
|
1478
|
+
const bounced = all.filter(e => e.bounced).length;
|
|
1479
|
+
const events = all.filter(e => !e.bounced);
|
|
1232
1480
|
const requests = events.length;
|
|
1233
1481
|
// "failed" is what the operator reads as the ACCOUNT misbehaving; a link fault is
|
|
1234
1482
|
// the pool's own tunnel to the team pool and gets its own field (2026-11-12).
|
|
@@ -1243,6 +1491,7 @@ export class AccountManager {
|
|
|
1243
1491
|
requests,
|
|
1244
1492
|
failed,
|
|
1245
1493
|
linkFailed,
|
|
1494
|
+
bounced,
|
|
1246
1495
|
weight,
|
|
1247
1496
|
avgMs: timed.length ? Math.round(durationMs / timed.length) : null,
|
|
1248
1497
|
};
|
|
@@ -2679,7 +2928,7 @@ export class AccountManager {
|
|
|
2679
2928
|
// timeout the account is deprioritised for NEW placements only — a cold pick
|
|
2680
2929
|
// takes it last. A conversation it HOLDS (_holdsRequest) is not touched:
|
|
2681
2930
|
// evicting those is exactly what a request-scoped fault must never do.
|
|
2682
|
-
if (this._softCooling(account, now) && !this._holdsRequest(account, requestInfo)) {
|
|
2931
|
+
if ((this._softCooling(account, now) || this._pinBusyForNew(account, now)) && !this._holdsRequest(account, requestInfo)) {
|
|
2683
2932
|
if (!coolBest || priority < coolBest.priority || (priority === coolBest.priority && score < coolBest.score)) {
|
|
2684
2933
|
coolBest = { account, priority, score, critUnlock };
|
|
2685
2934
|
}
|
|
@@ -2951,23 +3200,103 @@ export class AccountManager {
|
|
|
2951
3200
|
return profiles.includes(profile);
|
|
2952
3201
|
}
|
|
2953
3202
|
|
|
3203
|
+
/** The in-flight depth at which a PINNED request is refused busy (the cloud's only
|
|
3204
|
+
* hard per-account cap for pinned traffic). One helper for the gate and the export. */
|
|
3205
|
+
_pinBusyLimit(_account) {
|
|
3206
|
+
return 2 * this.scheduler.perAccountConcurrencyTarget;
|
|
3207
|
+
}
|
|
3208
|
+
|
|
3209
|
+
/** A cloud row NEW placements should skip right now: it just refused busy
|
|
3210
|
+
* (pinBusyUntil), or its last known depth is one short of the cloud's pin limit.
|
|
3211
|
+
* Placement-scoped only — never in _isAvailable, so a holder still reaches it. */
|
|
3212
|
+
_pinBusyForNew(a, now = Date.now()) {
|
|
3213
|
+
if (this._recentlyRefusedBusy(a, now)) return true;
|
|
3214
|
+
const depth = this._remoteDepthEstimate(a, now);
|
|
3215
|
+
return Boolean(a.remote && Number.isFinite(a._remotePinLimit) && depth != null
|
|
3216
|
+
&& depth >= a._remotePinLimit - 1);
|
|
3217
|
+
}
|
|
3218
|
+
|
|
3219
|
+
/** A cloud row's in-flight depth NOW, as best this pool can tell (review 2026-11-16,
|
|
3220
|
+
* 2c). The poll snapshot (every ~30 s) counted our own leases at poll time; swap those
|
|
3221
|
+
* for our current leases. A refusal's depth header is fresher but describes one
|
|
3222
|
+
* instant, so it counts only within the busy window it came with (pinBusyUntil). */
|
|
3223
|
+
_remoteDepthEstimate(a, now = Date.now()) {
|
|
3224
|
+
if (Number.isFinite(a._refusalDepth) && a._refusalDepthUntil > now) return a._refusalDepth;
|
|
3225
|
+
if (!Number.isFinite(a._remoteInflightCount)) return null;
|
|
3226
|
+
return Math.max(0, a._remoteInflightCount - (a._remoteOwnAtSync || 0)) + (a.inFlight || 0);
|
|
3227
|
+
}
|
|
3228
|
+
|
|
3229
|
+
/** Only the TIME-CAPPED half of the busy signal: the row refused busy within its
|
|
3230
|
+
* 1-5 s pinBusyUntil window. Route-existence checks use this alone; the depth
|
|
3231
|
+
* snapshot (30 s poll) ranks placement but never denies a route. */
|
|
3232
|
+
_recentlyRefusedBusy(a, now = Date.now()) {
|
|
3233
|
+
return Boolean(a.pinBusyUntil && now < a.pinBusyUntil);
|
|
3234
|
+
}
|
|
3235
|
+
|
|
3236
|
+
/** A busy refusal on this row: repel new placements for `ms` (clamped 1-5 s). A
|
|
3237
|
+
* success does NOT clear it — the refusal said the row is at its pin limit. */
|
|
3238
|
+
markPinBusy(index, ms) {
|
|
3239
|
+
const a = this.accounts[index];
|
|
3240
|
+
if (!a) return;
|
|
3241
|
+
const d = Math.min(PIN_BUSY_MAX_MS, Math.max(PIN_BUSY_MIN_MS, Number(ms) || PIN_BUSY_MIN_MS));
|
|
3242
|
+
a.pinBusyUntil = Math.max(a.pinBusyUntil || 0, Date.now() + d);
|
|
3243
|
+
}
|
|
3244
|
+
|
|
3245
|
+
/** A throttled linked pool (its ANTHROPIC egress is held): cool every Claude-egress
|
|
3246
|
+
* mirror row of that link at once, so the request moves off the link in ONE hop.
|
|
3247
|
+
* Provider (GLM/Kimi) mirrors are NEVER cooled by a Claude throttle — the cloud
|
|
3248
|
+
* only refuses 'throttled' for account.type !== 'provider' (see _matchesRequest),
|
|
3249
|
+
* so those rows can still serve, and cooling them would black out GLM/'all'
|
|
3250
|
+
* routing for up to 30 s. */
|
|
3251
|
+
markLinkMirrorsCooldown(fromAccount, ms) {
|
|
3252
|
+
const linkName = fromAccount?.remote?.linkName;
|
|
3253
|
+
if (!linkName) return [];
|
|
3254
|
+
const until = Date.now() + Math.max(0, Number(ms) || 0);
|
|
3255
|
+
const cooled = this.accounts.filter(a => a.remote && a.remote.linkName === linkName && a.type !== 'provider');
|
|
3256
|
+
for (const a of cooled) a.cooldownUntil = Math.max(a.cooldownUntil || 0, until);
|
|
3257
|
+
return cooled;
|
|
3258
|
+
}
|
|
3259
|
+
|
|
3260
|
+
/** Fresher depth for a cloud row from a refusal's x-maxpool-pin-* headers. */
|
|
3261
|
+
noteRemoteDepth(name, inflight, limit) {
|
|
3262
|
+
const a = this.accounts.find(x => x.remote && x.name === name);
|
|
3263
|
+
if (!a) return;
|
|
3264
|
+
// Held only for the busy window it came with — never overwrites the poll snapshot,
|
|
3265
|
+
// so a refusal can't keep the row repelled until the next poll.
|
|
3266
|
+
if (Number.isFinite(inflight)) {
|
|
3267
|
+
a._refusalDepth = inflight;
|
|
3268
|
+
a._refusalDepthUntil = Date.now() + PIN_BUSY_MAX_MS;
|
|
3269
|
+
}
|
|
3270
|
+
if (Number.isFinite(limit)) a._remotePinLimit ??= limit;
|
|
3271
|
+
}
|
|
3272
|
+
|
|
2954
3273
|
_matchesRequest(account, profile, requestInfo = {}) {
|
|
2955
3274
|
if (this.admissionPaused) return false;
|
|
2956
3275
|
// A pinned request (from a linked pool) may ONLY run on the named account, and
|
|
2957
3276
|
// never on a mirror row (no chained forwarding between pools).
|
|
2958
3277
|
if (requestInfo.pinAccount != null) {
|
|
2959
3278
|
if (account.remote || account.name !== requestInfo.pinAccount) return false;
|
|
3279
|
+
// P3: a pinned request isn't competing in this pool's queue — it skips the
|
|
3280
|
+
// queue-fairness gate (that alone 503'd every pinned Claude request). The
|
|
3281
|
+
// upstream throttle STAYS: it describes this pool's real egress to Anthropic.
|
|
3282
|
+
if (!this._isRequestCompatible(account, profile, requestInfo)) return false;
|
|
3283
|
+
if (account.type !== 'provider' && this._isUpstreamThrottleBlocking()) {
|
|
3284
|
+
requestInfo.pinRefusal = 'throttled';
|
|
3285
|
+
return false;
|
|
3286
|
+
}
|
|
2960
3287
|
// P1: the pin bypasses this pool's balancer, so keep its anti-dogpile floor
|
|
2961
3288
|
// here — refuse past 2x the soft per-account target; the caller fails over.
|
|
2962
|
-
|
|
3289
|
+
// 'busy' tells the edge "servable once a slot frees — wait for it", so it is said
|
|
3290
|
+
// ONLY for an account that is otherwise available (review 2026-11-16, 2d). An
|
|
3291
|
+
// unavailable one (rate-limited, cooling, exhausted) matches here and is rejected
|
|
3292
|
+
// by the selection loop's _isAvailable, so the refusal reads 'unavailable' and the
|
|
3293
|
+
// edge moves at once instead of waiting out its budget on it.
|
|
3294
|
+
if ((account.inFlight || 0) >= this._pinBusyLimit(account)
|
|
3295
|
+
&& this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true, model: requestInfo.model })) {
|
|
2963
3296
|
requestInfo.pinRefusedBusy = true;
|
|
3297
|
+
requestInfo.pinRefusal = 'busy';
|
|
2964
3298
|
return false;
|
|
2965
3299
|
}
|
|
2966
|
-
// P3: a pinned request isn't competing in this pool's queue — it skips the
|
|
2967
|
-
// queue-fairness gate (that alone 503'd every pinned Claude request). The
|
|
2968
|
-
// upstream throttle STAYS: it describes this pool's real egress to Anthropic.
|
|
2969
|
-
if (!this._isRequestCompatible(account, profile, requestInfo)) return false;
|
|
2970
|
-
if (account.type !== 'provider' && this._isUpstreamThrottleBlocking()) return false;
|
|
2971
3300
|
return true;
|
|
2972
3301
|
}
|
|
2973
3302
|
// Duplicate guard: a local account whose identity also lives in a team pool
|
|
@@ -3464,8 +3793,10 @@ export class AccountManager {
|
|
|
3464
3793
|
* roomy peers (a peer serving only 'all' takes none of its 'claude' requests).
|
|
3465
3794
|
* A self that serves no profile has no confirmed room (0).
|
|
3466
3795
|
* - routable NOW: enabled, not throttled/error/exhausted, not pool-benched
|
|
3467
|
-
* (
|
|
3468
|
-
* not auth/subscription-dead.
|
|
3796
|
+
* (a benched local copy is counted through its cloud mirror, never twice),
|
|
3797
|
+
* not auth/subscription-dead. A cloud MIRROR row counts like a local one,
|
|
3798
|
+
* judged on the cloud's exported state; its freshness is the cloud's own
|
|
3799
|
+
* quota.lastProbeOkAt (an older cloud that sends none = unknown = not counted).
|
|
3469
3800
|
* - weekly util (unified7d) leaves ≥ POLICY.holdPeerWeeklyRoom to the peer's
|
|
3470
3801
|
* weekly ceiling — its effective usage cap, else 1.0 (uncapped: below the 0.65
|
|
3471
3802
|
* `soft` tier the router already spreads load at). Red-team #4: room is
|
|
@@ -3474,8 +3805,9 @@ export class AccountManager {
|
|
|
3474
3805
|
* session bench (_sessionBenchThreshold: its 5h cap, else the 0.90 switch
|
|
3475
3806
|
* threshold) — a session-benched account absorbs nothing.
|
|
3476
3807
|
* - a DIFFERENT identity (accountUuid/credential fingerprint) — not this same
|
|
3477
|
-
* account under a second name; peers sharing one
|
|
3478
|
-
* names for one subscription) count ONCE
|
|
3808
|
+
* account under a second name; peers sharing one capacity group (two local
|
|
3809
|
+
* names, or a mirror and its org-less twin, for one subscription) count ONCE
|
|
3810
|
+
* (red-team #6).
|
|
3479
3811
|
* - every ACTIVE model bucket the peer reports also has ≥ holdPeerWeeklyRoom
|
|
3480
3812
|
* left (red-team #1) — the walled account serves no model at all, so a peer
|
|
3481
3813
|
* whose Fable is exhausted/full (or 429-marked, or unreadable) would leave
|
|
@@ -3488,18 +3820,27 @@ export class AccountManager {
|
|
|
3488
3820
|
* Returns a plain integer (the minimum over self's profiles); the POLICY decides
|
|
3489
3821
|
* sufficiency (holdMinPeers × holders). */
|
|
3490
3822
|
countPeersWithRoom(self, now = Date.now()) {
|
|
3491
|
-
|
|
3823
|
+
// Peers are deduped by capacity GROUP, not identity string: an org-less twin
|
|
3824
|
+
// of the same subscription (uuid:U vs uuid:U:org) groups with self and must
|
|
3825
|
+
// never count as a second roomy member of the pool.
|
|
3826
|
+
const selfGroup = new Set(this.capacityGroupOf(self));
|
|
3492
3827
|
const selfProfiles = self?.profiles || ['claude', 'all'];
|
|
3493
3828
|
const roomLeft = (util, ceiling, room) => Number.isFinite(util) && util <= ceiling - room;
|
|
3494
3829
|
const roomy = [];
|
|
3495
3830
|
for (const p of this.accounts) {
|
|
3496
3831
|
const id = this.capacityIdentity(p);
|
|
3497
|
-
if (id == null ||
|
|
3832
|
+
if (id == null || selfGroup.has(p)) continue;
|
|
3498
3833
|
if (p.type !== 'oauth' || p.enabled === false) continue;
|
|
3499
|
-
|
|
3834
|
+
// A benched local copy is skipped; its identity is counted through the cloud
|
|
3835
|
+
// MIRROR of the same account (one peer per capacity group, below).
|
|
3836
|
+
if (p.poolBenchedBy) continue;
|
|
3500
3837
|
if (p.refreshDead || p.subscriptionGone) continue;
|
|
3501
3838
|
if (p.status === 'throttled' || p.status === 'error' || p.status === 'exhausted') continue;
|
|
3502
3839
|
if (p.cooldownUntil && p.cooldownUntil > now) continue;
|
|
3840
|
+
// A cloud MIRROR carries the cloud's own exported state (status, refreshDead,
|
|
3841
|
+
// quota incl. the cloud's lastProbeOkAt, cap, cap-lift) — judged below by the
|
|
3842
|
+
// same gates as a local row. Its cooldown is the cloud's, copied separately.
|
|
3843
|
+
if (p.remote && p.remoteCooldownUntil && p.remoteCooldownUntil > now) continue;
|
|
3503
3844
|
const q = p.quota || {};
|
|
3504
3845
|
// Probe-fresh only: a peer whose reading predates this sweep's decision is
|
|
3505
3846
|
// unknown capacity, and unknown capacity must not hold a claim.
|
|
@@ -3510,11 +3851,13 @@ export class AccountManager {
|
|
|
3510
3851
|
if (!roomLeft(q.unified5h, this._sessionBenchThreshold(p, now), POLICY.holdPeerSessionRoom)) continue;
|
|
3511
3852
|
const buckets = Object.values(q.scopedWeekly || {}).filter(e => e && e.isActive !== false);
|
|
3512
3853
|
if (buckets.some(e => !roomLeft(e.utilization, 1, POLICY.holdPeerWeeklyRoom))) continue;
|
|
3513
|
-
|
|
3854
|
+
// One peer per capacity GROUP (lowest member index as its key): an org-less twin
|
|
3855
|
+
// of the same login, or a mirror + a second mirror of it, is one subscription.
|
|
3856
|
+
roomy.push({ p, key: Math.min(...this.capacityGroupOf(p).map(x => x.index)) });
|
|
3514
3857
|
}
|
|
3515
3858
|
if (!selfProfiles.length) return 0;
|
|
3516
3859
|
return Math.min(...selfProfiles.map(pr =>
|
|
3517
|
-
new Set(roomy.filter(({ p }) => this._matchesProfile(p, pr)).map(({
|
|
3860
|
+
new Set(roomy.filter(({ p }) => this._matchesProfile(p, pr)).map(({ key }) => key)).size));
|
|
3518
3861
|
}
|
|
3519
3862
|
|
|
3520
3863
|
// ── R7 hold register (red-team r2 #1/#2) ──────────────────────────────────
|
|
@@ -3550,7 +3893,15 @@ export class AccountManager {
|
|
|
3550
3893
|
/** The walls a hold must budget for: every OTHER account holding now, plus
|
|
3551
3894
|
* `self` (the wall being decided). Feeds wallClaimHeld's holdersInPool. */
|
|
3552
3895
|
countHoldingWalls(self, now = Date.now()) {
|
|
3553
|
-
|
|
3896
|
+
// Cloud mirrors are peers in countPeersWithRoom, so the cloud's OWN holds draw on the
|
|
3897
|
+
// same spare capacity and count as holders here (review 2026-11-16, cluster 6) —
|
|
3898
|
+
// else two pools each budget the same roomy cloud accounts. A mirror of self's own
|
|
3899
|
+
// capacity group is self, never a second holder. An older cloud exports no
|
|
3900
|
+
// wallHolding, so it adds none (the previous behaviour).
|
|
3901
|
+
const selfGroup = new Set(this.capacityGroupOf(self).map(x => x.index));
|
|
3902
|
+
const remoteHolders = this.accounts.filter(a => a !== self && a.remote && a.enabled !== false
|
|
3903
|
+
&& a._remoteWallHolding === true && !selfGroup.has(a.index)).length;
|
|
3904
|
+
return 1 + this.accounts.filter(a => a !== self && this.isHoldingWall(a, now)).length + remoteHolders;
|
|
3554
3905
|
}
|
|
3555
3906
|
|
|
3556
3907
|
/**
|
|
@@ -3602,7 +3953,10 @@ export class AccountManager {
|
|
|
3602
3953
|
// A cloud mirror's depth includes OTHER teammates' live load on that same cloud
|
|
3603
3954
|
// account (mirrored from the link poll), so every routing term below — the
|
|
3604
3955
|
// unchanged balancing logic — sees it as busy as it really is.
|
|
3605
|
-
|
|
3956
|
+
// Other teammates' load in WEIGHT units (the cloud's activeWeight minus ours, 2026-11-15);
|
|
3957
|
+
// an older cloud that omits activeWeight falls back to its in-flight count.
|
|
3958
|
+
const remoteLoadW = Number.isFinite(account._remoteOtherWeight) ? account._remoteOtherWeight : (account._remoteOtherLoad || 0);
|
|
3959
|
+
const inflight = account.activeWeight + reqWeight + remoteLoadW;
|
|
3606
3960
|
const refillMult = this._fastRefillMultiplier(account);
|
|
3607
3961
|
|
|
3608
3962
|
// DOMINANT term: in-flight concurrency. Short-term throttling is driven by
|
|
@@ -3665,7 +4019,13 @@ export class AccountManager {
|
|
|
3665
4019
|
// benching, but wrong for load balancing: an account at 80% should be clearly less
|
|
3666
4020
|
// attractive than one at 10% even if both are "on pace". Measured 2026-08-10: cc at
|
|
3667
4021
|
// 80% scored 52.30 vs glm at 10% at 52.15 — a 0.15 gap drowned by round-robin.
|
|
3668
|
-
|
|
4022
|
+
// + the absolute weekly headroom tie-break (ABS_UTIL_WEIGHT), behind the same
|
|
4023
|
+
// weekly-aware flag so the 5h-only mode stays byte-identical.
|
|
4024
|
+
// A FULL lift refilled the week: like every other weekly term, the tie-break must
|
|
4025
|
+
// not price it on its pre-refill usage (else a lifted 0.97 is worst-of-pool again).
|
|
4026
|
+
const absUsed = this.scheduler.weeklyAwareScoring !== false && !this._weeklyRefilledByLift(account, now)
|
|
4027
|
+
? this._absWeeklyUsed(account) : 0;
|
|
4028
|
+
const utilizationCost = (this._rawUtilization(account) + ABS_UTIL_WEIGHT * absUsed) * this.scheduler.utilizationWeight * refillMult * resetMult;
|
|
3669
4029
|
|
|
3670
4030
|
// Per-model weekly de-preference: an account whose scoped weekly for THIS
|
|
3671
4031
|
// request's model (e.g. Fable) is high-but-not-exhausted is a poor pick for
|
|
@@ -3679,7 +4039,8 @@ export class AccountManager {
|
|
|
3679
4039
|
|
|
3680
4040
|
const fleetRecentWeight = ctx?.fleetRecentWeight ?? 0;
|
|
3681
4041
|
const recentWeight = this._loadSummary(account, this.scheduler.spreadWindowMs, now).weight;
|
|
3682
|
-
|
|
4042
|
+
// Clamped: a provider is outside the denominator, so its raw share can exceed 1.
|
|
4043
|
+
const share = fleetRecentWeight > 0 ? Math.min(1, recentWeight / fleetRecentWeight) : 0;
|
|
3683
4044
|
// SPREAD also carries the fast-refill discount (2026-08-29, owner option 1). The
|
|
3684
4045
|
// spread term is the equaliser that pulls every account back to fleet parity — at
|
|
3685
4046
|
// converged equal share it is ~10x larger than the utilization+pace gap the
|
|
@@ -3736,8 +4097,9 @@ export class AccountManager {
|
|
|
3736
4097
|
* Applies in every scored mode (balance, prefer-*); never in sticky (see below).
|
|
3737
4098
|
*/
|
|
3738
4099
|
_noteConversationAccount(requestInfo, account) {
|
|
3739
|
-
|
|
3740
|
-
|
|
4100
|
+
// conversationLastAccount is written at the first real upstream byte
|
|
4101
|
+
// (noteConversationServed), never here: a refused/failed hop must not re-point the
|
|
4102
|
+
// conversation (2026-11-15).
|
|
3741
4103
|
// Home memory: ONE key per conversation, independent of identity health, so the
|
|
3742
4104
|
// degraded <-> ok flip of _affinityKey can neither hide a home nor lend it to the
|
|
3743
4105
|
// sibling conversations that share the terminal key. The entry carries its own
|
|
@@ -3754,6 +4116,39 @@ export class AccountManager {
|
|
|
3754
4116
|
}
|
|
3755
4117
|
}
|
|
3756
4118
|
|
|
4119
|
+
/** The conversation was SERVED by `account` (first real upstream byte). Idempotent
|
|
4120
|
+
* per request. Records whether this turn moved the conversation (for telemetry). */
|
|
4121
|
+
noteConversationServed(requestInfo, account) {
|
|
4122
|
+
if (!requestInfo || !account || requestInfo._notedServed === account.name) return;
|
|
4123
|
+
// count_tokens carries no prompt cache: it must not re-point the conversation's
|
|
4124
|
+
// account memory (holdsWarmConversation / move telemetry read it).
|
|
4125
|
+
if (requestInfo.isCountTokens) return;
|
|
4126
|
+
requestInfo._notedServed = account.name;
|
|
4127
|
+
requestInfo._servedAt = Date.now();
|
|
4128
|
+
const key = this._affinityKey(requestInfo);
|
|
4129
|
+
if (!key) return;
|
|
4130
|
+
const prev = this.conversationLastAccount.get(key);
|
|
4131
|
+
requestInfo._newConversation = prev == null;
|
|
4132
|
+
requestInfo._servedMoved = prev != null && prev !== account.name;
|
|
4133
|
+
lruSet(this.conversationLastAccount, key, account.name);
|
|
4134
|
+
lruSet(this.conversationLastServedAt, key, Date.now());
|
|
4135
|
+
}
|
|
4136
|
+
|
|
4137
|
+
/** Does `account` hold this conversation's WARM Claude cache — it served the last
|
|
4138
|
+
* turn, within the cache window? (The sticky busy wait is only worth it then.) */
|
|
4139
|
+
holdsWarmConversation(account, requestInfo = {}) {
|
|
4140
|
+
if (!isClaudeAccount(account)) return false;
|
|
4141
|
+
const key = this._affinityKey(requestInfo);
|
|
4142
|
+
if (!key || this.conversationLastAccount.get(key) !== account.name) return false;
|
|
4143
|
+
return Date.now() - (this.conversationLastServedAt.get(key) || 0) <= this._cacheAffinityWindowMs();
|
|
4144
|
+
}
|
|
4145
|
+
|
|
4146
|
+
/** cacheAffinityReturnWindowMs, falling back to the default when unset/invalid. */
|
|
4147
|
+
_cacheAffinityWindowMs() {
|
|
4148
|
+
const w = Number(this.scheduler.cacheAffinityReturnWindowMs);
|
|
4149
|
+
return Number.isFinite(w) && w >= 0 ? w : DEFAULT_SCHEDULER.cacheAffinityReturnWindowMs;
|
|
4150
|
+
}
|
|
4151
|
+
|
|
3757
4152
|
/** Key of the conversation's home entry: the conversation itself, never the shared
|
|
3758
4153
|
* terminal key when a conversation id exists. When identity is genuinely broken
|
|
3759
4154
|
* (a fresh hash every request) the home is simply never found, so the normal pick
|
|
@@ -3775,9 +4170,7 @@ export class AccountManager {
|
|
|
3775
4170
|
const home = this.conversationLastClaudeAccount.get(homeKey);
|
|
3776
4171
|
if (!home || !home.detoured) return null;
|
|
3777
4172
|
const ageMs = now - home.at;
|
|
3778
|
-
|
|
3779
|
-
if (!Number.isFinite(windowMs) || windowMs < 0) windowMs = DEFAULT_SCHEDULER.cacheAffinityReturnWindowMs;
|
|
3780
|
-
return { name: home.name, stale: ageMs > windowMs, ageMs };
|
|
4173
|
+
return { name: home.name, stale: ageMs > this._cacheAffinityWindowMs(), ageMs };
|
|
3781
4174
|
}
|
|
3782
4175
|
|
|
3783
4176
|
/** Conversation identity: terminal + prefix hash when the server computed one;
|
|
@@ -4042,6 +4435,13 @@ export class AccountManager {
|
|
|
4042
4435
|
return util;
|
|
4043
4436
|
}
|
|
4044
4437
|
|
|
4438
|
+
/** Absolute weekly used fraction (0..1), not pace-adjusted — the tie-break input. */
|
|
4439
|
+
_absWeeklyUsed(account) {
|
|
4440
|
+
const q = account?.quota;
|
|
4441
|
+
if (!q) return 0;
|
|
4442
|
+
return Math.max(q.unified7d != null ? clamp01(q.unified7d) : 0, q.providerWk != null ? clamp01(q.providerWk) : 0);
|
|
4443
|
+
}
|
|
4444
|
+
|
|
4045
4445
|
_windowScarcity(util, resetMs, windowLen, now = Date.now()) {
|
|
4046
4446
|
const used = clamp01(util);
|
|
4047
4447
|
if (!resetMs || resetMs <= now) return used; // unknown / just-reset → face value
|
|
@@ -4964,7 +5364,7 @@ export class AccountManager {
|
|
|
4964
5364
|
* fingerprint (same key shape). null = unresolvable (never merged by name). */
|
|
4965
5365
|
capacityIdentity(a) {
|
|
4966
5366
|
if (!a) return null;
|
|
4967
|
-
if (a.remote) return remoteAccountIdentity({ type: a.type, provider: a.provider, accountUuid: a.remoteUuid, keyFingerprint: a.remoteKeyFp });
|
|
5367
|
+
if (a.remote) return remoteAccountIdentity({ type: a.type, provider: a.provider, accountUuid: a.remoteUuid, organizationUuid: a.remoteOrganizationUuid, keyFingerprint: a.remoteKeyFp });
|
|
4968
5368
|
return accountIdentity(a);
|
|
4969
5369
|
}
|
|
4970
5370
|
|
|
@@ -4972,7 +5372,19 @@ export class AccountManager {
|
|
|
4972
5372
|
capacityGroupOf(a) {
|
|
4973
5373
|
const id = this.capacityIdentity(a);
|
|
4974
5374
|
if (!id) return [a];
|
|
4975
|
-
|
|
5375
|
+
const same = this.accounts.filter(x => this.capacityIdentity(x) === id);
|
|
5376
|
+
// `uuid:<user>[:<org>]`: a row whose organization is unknown (saved before orgs
|
|
5377
|
+
// were recorded, or mirrored from an older pool) is still the same login as the
|
|
5378
|
+
// user's org-bearing row while that user has ONE known organization here. Two
|
|
5379
|
+
// known organizations are two subscriptions: then only exact ids group.
|
|
5380
|
+
if (!id.startsWith('uuid:')) return same;
|
|
5381
|
+
const user = id.split(':')[1];
|
|
5382
|
+
const rows = this.accounts.filter(x => {
|
|
5383
|
+
const xid = this.capacityIdentity(x);
|
|
5384
|
+
return xid && xid.startsWith('uuid:') && xid.split(':')[1] === user;
|
|
5385
|
+
});
|
|
5386
|
+
const orgs = new Set(rows.map(x => this.capacityIdentity(x).split(':')[2]).filter(Boolean));
|
|
5387
|
+
return orgs.size > 1 ? same : rows;
|
|
4976
5388
|
}
|
|
4977
5389
|
|
|
4978
5390
|
/** Merged closed-cycle history of an account's whole identity: this pool's ledger
|
|
@@ -5532,6 +5944,7 @@ export class AccountManager {
|
|
|
5532
5944
|
type: acctData.type,
|
|
5533
5945
|
provider: acctData.provider || (acctData.type === 'provider' ? 'provider' : 'anthropic'),
|
|
5534
5946
|
accountUuid: acctData.accountUuid || null,
|
|
5947
|
+
organizationUuid: acctData.organizationUuid || null,
|
|
5535
5948
|
credential: acctData.accessToken || acctData.authToken || acctData.apiKey,
|
|
5536
5949
|
upstream: acctData.upstream || null,
|
|
5537
5950
|
authHeader: acctData.authHeader || null,
|
|
@@ -5793,11 +6206,17 @@ export class AccountManager {
|
|
|
5793
6206
|
return true;
|
|
5794
6207
|
}
|
|
5795
6208
|
|
|
5796
|
-
|
|
5797
|
-
|
|
6209
|
+
/**
|
|
6210
|
+
* Match a saved state entry to a live account by stable identity: the account
|
|
6211
|
+
* UUID + organization UUID when both have a UUID, otherwise the name. A saved
|
|
6212
|
+
* entry without an organization (written before organizations were recorded)
|
|
6213
|
+
* also matches the same user under the same name, so quota survives the first
|
|
6214
|
+
* restart after an account learns its organization.
|
|
6215
|
+
*/
|
|
5798
6216
|
_sameIdentity(saved, account) {
|
|
5799
|
-
if (saved.accountUuid
|
|
5800
|
-
return saved
|
|
6217
|
+
if (!saved.accountUuid || !account.accountUuid) return saved.name === account.name;
|
|
6218
|
+
return sameAccountIdentity(saved, account)
|
|
6219
|
+
|| (!saved.organizationUuid && sameUser(saved, account) && saved.name === account.name);
|
|
5801
6220
|
}
|
|
5802
6221
|
|
|
5803
6222
|
/**
|
|
@@ -5808,7 +6227,7 @@ export class AccountManager {
|
|
|
5808
6227
|
return this.accounts.filter(a => !a.remote).map(a => {
|
|
5809
6228
|
const quota = {};
|
|
5810
6229
|
for (const f of PERSISTED_QUOTA_FIELDS) quota[f] = a.quota[f];
|
|
5811
|
-
const out = { accountUuid: a.accountUuid, name: a.name, quota };
|
|
6230
|
+
const out = { accountUuid: a.accountUuid, organizationUuid: a.organizationUuid, name: a.name, quota };
|
|
5812
6231
|
// OUTSIDE-USAGE flag survives restart (red-team R1.2): a restart is exactly
|
|
5813
6232
|
// when a leak's cards are most at risk — remediation rotates keys and
|
|
5814
6233
|
// restarts the pool, and the in-memory flag would vanish mid-remediation.
|
|
@@ -6018,10 +6437,14 @@ export class AccountManager {
|
|
|
6018
6437
|
// persistence, re-auth, disk sync) can never resolve to a mirror row.
|
|
6019
6438
|
a.accountUuid = null;
|
|
6020
6439
|
a.remoteUuid = ra.accountUuid || null;
|
|
6440
|
+
a.remoteOrganizationUuid = ra.organizationUuid || null;
|
|
6021
6441
|
a.provider = ra.provider || a.provider;
|
|
6022
6442
|
a.quota = { ...emptyQuota(), ...(ra.quota || {}) };
|
|
6023
6443
|
a.status = ra.status || 'active';
|
|
6024
6444
|
a.refreshDead = Boolean(ra.refreshDead);
|
|
6445
|
+
// The CLOUD's cooldown (additive export; an older cloud omits it = none). Kept off
|
|
6446
|
+
// `cooldownUntil`, which is this process's own routing gate for the mirror row.
|
|
6447
|
+
{ const cd = ra.cooldownUntil ? Date.parse(ra.cooldownUntil) : NaN; a.remoteCooldownUntil = Number.isFinite(cd) ? cd : null; }
|
|
6025
6448
|
a.lastError = ra.lastError || null;
|
|
6026
6449
|
a.rateLimitedUntil = ra.rateLimitedUntil ? Date.parse(ra.rateLimitedUntil) : null;
|
|
6027
6450
|
a.capUtilization = ra.capUtilization ?? null;
|
|
@@ -6046,7 +6469,16 @@ export class AccountManager {
|
|
|
6046
6469
|
// Other teammates' live load on this cloud account (remote inFlight minus our
|
|
6047
6470
|
// own leases) — so the balancer sees the account as busy as it really is.
|
|
6048
6471
|
a._remoteOtherLoad = Math.max(0, (ra.inFlight || 0) - (a.inFlight || 0));
|
|
6472
|
+
// The same in WEIGHT units (NaN → fallback when an older cloud omits activeWeight).
|
|
6473
|
+
a._remoteOtherWeight = Number.isFinite(ra.activeWeight) ? Math.max(0, ra.activeWeight - (a.activeWeight || 0)) : NaN;
|
|
6474
|
+
a._remoteInflightCount = Number.isFinite(ra.inFlight) ? ra.inFlight : null;
|
|
6475
|
+
// Our own leases on this row at poll time (already inside ra.inFlight): the depth
|
|
6476
|
+
// estimate swaps them for our CURRENT leases, so our finished requests stop
|
|
6477
|
+
// counting against the row until the next poll (review 2026-11-16, 2c).
|
|
6478
|
+
a._remoteOwnAtSync = a.inFlight || 0;
|
|
6479
|
+
a._remotePinLimit = Number.isFinite(ra.pinBusyAt) ? ra.pinBusyAt : null;
|
|
6049
6480
|
a._remoteLoad = ra.load || null;
|
|
6481
|
+
a._remoteWallHolding = ra.wallHolding === true;
|
|
6050
6482
|
a.probing = false;
|
|
6051
6483
|
a.enabled = true;
|
|
6052
6484
|
}
|
|
@@ -6279,6 +6711,11 @@ export class AccountManager {
|
|
|
6279
6711
|
since: this.identityHealth.since,
|
|
6280
6712
|
},
|
|
6281
6713
|
currentAccount: this.accounts[this.currentIndex]?.name,
|
|
6714
|
+
// Overall cache / move / bounce / wait figures — the SAME merge the TUI Cache line
|
|
6715
|
+
// uses (mirror rows: cache + served from the serving pool's export, edge-only
|
|
6716
|
+
// fields from this pool's own events), plus edge client outcomes.
|
|
6717
|
+
efficiency: { last15m: this.efficiencyOverall('last15m', now), last1h: this.efficiencyOverall('last1h', now) },
|
|
6718
|
+
clientOutcomes: { last15m: this.clientOutcomeSummary(15 * 60_000, now), last1h: this.clientOutcomeSummary(60 * 60_000, now) },
|
|
6282
6719
|
switchThreshold: this.switchThreshold,
|
|
6283
6720
|
routing: {
|
|
6284
6721
|
mode: this.routingMode,
|
|
@@ -6332,6 +6769,7 @@ export class AccountManager {
|
|
|
6332
6769
|
// fingerprint. Without these a linked pool can't see a duplicate Claude login
|
|
6333
6770
|
// (live-rehearsal 2026-10-29: status omitted accountUuid → no bench).
|
|
6334
6771
|
accountUuid: a.type === 'oauth' ? (a.accountUuid || null) : null,
|
|
6772
|
+
organizationUuid: a.type === 'oauth' ? (a.organizationUuid || null) : null,
|
|
6335
6773
|
keyFingerprint: (a.type === 'provider' && a.credential) ? credentialFp(a.credential) : null,
|
|
6336
6774
|
// Plan config (display-only) + this pool's closed cycles per window so a linked
|
|
6337
6775
|
// pool can UNION history instead of picking one ledger (bounded: 12 weekly /
|
|
@@ -6361,15 +6799,29 @@ export class AccountManager {
|
|
|
6361
6799
|
inFlight: a.inFlight,
|
|
6362
6800
|
activeWeight: a.activeWeight,
|
|
6363
6801
|
},
|
|
6364
|
-
last15m: this._loadSummary(a, 15 * 60 * 1000, now),
|
|
6365
|
-
last1h: this._loadSummary(a, 60 * 60 * 1000, now),
|
|
6802
|
+
last15m: { ...this._loadSummary(a, 15 * 60 * 1000, now), efficiency: this.rowEfficiencySummary(a, 'last15m', now) },
|
|
6803
|
+
last1h: { ...this._loadSummary(a, 60 * 60 * 1000, now), efficiency: this.rowEfficiencySummary(a, 'last1h', now) },
|
|
6366
6804
|
},
|
|
6805
|
+
// The cloud's pin limit for this row (linked pools skip it one short of this).
|
|
6806
|
+
pinBusyAt: a.remote ? null : this._pinBusyLimit(a),
|
|
6807
|
+
// A cloud row's cache figures as the SERVING pool exported them (team-wide on
|
|
6808
|
+
// that account); null on an older cloud or a local row.
|
|
6809
|
+
teamEfficiency: a.remote ? {
|
|
6810
|
+
last15m: a._remoteLoad?.last15m?.efficiency ?? null,
|
|
6811
|
+
last1h: a._remoteLoad?.last1h?.efficiency ?? null,
|
|
6812
|
+
} : null,
|
|
6367
6813
|
lastError: a.lastError,
|
|
6368
6814
|
lastErrorAt: a.lastErrorAt ? new Date(a.lastErrorAt).toISOString() : null,
|
|
6369
6815
|
cooldownUntil: Math.max(a.cooldownUntil || 0, a.provisionalUpstreamUntil || 0)
|
|
6370
6816
|
? new Date(Math.max(a.cooldownUntil || 0, a.provisionalUpstreamUntil || 0)).toISOString()
|
|
6371
6817
|
: null,
|
|
6372
|
-
|
|
6818
|
+
// A BENCHED duplicate is never probed or refreshed locally, so its own quota is
|
|
6819
|
+
// stale by design — don't export it as current. The cloud pool owns the live
|
|
6820
|
+
// numbers (the TUI overlays them via remoteRowFor); consumers read this marker.
|
|
6821
|
+
quota: a.poolBenchedBy ? null : { ...a.quota },
|
|
6822
|
+
quotaSource: a.poolBenchedBy ? 'cloud-mirror' : 'local',
|
|
6823
|
+
// A linked pool counts this row's hold against the spare capacity it shares.
|
|
6824
|
+
wallHolding: a.remote ? false : this.isHoldingWall(a, now),
|
|
6373
6825
|
outsideUsage: a.outsideUsage ? { ...a.outsideUsage } : null,
|
|
6374
6826
|
weekly: {
|
|
6375
6827
|
state: this._weeklyState(a),
|