maxpool 1.24.99 → 1.24.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +307 -39
- package/src/capacity-ledger.js +4 -1
- package/src/index.js +8 -4
- package/src/server.js +50 -10
- package/src/thread-gate.js +68 -7
- package/src/tui.js +41 -17
package/package.json
CHANGED
package/src/account-manager.js
CHANGED
|
@@ -230,6 +230,13 @@ const DEFAULT_SCHEDULER = {
|
|
|
230
230
|
// its whole prompt cache). Edited live from the TUI routing screen (`b`); env
|
|
231
231
|
// MAXPOOL_STICKY_BUSY_BUDGET_MS overrides it when set. 0 = never wait.
|
|
232
232
|
stickyBusyBudgetMs: 60_000,
|
|
233
|
+
// REPEAT COLD-HOP HOLD (owner 2026-11-17): a long conversation that already paid one
|
|
234
|
+
// cold Claude re-write after a GLM turn, and is back on GLM, waits up to this long for
|
|
235
|
+
// a GLM row instead of paying a second cold re-write on Claude (audit 2026-11-16: 55-58
|
|
236
|
+
// conversations paid a >50k write more than once, 36M tokens in ~5h). Streaming only
|
|
237
|
+
// (the SSE heartbeat keeps the client alive); off when no GLM row can serve soon.
|
|
238
|
+
// 0 = never hold.
|
|
239
|
+
coldClaudeRepeatHoldMs: 120_000,
|
|
233
240
|
concurrencyWeight: 2, // multiplies in-flight load (activeWeight+reqWeight) — dominant
|
|
234
241
|
perAccountConcurrencyTarget: 3, // D: soft per-account in-flight target; past it, capPenalty bites
|
|
235
242
|
capPenaltyWeight: 10, // steep penalty per unit of in-flight depth past D (throttle safety floor)
|
|
@@ -415,15 +422,19 @@ const EDGE_EFFICIENCY_FIELDS = ['bounced', 'refused', 'waited', 'waitMs', 'waitT
|
|
|
415
422
|
const efficiencyWindowMs = (windowKey) => (windowKey === 'last1h' ? 60 * 60_000 : 15 * 60_000);
|
|
416
423
|
|
|
417
424
|
function emptyEfficiency() {
|
|
418
|
-
return { served: 0, in: 0, out: 0, cr: 0, cc: 0, moved: 0,
|
|
425
|
+
return { served: 0, in: 0, out: 0, cr: 0, cc: 0, moved: 0, movedSame: 0, movedCross: 0,
|
|
419
426
|
movedByCause: { busy: 0, wall: 0, balance: 0, link: 0, other: 0 },
|
|
420
427
|
newConversation: 0, bounced: 0, refused: 0, waited: 0, waitMs: 0, waitThenMoved: 0, e2e: [] };
|
|
421
428
|
}
|
|
422
|
-
/** Add one exported/summarized efficiency block into an accumulator (missing fields = 0).
|
|
429
|
+
/** Add one exported/summarized efficiency block into an accumulator (missing fields = 0).
|
|
430
|
+
* Old blocks without movedSame/movedCross keep their moved total (shown as same-family
|
|
431
|
+
* — the cheap side, so a mixed fleet under-reports cross moves rather than inventing them). */
|
|
423
432
|
function addEfficiency(acc, e) {
|
|
424
|
-
|
|
433
|
+
const hasSplit = Number.isFinite(Number(e.movedSame)) || Number.isFinite(Number(e.movedCross));
|
|
434
|
+
for (const k of ['served', 'in', 'out', 'cr', 'cc', 'moved', 'movedSame', 'movedCross', 'newConversation', 'bounced', 'refused', 'waited', 'waitMs', 'waitThenMoved']) {
|
|
425
435
|
acc[k] += Number(e[k]) || 0;
|
|
426
436
|
}
|
|
437
|
+
if (!hasSplit) acc.movedSame += Number(e.moved) || 0;
|
|
427
438
|
for (const c of Object.keys(acc.movedByCause)) acc.movedByCause[c] += Number(e.movedByCause?.[c]) || 0;
|
|
428
439
|
// Latency samples exist for this pool's own rows only (percentiles do not add).
|
|
429
440
|
if (Array.isArray(e.e2e)) acc.e2e.push(...e.e2e);
|
|
@@ -433,14 +444,21 @@ const pct = (arr, p) => {
|
|
|
433
444
|
const s = [...arr].sort((x, y) => x - y);
|
|
434
445
|
return s[Math.min(s.length - 1, Math.floor(p * s.length))];
|
|
435
446
|
};
|
|
436
|
-
/** Derived figures: hit = cr/(cr+cc+in) (GLM has no cc → r/(r+in)); missTokens = in+cc
|
|
447
|
+
/** Derived figures: hit = cr/(cr+cc+in) (GLM has no cc → r/(r+in)); missTokens = in+cc;
|
|
448
|
+
* writeShare = 1.25·cc / (0.1·cr + 1.25·cc + in) — the price-weighted share of
|
|
449
|
+
* cost-equivalent spend the cache WRITES carry (reads 0.1x, writes 1.25x, uncached
|
|
450
|
+
* input 1x). hit% alone read green while writes carried 55% of Claude spend
|
|
451
|
+
* (2026-11-17 audit, finding 2). Cost-equivalent = quota-burn proxy (OAuth
|
|
452
|
+
* subscriptions), not dollars. */
|
|
437
453
|
function finishEfficiency(acc, provider) {
|
|
438
454
|
const prompt = acc.cr + acc.cc + acc.in;
|
|
455
|
+
const cost = 0.1 * acc.cr + 1.25 * acc.cc + acc.in;
|
|
439
456
|
const { e2e, ...sums } = acc;
|
|
440
457
|
return {
|
|
441
458
|
...sums,
|
|
442
459
|
provider,
|
|
443
460
|
hit: prompt > 0 ? acc.cr / prompt : null,
|
|
461
|
+
writeShare: cost > 0 ? (1.25 * acc.cc) / cost : null,
|
|
444
462
|
missTokens: acc.in + acc.cc,
|
|
445
463
|
promptTokens: prompt,
|
|
446
464
|
lowVolume: acc.served < 20 || prompt < 100_000,
|
|
@@ -467,6 +485,27 @@ function lruSet(map, key, value) {
|
|
|
467
485
|
/** An Anthropic account with its own prompt cache (allow-list: a provider or legacy
|
|
468
486
|
* pool row can forward to GLM, so "not a provider" is not proof of a Claude cache). */
|
|
469
487
|
const isClaudeAccount = (a) => a?.type === 'oauth' || a?.type === 'apikey';
|
|
488
|
+
// Providers whose prompt cache is shared by every account of that provider, so a
|
|
489
|
+
// conversation warm on one row is warm on all of them. Measured 2026-11-16 (Mac pool,
|
|
490
|
+
// 4.9h): z.ai GLM->GLM hops re-wrote ~150-2.6k tokens vs ~190k per GLM->Claude hop.
|
|
491
|
+
// Kimi is unmeasured, so it stays per account.
|
|
492
|
+
const SHARED_CACHE_PROVIDERS = new Set(['zai']);
|
|
493
|
+
const sharesProviderCache = (a, b) => Boolean(a && b && a.type === 'provider' && b.type === 'provider'
|
|
494
|
+
&& a.provider === b.provider && SHARED_CACHE_PROVIDERS.has(a.provider));
|
|
495
|
+
// A Claude copy of a conversation that went to GLM is treated as dead after this long:
|
|
496
|
+
// audit 2026-11-16 — returns <5 min after the last Claude turn stayed cheap (32 hops,
|
|
497
|
+
// 0.5M), later ones re-wrote nearly in full (median dead-copy age 519 s; a detour pushes
|
|
498
|
+
// the prefix past the cache lookback, so the 1-hour copy does not survive it).
|
|
499
|
+
const COLD_CLAUDE_AFTER_MS = 300_000;
|
|
500
|
+
// How long a paid cold GLM->Claude hop marks the conversation as a repeat offender.
|
|
501
|
+
const COLD_HOP_MEMORY_MS = 6 * 3_600_000;
|
|
502
|
+
// Only conversations big enough that a re-write matters (~40k+ tokens).
|
|
503
|
+
const COLD_HOLD_MIN_BYTES = 150_000;
|
|
504
|
+
/** ROUTING FAMILY for the moved split (2026-11-17 audit, finding 4): every Claude
|
|
505
|
+
* account shares one cache-per-account world; each provider (zai, kimi) is its own
|
|
506
|
+
* family — z.ai's cache is shared across that provider's accounts, so a zai→zai hop
|
|
507
|
+
* re-writes ~nothing while a Claude↔provider hop re-writes the whole prompt. */
|
|
508
|
+
const accountFamily = (a) => (isClaudeAccount(a) ? 'claude' : String(a?.provider || 'provider'));
|
|
470
509
|
const IDENTITY_WINDOW_MS = 15 * 60_000;
|
|
471
510
|
const IDENTITY_MAX_SAMPLES = 4000;
|
|
472
511
|
const IDENTITY_MIN_PER_SESSION = 8;
|
|
@@ -742,6 +781,7 @@ export class AccountManager {
|
|
|
742
781
|
usage: {
|
|
743
782
|
totalInputTokens: 0,
|
|
744
783
|
totalOutputTokens: 0,
|
|
784
|
+
totalPaidTokens: 0,
|
|
745
785
|
totalRequests: 0,
|
|
746
786
|
lastUsed: null,
|
|
747
787
|
},
|
|
@@ -783,6 +823,9 @@ export class AccountManager {
|
|
|
783
823
|
// account only (a GLM/Kimi turn never overwrites the name; it sets `detoured`), so a
|
|
784
824
|
// conversation returning from a provider detour can go home.
|
|
785
825
|
this.conversationLastClaudeAccount = new Map();
|
|
826
|
+
// _homeKey → time of the last SERVED GLM->Claude hop that landed on a cold Claude copy
|
|
827
|
+
// (repeat cold-hop hold, see _coldClaudeHold).
|
|
828
|
+
this.conversationColdClaudeHops = new Map();
|
|
786
829
|
// Conversation-identity HEALTH (owner 2026-11-08: "make sure this cannot degrade
|
|
787
830
|
// silently"). Twice in one day a Claude Code request-shape detail (a moving
|
|
788
831
|
// cache_control marker, then a per-request billing checksum) made every turn hash
|
|
@@ -1115,23 +1158,45 @@ export class AccountManager {
|
|
|
1115
1158
|
this._bumpCaller(callerName, 1, 0);
|
|
1116
1159
|
}
|
|
1117
1160
|
|
|
1118
|
-
/** Attribute response TOKENS to the caller (SC22: tokens + requests).
|
|
1161
|
+
/** Attribute response TOKENS to the caller (SC22: tokens + requests).
|
|
1162
|
+
* Counting callers are callers with no full-prompt booking (tests, legacy paths);
|
|
1163
|
+
* full-prompt accounting books the SAME buckets once per request via
|
|
1164
|
+
* noteCallerPrompt (the running-max capacity figures), so the two never mix on a
|
|
1165
|
+
* request — releaseAccount books `caller` only when _callerPromptBooked is unset. */
|
|
1119
1166
|
countCallerTokens(callerName, tokens) {
|
|
1120
1167
|
if (!callerName || !tokens) return;
|
|
1121
|
-
this._bumpCaller(callerName, 0, tokens);
|
|
1122
|
-
}
|
|
1123
|
-
|
|
1124
|
-
|
|
1168
|
+
this._bumpCaller(callerName, 0, tokens, 0);
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
/** Book one request's FULL token figures for the caller (audit 2026-11-17, finding 3):
|
|
1172
|
+
* prompt = input + cache_read + cache_creation (provider-native), plus output; paid =
|
|
1173
|
+
* cache write + uncached input — the part that actually consumes quota/money.
|
|
1174
|
+
* Called ONCE per request, at the terminal release, off the request's running-max
|
|
1175
|
+
* capacity book (per-request max ⇒ a rerouted request books once, not per hop).
|
|
1176
|
+
* Legacy SSE/non-stream per-event countCallerTokens calls are dropped for requests
|
|
1177
|
+
* that route through this — this supersedes them (they booked message_start input
|
|
1178
|
+
* + output only: a 908k-token cache-write MISS booked 2 tokens). */
|
|
1179
|
+
noteCallerPrompt(callerName, { input = 0, cacheRead = 0, cacheCreation = 0, output = 0 }) {
|
|
1180
|
+
if (!callerName) return 0;
|
|
1181
|
+
const prompt = (input || 0) + (cacheRead || 0) + (cacheCreation || 0);
|
|
1182
|
+
const paid = (input || 0) + (cacheCreation || 0);
|
|
1183
|
+
if (!(prompt + (output || 0) > 0)) return 0;
|
|
1184
|
+
this._bumpCaller(callerName, 0, prompt + (output || 0), paid);
|
|
1185
|
+
return prompt;
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
_bumpCaller(callerName, req, tok, paid = 0) {
|
|
1125
1189
|
const now = new Date();
|
|
1126
1190
|
const hourKey = now.toISOString().slice(0, 13); // 'YYYY-MM-DDTHH' UTC
|
|
1127
1191
|
const dayKey = now.toISOString().slice(0, 10); // 'YYYY-MM-DD' UTC
|
|
1128
1192
|
let st = this.callerStats.get(callerName);
|
|
1129
|
-
if (!st) { st = { hours: {}, days: {}, total: { req: 0, tok: 0 } }; this.callerStats.set(callerName, st); }
|
|
1193
|
+
if (!st) { st = { hours: {}, days: {}, total: { req: 0, tok: 0, paid: 0 } }; this.callerStats.set(callerName, st); }
|
|
1130
1194
|
for (const [bucket, key] of [[st.hours, hourKey], [st.days, dayKey]]) {
|
|
1131
|
-
bucket[key] = bucket[key] || { req: 0, tok: 0 };
|
|
1132
|
-
bucket[key].req += req; bucket[key].tok += tok;
|
|
1195
|
+
bucket[key] = bucket[key] || { req: 0, tok: 0, paid: 0 };
|
|
1196
|
+
bucket[key].req += req; bucket[key].tok += tok; bucket[key].paid += paid;
|
|
1133
1197
|
}
|
|
1134
1198
|
st.total.req += req; st.total.tok += tok;
|
|
1199
|
+
st.total.paid = (st.total.paid || 0) + paid;
|
|
1135
1200
|
this._pruneCallerBuckets(st);
|
|
1136
1201
|
}
|
|
1137
1202
|
|
|
@@ -1174,22 +1239,22 @@ export class AccountManager {
|
|
|
1174
1239
|
const now = Date.now();
|
|
1175
1240
|
const out = new Map();
|
|
1176
1241
|
const win = (st, ms, hourly) => {
|
|
1177
|
-
let req = 0, tok = 0;
|
|
1242
|
+
let req = 0, tok = 0, paid = 0;
|
|
1178
1243
|
const src = hourly ? st.hours : st.days;
|
|
1179
1244
|
for (const [k, v] of Object.entries(src)) {
|
|
1180
1245
|
const t = Date.parse(hourly ? k + ':00:00Z' : k + 'T00:00:00Z');
|
|
1181
|
-
if (t <= now && now - t < ms) { req += v.req; tok += v.tok; }
|
|
1246
|
+
if (t <= now && now - t < ms) { req += v.req; tok += v.tok; paid += v.paid || 0; }
|
|
1182
1247
|
}
|
|
1183
|
-
return { req, tok };
|
|
1248
|
+
return { req, tok, paid };
|
|
1184
1249
|
};
|
|
1185
1250
|
const winPrior = (st, fromMs, toMs) => {
|
|
1186
|
-
let req = 0, tok = 0;
|
|
1251
|
+
let req = 0, tok = 0, paid = 0;
|
|
1187
1252
|
for (const [k, v] of Object.entries(st.days)) {
|
|
1188
1253
|
const t = Date.parse(k + 'T00:00:00Z');
|
|
1189
1254
|
const age = now - t;
|
|
1190
|
-
if (t <= now && age >= fromMs && age < toMs) { req += v.req; tok += v.tok; }
|
|
1255
|
+
if (t <= now && age >= fromMs && age < toMs) { req += v.req; tok += v.tok; paid += v.paid || 0; }
|
|
1191
1256
|
}
|
|
1192
|
-
return { req, tok };
|
|
1257
|
+
return { req, tok, paid };
|
|
1193
1258
|
};
|
|
1194
1259
|
for (const [name, st] of this.callerStats) {
|
|
1195
1260
|
out.set(name, {
|
|
@@ -1232,6 +1297,13 @@ export class AccountManager {
|
|
|
1232
1297
|
console.log(`[Maxpool] return-home lease throttled (${outcome.status} ${outcome.error || '-'}${scope}, ${lease.returnedHomeSwitched ? 'switched' : 'natural'}): ${account.name} depth=${depthBefore}`);
|
|
1233
1298
|
}
|
|
1234
1299
|
|
|
1300
|
+
// Per-account + per-caller token totals, booked ONCE per request at its terminal
|
|
1301
|
+
// release off the running-max capacity book (finding 3, 2026-11-17): a stream that
|
|
1302
|
+
// died mid-flight still delivered its tokens, so this runs for every non-neutral
|
|
1303
|
+
// outcome, success or not. A request with no capacity figures (a refusal, a bounce)
|
|
1304
|
+
// books nothing.
|
|
1305
|
+
if (!outcome.neutral) this._bookRequestTokens(lease, account, outcome);
|
|
1306
|
+
|
|
1235
1307
|
if (lease.upstreamThrottleProbe) {
|
|
1236
1308
|
if (outcome.success) {
|
|
1237
1309
|
this.clearUpstreamThrottle('successful recovery probe');
|
|
@@ -1294,7 +1366,7 @@ export class AccountManager {
|
|
|
1294
1366
|
}
|
|
1295
1367
|
// Backstop for callers that never saw a first byte (unit callers): a 2xx served
|
|
1296
1368
|
// the conversation. Idempotent — the first-byte note already ran on the real paths.
|
|
1297
|
-
if (lease.requestInfo && !(outcome.status >= 300)) this.noteConversationServed(lease.requestInfo, account);
|
|
1369
|
+
if (lease.requestInfo && !(outcome.status >= 300)) { this.stampMovedFrom(lease.requestInfo); this.noteConversationServed(lease.requestInfo, account); }
|
|
1298
1370
|
// Efficiency fields derived from the served request; a field the caller passed
|
|
1299
1371
|
// explicitly on the outcome wins.
|
|
1300
1372
|
const tel = this._servedTelemetry(lease.requestInfo, account);
|
|
@@ -1335,6 +1407,17 @@ export class AccountManager {
|
|
|
1335
1407
|
repeatBounce: Boolean(outcome.repeatBounce),
|
|
1336
1408
|
usage: outcome.usage || null,
|
|
1337
1409
|
moved: Boolean(outcome.moved),
|
|
1410
|
+
// Moved split by routing family (finding 4): same = within one family (a zai→zai
|
|
1411
|
+
// hop, near-free — z.ai's cache is shared across its accounts); cross = the
|
|
1412
|
+
// prompt re-wrote wholesale on the new family (e.g. Claude↔provider). A caller
|
|
1413
|
+
// passing `moved` without a family gets it classified HERE (off the lease's
|
|
1414
|
+
// request when it exists, else off the serving account alone).
|
|
1415
|
+
movedSame: Boolean(outcome.movedSame)
|
|
1416
|
+
|| (Boolean(outcome.moved) && outcome.movedCross !== true
|
|
1417
|
+
&& !this._outcomeMoveCross(outcome, lease, account)),
|
|
1418
|
+
movedCross: Boolean(outcome.movedCross)
|
|
1419
|
+
|| (Boolean(outcome.moved) && outcome.movedSame !== true
|
|
1420
|
+
&& this._outcomeMoveCross(outcome, lease, account)),
|
|
1338
1421
|
newConversation: Boolean(outcome.newConversation),
|
|
1339
1422
|
moveCause: outcome.moveCause || null,
|
|
1340
1423
|
busyWaitMs: outcome.busyWaitMs || 0,
|
|
@@ -1346,6 +1429,58 @@ export class AccountManager {
|
|
|
1346
1429
|
this._pruneLoadEvents(account, now);
|
|
1347
1430
|
}
|
|
1348
1431
|
|
|
1432
|
+
/** Book one request's FULL token figures (input + cache read + cache write, provider-
|
|
1433
|
+
* native, plus output) on the account's usage and the named caller — once per
|
|
1434
|
+
* request (`_tokensBooked`), at the terminal release (finding 3, 2026-11-17: the
|
|
1435
|
+
* old per-event booking counted message_start input + output only, so a 908k-token
|
|
1436
|
+
* cache-write miss booked 2 tokens, and GLM input — which rides message_delta —
|
|
1437
|
+
* never landed). Unit callers that pass `outcome.usage` instead of filling the
|
|
1438
|
+
* request's capacity book are honoured when the request carries no figures. */
|
|
1439
|
+
_bookRequestTokens(lease, account, outcome) {
|
|
1440
|
+
const ri = lease.requestInfo;
|
|
1441
|
+
if (ri?.isCountTokens || ri?._tokensBooked) return;
|
|
1442
|
+
// A cloud MIRROR row is served by the linked pool, which books its own tokens (the
|
|
1443
|
+
// People page counts each request once, at the pool that served it); the edge only
|
|
1444
|
+
// taps the usage for efficiency figures.
|
|
1445
|
+
if (account.remote) return;
|
|
1446
|
+
const has = ri && (ri._capacityInput != null || ri._capacityCacheRead != null
|
|
1447
|
+
|| ri._capacityCacheCreation != null || ri._capacityOutput != null);
|
|
1448
|
+
const u = has
|
|
1449
|
+
? { input: ri._capacityInput || 0, cacheRead: ri._capacityCacheRead || 0, cacheCreation: ri._capacityCacheCreation || 0, output: ri._capacityOutput || 0 }
|
|
1450
|
+
: (outcome.success && outcome.usage
|
|
1451
|
+
? { input: outcome.usage.in || 0, cacheRead: outcome.usage.cr || 0, cacheCreation: outcome.usage.cc || 0, output: outcome.usage.out || 0 }
|
|
1452
|
+
: null);
|
|
1453
|
+
if (!u || !(u.input + u.cacheRead + u.cacheCreation + u.output > 0)) return;
|
|
1454
|
+
if (ri) ri._tokensBooked = true;
|
|
1455
|
+
const idx = Number.isInteger(account.index) ? account.index : this.accounts.indexOf(account);
|
|
1456
|
+
this.updateUsage(idx, { input: u.input, cacheRead: u.cacheRead, cacheCreation: u.cacheCreation }, u.output);
|
|
1457
|
+
this.noteCallerPrompt(ri?.caller ?? outcome.caller, u);
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1460
|
+
/** Classify an outcome-passed move (no family on the outcome): cross unless the
|
|
1461
|
+
* request's remembered previous holder is a same-family account. No previous holder
|
|
1462
|
+
* known (no request identity, unit callers) → a Claude serve is cross (unknown prev
|
|
1463
|
+
* could have re-written the whole prompt — over-report the costly side), a provider
|
|
1464
|
+
* serve same (its family's shared cache makes the rewrite assumption weak). */
|
|
1465
|
+
_outcomeMoveCross(outcome, lease, account) {
|
|
1466
|
+
const ri = lease?.requestInfo;
|
|
1467
|
+
const fromName = ri ? (ri._servedMovedFrom ?? ri._prevServedName ?? null) : null;
|
|
1468
|
+
const fromAccount = fromName != null ? this.accounts.find(a => a.name === fromName) : null;
|
|
1469
|
+
if (fromAccount == null) return isClaudeAccount(account);
|
|
1470
|
+
return accountFamily(fromAccount) !== accountFamily(account);
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
/** Remember which account served this conversation's PREVIOUS turn, on the request,
|
|
1474
|
+
* BEFORE noteConversationServed overwrites the memory (finding 4: the move's family
|
|
1475
|
+
* split needs the previous holder). Idempotent per request; a request that already
|
|
1476
|
+
* carries `_servedMovedFrom` (set by noteConversationServed itself) is left alone.
|
|
1477
|
+
* Never touches the conversation maps — display/telemetry only. */
|
|
1478
|
+
stampMovedFrom(requestInfo) {
|
|
1479
|
+
if (!requestInfo || requestInfo._notedServed || requestInfo.isCountTokens || '_prevServedName' in requestInfo) return;
|
|
1480
|
+
const key = this._affinityKey(requestInfo);
|
|
1481
|
+
requestInfo._prevServedName = key ? (this.conversationLastAccount.get(key) ?? null) : null;
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1349
1484
|
/** Efficiency fields of a SERVED event, read off the request that was served. */
|
|
1350
1485
|
_servedTelemetry(ri, account) {
|
|
1351
1486
|
if (!ri) return {};
|
|
@@ -1355,6 +1490,19 @@ export class AccountManager {
|
|
|
1355
1490
|
// add a latency sample that starts AFTER the edge's wait. Served + tokens stay here.
|
|
1356
1491
|
const pinned = ri.pinAccount != null;
|
|
1357
1492
|
const moved = !pinned && Boolean(ri._servedMoved) && ri._notedServed === account.name;
|
|
1493
|
+
// Family split of the move (finding 4): prev family read off the request — the
|
|
1494
|
+
// other engineer's noteConversationServed will soon stamp `ri._servedMovedFrom`;
|
|
1495
|
+
// until then the name is derived here (same key, same map, no edit there).
|
|
1496
|
+
let movedSame = null, movedCross = null;
|
|
1497
|
+
if (moved) {
|
|
1498
|
+
const fromName = ri._servedMovedFrom ?? ri._prevServedName ?? null;
|
|
1499
|
+
const fromAccount = fromName != null ? this.accounts.find(a => a.name === fromName) : null;
|
|
1500
|
+
// No prev row found (renamed/offline) → CROSS: unknown prev family could have
|
|
1501
|
+
// re-written the whole prompt, and over-reporting the costly side is the honest
|
|
1502
|
+
// failure direction for an alarm.
|
|
1503
|
+
const cross = fromAccount == null || accountFamily(fromAccount) !== accountFamily(account);
|
|
1504
|
+
if (cross) movedCross = true; else movedSame = true;
|
|
1505
|
+
}
|
|
1358
1506
|
return {
|
|
1359
1507
|
countTokens: Boolean(ri.isCountTokens),
|
|
1360
1508
|
usage: ri.isCountTokens ? null : {
|
|
@@ -1362,6 +1510,7 @@ export class AccountManager {
|
|
|
1362
1510
|
cr: ri._capacityCacheRead || 0, cc: ri._capacityCacheCreation || 0,
|
|
1363
1511
|
},
|
|
1364
1512
|
moved,
|
|
1513
|
+
movedSame, movedCross,
|
|
1365
1514
|
moveCause: moved ? (ri._moveCause || (ri._hop > 0 ? 'other' : 'balance')) : null,
|
|
1366
1515
|
newConversation: !pinned && Boolean(ri._newConversation) && ri._notedServed === account.name,
|
|
1367
1516
|
|
|
@@ -1393,6 +1542,8 @@ export class AccountManager {
|
|
|
1393
1542
|
if (e.usage) { out.in += e.usage.in; out.out += e.usage.out; out.cr += e.usage.cr; out.cc += e.usage.cc; }
|
|
1394
1543
|
if (e.moved) {
|
|
1395
1544
|
out.moved++;
|
|
1545
|
+
if (e.movedCross) out.movedCross++;
|
|
1546
|
+
else out.movedSame++; // legacy events (no split) default to the cheap side
|
|
1396
1547
|
const c = Object.hasOwn(out.movedByCause, e.moveCause) ? e.moveCause : 'other';
|
|
1397
1548
|
out.movedByCause[c]++;
|
|
1398
1549
|
if (e.busyWaitMs > 0) out.waitThenMoved++;
|
|
@@ -1420,6 +1571,11 @@ export class AccountManager {
|
|
|
1420
1571
|
if (!exp) return edge;
|
|
1421
1572
|
const acc = emptyEfficiency();
|
|
1422
1573
|
for (const k of ['served', 'in', 'out', 'cr', 'cc']) acc[k] = Number(exp[k]) || 0;
|
|
1574
|
+
// Family split (finding 4): an export that carries it adds both halves; an older
|
|
1575
|
+
// export keeps its moved total on the cheap side (same) — never invents cross moves.
|
|
1576
|
+
const expSplit = Number.isFinite(Number(exp.movedSame)) || Number.isFinite(Number(exp.movedCross));
|
|
1577
|
+
acc.movedSame = (expSplit ? Number(exp.movedSame) || 0 : Number(exp.moved) || 0) + edge.movedSame;
|
|
1578
|
+
acc.movedCross = (expSplit ? Number(exp.movedCross) || 0 : 0) + edge.movedCross;
|
|
1423
1579
|
for (const k of EDGE_EFFICIENCY_FIELDS) acc[k] = (Number(exp[k]) || 0) + edge[k];
|
|
1424
1580
|
for (const c of Object.keys(acc.movedByCause)) {
|
|
1425
1581
|
acc.movedByCause[c] = (Number(exp.movedByCause?.[c]) || 0) + edge.movedByCause[c];
|
|
@@ -1869,7 +2025,12 @@ export class AccountManager {
|
|
|
1869
2025
|
// Fairness stays: the skip only fires after a real wait, and the head's own
|
|
1870
2026
|
// window (queueAndRetry's ceiling) still bounds IT.
|
|
1871
2027
|
const HEAD_SKIP_MS = 60_000;
|
|
1872
|
-
|
|
2028
|
+
// A cold-held ticket (waiting for a GLM row only) is no head for anyone else to
|
|
2029
|
+
// queue behind (red-team 2026-11-17 f1); it keeps its own FIFO place for GLM rows.
|
|
2030
|
+
const head = this._coldHeldTicket(ticket)
|
|
2031
|
+
? this.queueState.waiting[0]
|
|
2032
|
+
: this.queueState.waiting.find(t => !this._coldHeldTicket(t));
|
|
2033
|
+
if (!head || head.id === ticket.id) {
|
|
1873
2034
|
// fast path: unchanged FIFO
|
|
1874
2035
|
} else {
|
|
1875
2036
|
const waited = Date.now() - (ticket.queuedAt ?? Date.now());
|
|
@@ -3324,16 +3485,95 @@ export class AccountManager {
|
|
|
3324
3485
|
// Cloud mirrors egress from the cloud's own IP — this machine's upstream throttle
|
|
3325
3486
|
// and local queue fairness don't describe them.
|
|
3326
3487
|
const providerLike = account.type === 'provider' || Boolean(account.remote);
|
|
3488
|
+
// A request parked by the repeat cold-hop hold (red-team 2026-11-17 f1) still
|
|
3489
|
+
// registers a queue ticket like any queued request, but its ticket must not count
|
|
3490
|
+
// as a WAITER for anyone else: a queued waiter makes local Claude rows refuse every
|
|
3491
|
+
// NEW request without a ticket, and a ticket-holder queues behind it — so one held
|
|
3492
|
+
// conversation stalled unrelated traffic on an idle Claude row for up to
|
|
3493
|
+
// HEAD_SKIP_MS (60 s). Held tickets are therefore skipped by the fairness gate
|
|
3494
|
+
// below and by the head pick in canAdmitQueuedRequest (_coldHeldTicket).
|
|
3495
|
+
const coldHeld = isClaudeAccount(account) && this._coldClaudeHold(requestInfo);
|
|
3327
3496
|
if (
|
|
3328
3497
|
!providerLike
|
|
3329
|
-
&& this.queueState.waiting.
|
|
3498
|
+
&& this.queueState.waiting.some(t => !this._coldHeldTicket(t))
|
|
3330
3499
|
&& !requestInfo.queueTicket
|
|
3331
3500
|
&& !requestInfo.queueAdmitted
|
|
3332
3501
|
) return false;
|
|
3333
3502
|
if (!providerLike && this._isUpstreamThrottleBlocking()) return false;
|
|
3503
|
+
if (coldHeld) return false;
|
|
3504
|
+
return true;
|
|
3505
|
+
}
|
|
3506
|
+
|
|
3507
|
+
/**
|
|
3508
|
+
* REPEAT COLD-HOP HOLD (owner 2026-11-17, "yes"): true while Claude rows must NOT take
|
|
3509
|
+
* this request, so it waits (queue + SSE heartbeat) for a GLM row instead of paying a
|
|
3510
|
+
* second cold Claude re-write. All of: the knob is on; a big streaming /v1/messages
|
|
3511
|
+
* turn; the conversation is warm on z.ai (served there last); it already paid a cold
|
|
3512
|
+
* GLM->Claude hop in the last COLD_HOP_MEMORY_MS; its Claude copy is dead; some z.ai
|
|
3513
|
+
* row can serve before the cap ends; and the cap (from the first time the hold
|
|
3514
|
+
* engaged for this request) has not run out.
|
|
3515
|
+
*/
|
|
3516
|
+
_coldClaudeHold(requestInfo = {}, now = Date.now()) {
|
|
3517
|
+
const capMs = Number(this.scheduler.coldClaudeRepeatHoldMs);
|
|
3518
|
+
if (!(capMs > 0) || requestInfo.isCountTokens || !requestInfo.stream) return false;
|
|
3519
|
+
if ((requestInfo.bodyBytes || 0) < COLD_HOLD_MIN_BYTES) return false;
|
|
3520
|
+
const homeKey = this._homeKey(requestInfo);
|
|
3521
|
+
const key = this._affinityKey(requestInfo);
|
|
3522
|
+
if (!homeKey || !key) return false;
|
|
3523
|
+
const last = this.accounts.find(a => a.name === this.conversationLastAccount.get(key));
|
|
3524
|
+
// A non-z.ai holder (Claude, Kimi) ends at _sharedCacheRowSoon: no row shares its cache.
|
|
3525
|
+
if (!last) return false;
|
|
3526
|
+
const hopAt = this.conversationColdClaudeHops.get(homeKey);
|
|
3527
|
+
if (hopAt == null || now - hopAt > COLD_HOP_MEMORY_MS) return false;
|
|
3528
|
+
const home = this.conversationLastClaudeAccount.get(homeKey);
|
|
3529
|
+
if (home && now - home.at <= COLD_CLAUDE_AFTER_MS) return false;
|
|
3530
|
+
const started = requestInfo._coldHoldStartedAt ?? now;
|
|
3531
|
+
const until = started + capMs;
|
|
3532
|
+
if (now >= until) return false;
|
|
3533
|
+
if (!this._sharedCacheRowSoon(last, requestInfo, until, now)) return false;
|
|
3534
|
+
if (requestInfo._coldHoldStartedAt == null) {
|
|
3535
|
+
requestInfo._coldHoldStartedAt = now;
|
|
3536
|
+
console.log(`[Maxpool] holding a ${requestInfo.bodyBytes}B conversation for GLM up to ${Math.round(capMs / 1000)}s: it already paid one cold Claude re-write and its Claude copy is ${home ? `${Math.round((now - home.at) / 1000)}s old` : 'absent'}`);
|
|
3537
|
+
}
|
|
3334
3538
|
return true;
|
|
3335
3539
|
}
|
|
3336
3540
|
|
|
3541
|
+
/** Is this queue ticket's request inside a LIVE repeat cold-hop hold? The stamp
|
|
3542
|
+
* alone is not enough (red-team r2): when the hold releases early — every GLM row
|
|
3543
|
+
* went dead mid-wait — the ticket becomes an ordinary fair waiter again at once,
|
|
3544
|
+
* instead of staying invisible to the head pick until the cap expires. */
|
|
3545
|
+
_coldHeldTicket(t, now = Date.now()) {
|
|
3546
|
+
const started = t?.requestInfo?._coldHoldStartedAt;
|
|
3547
|
+
if (started == null) return false;
|
|
3548
|
+
const capMs = Number(this.scheduler.coldClaudeRepeatHoldMs);
|
|
3549
|
+
return capMs > 0 && now < started + capMs && this._coldClaudeHold(t.requestInfo, now);
|
|
3550
|
+
}
|
|
3551
|
+
|
|
3552
|
+
/** Could a z.ai row sharing `holder`'s cache serve this request before `until`?
|
|
3553
|
+
* (Enabled, eligible, not weekly-out, not benched or 5h-walled past `until`.) */
|
|
3554
|
+
_sharedCacheRowSoon(holder, requestInfo, until, now) {
|
|
3555
|
+
const profile = requestInfo.profile || 'claude';
|
|
3556
|
+
return this.accounts.some(a => {
|
|
3557
|
+
if (a.enabled === false || !sharesProviderCache(holder, a)) return false;
|
|
3558
|
+
// Permanent outs (red-team 2026-11-17 f2): auth-dead, subscription gone,
|
|
3559
|
+
// terminal statuses, a dead cloud mirror — no wait can end in a serve, so the
|
|
3560
|
+
// hold must release at once instead of idling to the cap.
|
|
3561
|
+
if (a.status === 'error' || a.status === 'exhausted' || a.subscriptionGone) return false;
|
|
3562
|
+
if (a.remote && (a.refreshDead || a.status === 'error')) return false;
|
|
3563
|
+
if (!this._matchesRequest(a, profile, requestInfo)) return false;
|
|
3564
|
+
if ((a.rateLimitedUntil || 0) > until && a.status === 'throttled') return false;
|
|
3565
|
+
if ((a.cooldownUntil || 0) > until) return false;
|
|
3566
|
+
if ((a.provisionalUpstreamUntil || 0) > until) return false;
|
|
3567
|
+
// Peak-hour hard bar (red-team r2 f3): peakCap 0 = never during the window, so
|
|
3568
|
+
// waiting cannot end in a serve. Judged at the END of the cap: a window that
|
|
3569
|
+
// closes before then still counts as soon.
|
|
3570
|
+
if (this._peakHardBarred(a, until)) return false;
|
|
3571
|
+
if (this._isSessionQuotaUnavailable(a, now)) return false;
|
|
3572
|
+
const w = this._weeklyRawState(a, now);
|
|
3573
|
+
return w !== 'exhausted' && w !== 'capped';
|
|
3574
|
+
});
|
|
3575
|
+
}
|
|
3576
|
+
|
|
3337
3577
|
_crossProviderFallbackPolicy() {
|
|
3338
3578
|
const p = this.scheduler.crossProviderFallbackPolicy;
|
|
3339
3579
|
return (p === 'never' || p === 'always') ? p : 'when-exhausted';
|
|
@@ -4114,23 +4354,32 @@ export class AccountManager {
|
|
|
4114
4354
|
* there but costs nothing either, and one rule is simpler to reason about.
|
|
4115
4355
|
* Applies in every scored mode (balance, prefer-*); never in sticky (see below).
|
|
4116
4356
|
*/
|
|
4117
|
-
_noteConversationAccount(
|
|
4118
|
-
//
|
|
4119
|
-
// (noteConversationServed), never here: a refused/failed
|
|
4120
|
-
// conversation (2026-11-15).
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
4357
|
+
_noteConversationAccount(_requestInfo, _account) {
|
|
4358
|
+
// Both conversation memories are written at the first real upstream byte
|
|
4359
|
+
// (noteConversationServed), never here: a refused/failed attempt must not re-point
|
|
4360
|
+
// the conversation (last account: 2026-11-15; Claude home + detour flag: 2026-11-17).
|
|
4361
|
+
}
|
|
4362
|
+
|
|
4363
|
+
/** Home memory: ONE key per conversation, independent of identity health, so the
|
|
4364
|
+
* degraded <-> ok flip of _affinityKey can neither hide a home nor lend it to the
|
|
4365
|
+
* sibling conversations that share the terminal key. The entry carries its own detour
|
|
4366
|
+
* state: a served Claude turn clears it, a served provider turn sets it (same name and
|
|
4367
|
+
* `at`, only the LRU position moves so other traffic cannot evict a long detour).
|
|
4368
|
+
* Also remembers a served GLM->Claude hop that landed on a dead Claude copy. */
|
|
4369
|
+
_noteHomeServed(requestInfo, account, prevName) {
|
|
4126
4370
|
const homeKey = this._homeKey(requestInfo);
|
|
4127
4371
|
if (!homeKey) return;
|
|
4372
|
+
const now = requestInfo.now ?? Date.now();
|
|
4373
|
+
const home = this.conversationLastClaudeAccount.get(homeKey);
|
|
4128
4374
|
if (isClaudeAccount(account)) {
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4132
|
-
|
|
4133
|
-
|
|
4375
|
+
const prev = prevName != null ? this.accounts.find(a => a.name === prevName) : null;
|
|
4376
|
+
const copyDead = !home || home.name !== account.name || now - home.at > COLD_CLAUDE_AFTER_MS;
|
|
4377
|
+
if (prev && SHARED_CACHE_PROVIDERS.has(prev.provider) && prev.type === 'provider' && copyDead) {
|
|
4378
|
+
lruSet(this.conversationColdClaudeHops, homeKey, now);
|
|
4379
|
+
}
|
|
4380
|
+
lruSet(this.conversationLastClaudeAccount, homeKey, { name: account.name, at: now, detoured: false });
|
|
4381
|
+
} else if (home) {
|
|
4382
|
+
lruSet(this.conversationLastClaudeAccount, homeKey, { ...home, detoured: true });
|
|
4134
4383
|
}
|
|
4135
4384
|
}
|
|
4136
4385
|
|
|
@@ -4144,10 +4393,12 @@ export class AccountManager {
|
|
|
4144
4393
|
requestInfo._notedServed = account.name;
|
|
4145
4394
|
requestInfo._servedAt = Date.now();
|
|
4146
4395
|
const key = this._affinityKey(requestInfo);
|
|
4396
|
+
const prev = key ? this.conversationLastAccount.get(key) : undefined;
|
|
4397
|
+
this._noteHomeServed(requestInfo, account, prev);
|
|
4147
4398
|
if (!key) return;
|
|
4148
|
-
const prev = this.conversationLastAccount.get(key);
|
|
4149
4399
|
requestInfo._newConversation = prev == null;
|
|
4150
4400
|
requestInfo._servedMoved = prev != null && prev !== account.name;
|
|
4401
|
+
requestInfo._servedMovedFrom = requestInfo._servedMoved ? prev : null;
|
|
4151
4402
|
lruSet(this.conversationLastAccount, key, account.name);
|
|
4152
4403
|
lruSet(this.conversationLastServedAt, key, Date.now());
|
|
4153
4404
|
}
|
|
@@ -4284,7 +4535,12 @@ export class AccountManager {
|
|
|
4284
4535
|
if (base <= 0 || (this.scheduler.routingMode === 'sticky' && !retrySame)) return 0;
|
|
4285
4536
|
if (!retrySame) {
|
|
4286
4537
|
const key = this._affinityKey(requestInfo);
|
|
4287
|
-
|
|
4538
|
+
const holder = key ? (holderName ?? this.conversationLastAccount.get(key)) : null;
|
|
4539
|
+
// FAMILY AFFINITY (owner 2026-11-17): a z.ai holder makes every z.ai row warm (the
|
|
4540
|
+
// cache is shared), so when the holder is excluded/busy a GLM sibling — not a cold
|
|
4541
|
+
// Claude row — inherits the hold. Same bonus, release gates and anti-dogpile cap.
|
|
4542
|
+
if (!holder || (holder !== account.name
|
|
4543
|
+
&& !sharesProviderCache(this.accounts.find(a => a.name === holder), account))) return 0;
|
|
4288
4544
|
}
|
|
4289
4545
|
// Scale with the CONVERSATION'S SIZE too: losing the cache costs the prefix a
|
|
4290
4546
|
// hop re-writes, and requestInfo.bodyBytes ≈ prefix + this turn. Checkpoint
|
|
@@ -5594,12 +5850,24 @@ export class AccountManager {
|
|
|
5594
5850
|
|
|
5595
5851
|
/**
|
|
5596
5852
|
* Update cumulative token usage from response body data.
|
|
5853
|
+
* `inputTokens` may be a number (legacy: uncached input only) or a usage object
|
|
5854
|
+
* {input, cacheRead, cacheCreation} — the object books FULL prompt tokens
|
|
5855
|
+
* (input + cache read + cache write, provider-native; finding 3, 2026-11-17), and
|
|
5856
|
+
* paid (write + uncached) into totalPaidTokens beside it.
|
|
5597
5857
|
*/
|
|
5598
5858
|
updateUsage(accountIndex, inputTokens, outputTokens) {
|
|
5599
5859
|
const account = this.accounts[accountIndex];
|
|
5600
5860
|
if (!account) return;
|
|
5601
|
-
|
|
5861
|
+
let input = inputTokens, paid = null;
|
|
5862
|
+
if (inputTokens && typeof inputTokens === 'object') {
|
|
5863
|
+
input = (inputTokens.input || 0) + (inputTokens.cacheRead || 0) + (inputTokens.cacheCreation || 0);
|
|
5864
|
+
paid = (inputTokens.input || 0) + (inputTokens.cacheCreation || 0);
|
|
5865
|
+
}
|
|
5866
|
+
if (input) account.usage.totalInputTokens += input;
|
|
5602
5867
|
if (outputTokens) account.usage.totalOutputTokens += outputTokens;
|
|
5868
|
+
// Legacy two-arg calls carry uncached input → it counts as paid too.
|
|
5869
|
+
const paidAdd = paid != null ? paid : (typeof inputTokens === 'number' ? inputTokens : 0);
|
|
5870
|
+
if (paidAdd) account.usage.totalPaidTokens = (account.usage.totalPaidTokens || 0) + paidAdd;
|
|
5603
5871
|
}
|
|
5604
5872
|
|
|
5605
5873
|
/**
|
|
@@ -5986,7 +6254,7 @@ export class AccountManager {
|
|
|
5986
6254
|
// Unknown quota until the first response — probe it like startup accounts.
|
|
5987
6255
|
probing: true,
|
|
5988
6256
|
quota: emptyQuota(),
|
|
5989
|
-
usage: { totalInputTokens: 0, totalOutputTokens: 0, totalRequests: 0, lastUsed: null },
|
|
6257
|
+
usage: { totalInputTokens: 0, totalOutputTokens: 0, totalPaidTokens: 0, totalRequests: 0, lastUsed: null },
|
|
5990
6258
|
inFlight: 0,
|
|
5991
6259
|
activeWeight: 0,
|
|
5992
6260
|
completedRequests: 0,
|
package/src/capacity-ledger.js
CHANGED
|
@@ -385,7 +385,10 @@ export class CapacityLedger {
|
|
|
385
385
|
accrue(name, { input, output, cacheRead = 0, cacheCreation = 0, model = null }, at = this._now(), windows = ['ses', 'wk']) {
|
|
386
386
|
// TOKEN DEFINITION (v5, owner 2026-11-06): every token the provider processed =
|
|
387
387
|
// input + cache_read + cache_creation + output, identical for Anthropic and z.ai so
|
|
388
|
-
// both sit on one axis. (
|
|
388
|
+
// both sit on one axis. NOTE (audit 2026-11-17): "one axis" means
|
|
389
|
+
// one DEFINITION — counts stay provider-native (GLM's tokenizer reads ~0.6x Anthropic's
|
|
390
|
+
// for the same prompt) and a 0.1x cache read weighs the same as a 1.25x write; the
|
|
391
|
+
// Capacity page labels the axis "processed tokens (provider-native)". (The cache counters are also kept separately per cycle.)
|
|
389
392
|
const cr = cacheRead > 0 ? cacheRead : 0, cc = cacheCreation > 0 ? cacheCreation : 0;
|
|
390
393
|
const total = (input > 0 ? input : 0) + (output > 0 ? output : 0) + cr + cc;
|
|
391
394
|
if (!(total > 0)) return;
|
package/src/index.js
CHANGED
|
@@ -2275,7 +2275,11 @@ async function statusCommand() {
|
|
|
2275
2275
|
console.log(` Tokens: ${tok} used Requests: ${req} used`);
|
|
2276
2276
|
}
|
|
2277
2277
|
|
|
2278
|
-
|
|
2278
|
+
// Full prompt (input + cache read + cache write) + output, provider-native; paid =
|
|
2279
|
+
// cache write + uncached input (audit 2026-11-17, finding 3). Older servers book
|
|
2280
|
+
// uncached input only and carry no paid figure — then Total reads as before.
|
|
2281
|
+
const paidTok = acct.usage.totalPaidTokens;
|
|
2282
|
+
console.log(` Total: ${acct.usage.totalInputTokens + acct.usage.totalOutputTokens} tokens${paidTok != null ? ` (paid ${paidTok})` : ''}, ${acct.usage.totalRequests} requests`);
|
|
2279
2283
|
if (acct.rateLimitedUntil) console.log(` Throttled until: ${acct.rateLimitedUntil}`);
|
|
2280
2284
|
console.log('');
|
|
2281
2285
|
}
|
|
@@ -2806,10 +2810,10 @@ function efficiencyLines(data, key = 'last15m') {
|
|
|
2806
2810
|
const s = (ms) => (ms == null ? '—' : `${(ms / 1000).toFixed(1)}s`);
|
|
2807
2811
|
return [
|
|
2808
2812
|
`Efficiency (${key === 'last1h' ? '1h' : '15m'}):`,
|
|
2809
|
-
` Cache: Claude ${pctOrDash(claude.hit)} (miss ${(claude.missTokens / 1e6).toFixed(1)}M tok) · GLM ${pctOrDash(glm.hit)} (miss ${(glm.missTokens / 1e6).toFixed(1)}M tok)`,
|
|
2810
|
-
` Moves: ${served ? `${Math.round(sum('moved') * 100 / served)}/100 served` : '—'} (busy ${n(cause('busy'))} · balance ${n(cause('balance'))} · wall ${n(cause('wall'))} · link ${n(cause('link'))} · other ${n(cause('other'))})`,
|
|
2813
|
+
` Cache: Claude ${pctOrDash(claude.hit)} (miss ${(claude.missTokens / 1e6).toFixed(1)}M tok${claude.writeShare != null ? ` · writes ${pctOrDash(claude.writeShare)} of cost` : ''}) · GLM ${pctOrDash(glm.hit)} (miss ${(glm.missTokens / 1e6).toFixed(1)}M tok)`,
|
|
2814
|
+
` Moves: ${served ? `${Math.round(sum('moved') * 100 / served)}/100 served` : '—'} (cross-family ${n(sum('movedCross'))} · same-family ${n(sum('movedSame'))} · busy ${n(cause('busy'))} · balance ${n(cause('balance'))} · wall ${n(cause('wall'))} · link ${n(cause('link'))} · other ${n(cause('other'))})`,
|
|
2811
2815
|
` Bounced: ${n(sum('bounced'))} (refused ${n(sum('refused'))}) · failed ${failed} · link drops ${linkDrops}`,
|
|
2812
|
-
`
|
|
2816
|
+
` Busy waits: ${n(sum('waited'))}${sum('waited') ? ` (avg ${s(sum('waitMs') / sum('waited'))})` : ''} · waited then moved ${n(sum('waitThenMoved'))} (busy-bounce waits only; queue holds show in first-byte p95)`,
|
|
2813
2817
|
` Client errors: ${co ? `${co.poolErrors + co.requestErrors} of ${co.total} (pool ${co.poolErrors} · request ${co.requestErrors})` : '—'}`,
|
|
2814
2818
|
` First byte: p50 ${s(overall.e2eP50)} · p95 ${s(overall.e2eP95)} (this pool's clients, waits included)`,
|
|
2815
2819
|
'',
|
package/src/server.js
CHANGED
|
@@ -4,7 +4,7 @@ import { normalizeClientHeaders, rewriteMetadataUserIdRaw } from './client-ident
|
|
|
4
4
|
import { appendEventLog } from './event-log.js';
|
|
5
5
|
import { atomicConfigUpdate } from './config.js';
|
|
6
6
|
import { HOP_HEADER, nextHopCount, safeText } from './pool-link.js';
|
|
7
|
-
import { ThreadOwners, readThreadIntent, threadRefusalBody, isThreadlessAccount } from './thread-gate.js';
|
|
7
|
+
import { ThreadOwners, readThreadIntent, threadRefusalBody, isThreadlessAccount, isThreadedIntent } from './thread-gate.js';
|
|
8
8
|
import { ProviderThinkingScrubber, scrubProviderResponseBody, scrubEnabledFor, foreignToolBlockText, describeScrub } from './provider-thinking-scrub.js';
|
|
9
9
|
import { writeFile, mkdir } from 'node:fs/promises';
|
|
10
10
|
import { readFileSync } from 'node:fs';
|
|
@@ -653,6 +653,11 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
653
653
|
const queueConfig = { ...DEFAULT_QUEUE, ...(config.queue || {}) };
|
|
654
654
|
const canRetryBufferedBody = body.length <= retryConfig.maxRetryBufferBytes;
|
|
655
655
|
requestInfo = describeRequest(req, body);
|
|
656
|
+
// THREAD REPLY SINK: one object per client request, shared BY REFERENCE through
|
|
657
|
+
// every `{ ...requestInfo }` copy the repair/retry paths make, so the reply id the
|
|
658
|
+
// serving attempt captures reaches this handler's finally (a field written on a
|
|
659
|
+
// copy never would — every turn of an effort-latched session is served on one).
|
|
660
|
+
requestInfo._threadSink = { attemptAccount: null, replyId: null, account: null };
|
|
656
661
|
// Request clock: the direct client-gone line ("Client left after Xs" — request
|
|
657
662
|
// age) and the retry-same patience rule read it. (The link path reports time on
|
|
658
663
|
// THIS account, from lease.startedAt.) Was never set, so every direct-path
|
|
@@ -730,6 +735,15 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
730
735
|
}
|
|
731
736
|
} finally {
|
|
732
737
|
clearInterval(reaperTimer);
|
|
738
|
+
// THREAD OWNERSHIP, recorded at terminal release: the reply id captured during
|
|
739
|
+
// the response (message_start for streaming, body id for non-streaming) belongs
|
|
740
|
+
// to the account that SERVED the turn — pin it so the next continue is judged
|
|
741
|
+
// by thread identity. Only a served (2xx-5xx upstream answer) turn reaches here
|
|
742
|
+
// with _threadReplyId set; refusals return before the fetch.
|
|
743
|
+
const sink = requestInfo?._threadSink;
|
|
744
|
+
if (sink?.replyId && sink.account) {
|
|
745
|
+
threadOwners.noteThreadId(sink.replyId, sink.account);
|
|
746
|
+
}
|
|
733
747
|
accountManager.noteClientOutcome?.(requestInfo, ctx.status);
|
|
734
748
|
hooks.onRequestEnd?.(reqId, {
|
|
735
749
|
method: req.method, path: req.url,
|
|
@@ -1421,6 +1435,7 @@ async function forwardRequest(
|
|
|
1421
1435
|
if (upRes.status < 300) {
|
|
1422
1436
|
// The conversation now lives on this row (first REAL byte), and its usage
|
|
1423
1437
|
// is read off the relayed bytes — figures only, never a booking.
|
|
1438
|
+
accountManager.stampMovedFrom?.(requestInfo);
|
|
1424
1439
|
accountManager.noteConversationServed?.(requestInfo, account);
|
|
1425
1440
|
tapLinkUsage(value, requestInfo);
|
|
1426
1441
|
}
|
|
@@ -1539,7 +1554,15 @@ async function forwardRequest(
|
|
|
1539
1554
|
}
|
|
1540
1555
|
// This account is about to serve the turn, so it holds the thread from here on.
|
|
1541
1556
|
// Recorded optimistically: if the turn fails, the next one is refused anyway.
|
|
1557
|
+
// The reply's own message id is captured during the response (message_start / body
|
|
1558
|
+
// head — see parseSSEEvent and the non-streaming read) and recorded at release, so
|
|
1559
|
+
// the next continue is judged by THREAD identity rather than by terminal.
|
|
1542
1560
|
threadOwners.noteServed(requestInfo.sessionKey, account.name, threadIntent);
|
|
1561
|
+
// Arm reply capture for THIS attempt only when the turn is threaded: a plain turn's
|
|
1562
|
+
// id names no thread, and recording it would push live thread holders out of the map.
|
|
1563
|
+
if (requestInfo._threadSink) {
|
|
1564
|
+
requestInfo._threadSink.attemptAccount = isThreadedIntent(threadIntent) ? account.name : null;
|
|
1565
|
+
}
|
|
1543
1566
|
|
|
1544
1567
|
// Build log sections
|
|
1545
1568
|
const logSections = [];
|
|
@@ -1635,7 +1658,7 @@ async function forwardRequest(
|
|
|
1635
1658
|
requestInfo._upstreamForwardStartAt = upstreamForwardStartAt;
|
|
1636
1659
|
requestInfo._firstUpstreamByteAt = Date.now();
|
|
1637
1660
|
// A 2xx answer: the conversation now lives on this account (never on a refusal).
|
|
1638
|
-
if (upstreamRes.status < 300) accountManager.noteConversationServed?.(requestInfo, account);
|
|
1661
|
+
if (upstreamRes.status < 300) { accountManager.stampMovedFrom?.(requestInfo); accountManager.noteConversationServed?.(requestInfo, account); }
|
|
1639
1662
|
// Response arrived — the pre-response leak window is over. Stop guarding for
|
|
1640
1663
|
// client-disconnect via abort (streamResponse handles mid-stream disconnects).
|
|
1641
1664
|
res.off('close', onClientClose);
|
|
@@ -3268,7 +3291,7 @@ function isContextLengthError(errorBody) {
|
|
|
3268
3291
|
return /exceeded model token limit|maximum context length|context length exceeded|context window (?:size )?(?:exceeded|too)|prompt is too long|input is too long|reduce the length of|too many (?:input )?tokens|request too large/i.test(errorBody);
|
|
3269
3292
|
}
|
|
3270
3293
|
|
|
3271
|
-
export const __serverTest = { nextStickyBusyWaitMs, readPinRefusal, sleepUnlessClientGone, stickyBusyWaitEligible, stickyBusyBudgetMs, usageFromSseData, usageFromBodyTail, tapLinkUsage, stickyKnobs: () => ({ PIN_BUSY_MAX_RETRY_AFTER_S, STICKY_BUSY_MAX_ATTEMPTS, STICKY_BUSY_MIN_BYTES, STICKY_BUSY_WAIT_CAP_MS }), hasEligibleRoute, resendAllowed, directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, parsePatience, computeDeadline, computeLinkHop, nonStreamFirstByteWait, nonStreamBodyWait, nonStreamRemainingMs, deadlineKnobs, deadlineErrorPayload, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
3294
|
+
export const __serverTest = { nextStickyBusyWaitMs, readPinRefusal, sleepUnlessClientGone, stickyBusyWaitEligible, stickyBusyBudgetMs, usageFromSseData, usageFromBodyTail, tapLinkUsage, parseSSEEvent, extractUsageFromBody, threadOwners: () => threadOwners, stickyKnobs: () => ({ PIN_BUSY_MAX_RETRY_AFTER_S, STICKY_BUSY_MAX_ATTEMPTS, STICKY_BUSY_MIN_BYTES, STICKY_BUSY_WAIT_CAP_MS }), hasEligibleRoute, resendAllowed, directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, parsePatience, computeDeadline, computeLinkHop, nonStreamFirstByteWait, nonStreamBodyWait, nonStreamRemainingMs, deadlineKnobs, deadlineErrorPayload, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
3272
3295
|
|
|
3273
3296
|
async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
|
|
3274
3297
|
if (!upstreamRes.body) return '';
|
|
@@ -5295,23 +5318,38 @@ function tapLinkUsage(chunk, requestInfo) {
|
|
|
5295
5318
|
}
|
|
5296
5319
|
}
|
|
5297
5320
|
|
|
5321
|
+
/** Record a threaded reply's id against the account serving THIS attempt (armed at the
|
|
5322
|
+
* gate). Id and account are stored together, so a retry that moves accounts after an
|
|
5323
|
+
* earlier attempt captured an id can never pin that id to the wrong account. */
|
|
5324
|
+
function captureThreadReply(requestInfo, id) {
|
|
5325
|
+
const sink = requestInfo?._threadSink;
|
|
5326
|
+
if (!sink?.attemptAccount) return;
|
|
5327
|
+
sink.replyId = id;
|
|
5328
|
+
sink.account = sink.attemptAccount;
|
|
5329
|
+
}
|
|
5330
|
+
|
|
5298
5331
|
function parseSSEEvent(event, accountIndex, accountManager, requestInfo = {}) {
|
|
5299
5332
|
const dataLine = event.split('\n').find(l => l.startsWith('data: '));
|
|
5300
5333
|
if (!dataLine) return;
|
|
5301
5334
|
|
|
5302
5335
|
try {
|
|
5303
5336
|
const data = JSON.parse(dataLine.slice(6));
|
|
5337
|
+
// THREAD OWNERSHIP: message_start carries the reply's own id — the id the client
|
|
5338
|
+
// will continue from (previous_message_id). Capture it on the SERVING account so
|
|
5339
|
+
// the gate can judge the next continue by thread identity instead of by terminal.
|
|
5340
|
+
if (data?.type === 'message_start' && typeof data.message?.id === 'string') {
|
|
5341
|
+
captureThreadReply(requestInfo, data.message.id);
|
|
5342
|
+
}
|
|
5304
5343
|
// CAPACITY LEDGER (2026-08-22): stream-level usage feeds the per-cycle ledger.
|
|
5305
5344
|
// M3: Anthropic interim message_delta usage is CUMULATIVE — add-semantics inflates
|
|
5306
5345
|
// output on long generations. Track a per-stream RUNNING MAX, accrue once at end.
|
|
5307
5346
|
// M4: count_tokens responses are prompt-size echoes, not delivered work — skip.
|
|
5347
|
+
// Per-account / per-caller totals are booked ONCE per request at the terminal
|
|
5348
|
+
// release, off the running-max capacity book (finding 3, 2026-11-17) — booking
|
|
5349
|
+
// here would double-count hops and re-add cumulative deltas.
|
|
5308
5350
|
const usage = requestInfo.isCountTokens ? null : usageFromSseData(data);
|
|
5309
5351
|
if (usage) {
|
|
5310
|
-
|
|
5311
|
-
const n = isDelta ? usage.output_tokens : usage.input_tokens;
|
|
5312
|
-
accountManager.updateUsage(accountIndex, isDelta ? 0 : n, isDelta ? n : 0);
|
|
5313
|
-
accountManager.countCallerTokens?.(requestInfo.caller, n);
|
|
5314
|
-
noteCapacityUsage(requestInfo, usage, isDelta);
|
|
5352
|
+
noteCapacityUsage(requestInfo, usage, data.type === 'message_delta');
|
|
5315
5353
|
}
|
|
5316
5354
|
if (sseEventContainsThinking(data)) {
|
|
5317
5355
|
accountManager.markSessionThinkingProtected?.(requestInfo.sessionKey, requestInfo.model);
|
|
@@ -5361,9 +5399,11 @@ function classifyHeldStreamPrefix(prefix) {
|
|
|
5361
5399
|
function extractUsageFromBody(buffer, accountIndex, accountManager, requestInfo = {}) {
|
|
5362
5400
|
try {
|
|
5363
5401
|
const json = JSON.parse(buffer.toString());
|
|
5402
|
+
// THREAD OWNERSHIP: a non-streaming reply's own id is the next continue's anchor.
|
|
5403
|
+
if (typeof json.id === 'string') captureThreadReply(requestInfo, json.id);
|
|
5364
5404
|
if (json.usage) {
|
|
5365
|
-
|
|
5366
|
-
|
|
5405
|
+
// Per-account / per-caller totals book once per request at the terminal release
|
|
5406
|
+
// (finding 3, 2026-11-17) — off the running-max capacity figures booked here.
|
|
5367
5407
|
// CAPACITY LEDGER — M4: a /count_tokens response is a prompt-size echo with ZERO
|
|
5368
5408
|
// work delivered. Claude Code calls it constantly; counting it would inflate every
|
|
5369
5409
|
// cycle by whole prompt sizes.
|
package/src/thread-gate.js
CHANGED
|
@@ -27,7 +27,10 @@ export function isThreadlessAccount(account) {
|
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
/** What kind of thread intent a request body carries. Cheap: only the head of the body
|
|
30
|
-
* is JSON-parsed, and a non-JSON body is simply 'none'.
|
|
30
|
+
* is JSON-parsed, and a non-JSON body is simply 'none'. A `continue` also carries the
|
|
31
|
+
* id of the reply it follows — the ONE identifier that names the thread itself, so the
|
|
32
|
+
* gate returns it (session keys are per terminal and sibling agents inside one terminal
|
|
33
|
+
* hold threads on different accounts; 2026-11-17: 60 of 109 thread 404s). */
|
|
31
34
|
export function readThreadIntent(body) {
|
|
32
35
|
try {
|
|
33
36
|
const j = JSON.parse(body.toString('utf8'));
|
|
@@ -36,7 +39,10 @@ export function readThreadIntent(body) {
|
|
|
36
39
|
// `previous_message_id` can also ride in `diagnostics`; that alone is not a thread.
|
|
37
40
|
return { kind: 'none' };
|
|
38
41
|
}
|
|
39
|
-
if (t.type === 'continue')
|
|
42
|
+
if (t.type === 'continue') {
|
|
43
|
+
const previousMessageId = typeof t.previous_message_id === 'string' ? t.previous_message_id : null;
|
|
44
|
+
return { kind: 'continue', previousMessageId };
|
|
45
|
+
}
|
|
40
46
|
if (t.type === 'create') return { kind: 'create' };
|
|
41
47
|
return { kind: 'none' };
|
|
42
48
|
} catch {
|
|
@@ -44,6 +50,11 @@ export function readThreadIntent(body) {
|
|
|
44
50
|
}
|
|
45
51
|
}
|
|
46
52
|
|
|
53
|
+
/** Whether a turn carries thread state at all — only these replies name a thread. */
|
|
54
|
+
export function isThreadedIntent(intent) {
|
|
55
|
+
return intent?.kind === 'create' || intent?.kind === 'continue';
|
|
56
|
+
}
|
|
57
|
+
|
|
47
58
|
/** The exact body the client's classifier reads. `details.error_code` is the field it
|
|
48
59
|
* keys on; the message is free text and is never shown to a person. */
|
|
49
60
|
export function threadRefusalBody(accountName) {
|
|
@@ -60,6 +71,11 @@ export function threadRefusalBody(accountName) {
|
|
|
60
71
|
// Sessions whose last threaded turn we served, and how many times we have refused them.
|
|
61
72
|
// Two short strings per entry; bounded and LRU-evicted.
|
|
62
73
|
const MAX_SESSIONS = 500;
|
|
74
|
+
// Reply ids are one per threaded TURN (every superseded id is kept: a client retry or a
|
|
75
|
+
// rewind may still continue it), so the thread map needs a turn-volume bound, not a
|
|
76
|
+
// session bound: 500 was exhausted in ~10-20 min on the shared cloud pool and a busy
|
|
77
|
+
// thread's holder fell back to the per-terminal guess. ~100 bytes an entry → ~2 MB.
|
|
78
|
+
const MAX_THREADS = 20000;
|
|
63
79
|
// A session that keeps sending threaded turns after being refused is one whose client
|
|
64
80
|
// did NOT take the downgrade (a different agent id, a model switch, an older build).
|
|
65
81
|
// Refusing forever would double its request volume, so stop and forward instead.
|
|
@@ -69,9 +85,16 @@ const MAX_CONSECUTIVE_REFUSALS = 2;
|
|
|
69
85
|
const NO_SESSION_PREFIX = '\u0000nosession:';
|
|
70
86
|
|
|
71
87
|
export class ThreadOwners {
|
|
72
|
-
constructor({ maxSessions = MAX_SESSIONS, maxRefusals = MAX_CONSECUTIVE_REFUSALS } = {}) {
|
|
88
|
+
constructor({ maxSessions = MAX_SESSIONS, maxThreads = MAX_THREADS, maxRefusals = MAX_CONSECUTIVE_REFUSALS } = {}) {
|
|
73
89
|
this.map = new Map(); // sessionKey -> { owner, refusals }
|
|
90
|
+
// reply message id -> owning account name. The THREAD is keyed by the reply id the
|
|
91
|
+
// client sends back as previous_message_id — a session key is per TERMINAL, and
|
|
92
|
+
// sibling agents inside one terminal hold threads on different accounts, so the
|
|
93
|
+
// per-terminal entry both mis-serves siblings and never expires per thread.
|
|
94
|
+
// Bounded LRU, same lifetime class as the per-session map above it.
|
|
95
|
+
this.threads = new Map();
|
|
74
96
|
this.maxSessions = maxSessions;
|
|
97
|
+
this.maxThreads = maxThreads;
|
|
75
98
|
this.maxRefusals = maxRefusals;
|
|
76
99
|
}
|
|
77
100
|
|
|
@@ -87,6 +110,23 @@ export class ThreadOwners {
|
|
|
87
110
|
while (this.map.size > this.maxSessions) this.map.delete(this.map.keys().next().value);
|
|
88
111
|
}
|
|
89
112
|
|
|
113
|
+
/** Record which account produced a threaded reply (the id the client will continue
|
|
114
|
+
* from). Streaming: the message_start event's message.id; non-streaming: the body's
|
|
115
|
+
* leading id. Called on every served turn that carried thread intent — a `create`
|
|
116
|
+
* starts a thread, a `continue` extends it onto whoever served the follow-up. */
|
|
117
|
+
noteThreadId(threadId, accountName) {
|
|
118
|
+
if (!threadId || !accountName) return;
|
|
119
|
+
this.threads.delete(threadId);
|
|
120
|
+
this.threads.set(threadId, accountName);
|
|
121
|
+
while (this.threads.size > this.maxThreads) this.threads.delete(this.threads.keys().next().value);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** The account that holds `threadId`, if we saw its reply. Null = unknown. */
|
|
125
|
+
threadOwner(threadId) {
|
|
126
|
+
const v = threadId ? this.threads.get(threadId) : undefined;
|
|
127
|
+
return v ?? null;
|
|
128
|
+
}
|
|
129
|
+
|
|
90
130
|
/** Decide AFTER routing has chosen. Returns true only for a `continue` turn that the
|
|
91
131
|
* chosen account cannot serve, and only while refusals are still under the bound.
|
|
92
132
|
*
|
|
@@ -106,14 +146,32 @@ export class ThreadOwners {
|
|
|
106
146
|
shouldRefuse(sessionKey, accountName, intent, isProvider = false) {
|
|
107
147
|
if (!accountName) return false;
|
|
108
148
|
if (intent?.kind !== 'continue') return false; // `create` carries the full transcript
|
|
109
|
-
if (isProvider) return true; // can never serve an Anthropic thread
|
|
149
|
+
if (isProvider) return true; // can never serve an Anthropic thread — even as the recorded holder
|
|
150
|
+
// THREAD identity first: the previous_message_id names the exact thread being
|
|
151
|
+
// continued. When we know its holder, the answer is exact (the holder passes,
|
|
152
|
+
// everyone else is refused) and never falls through to the terminal-keyed guess —
|
|
153
|
+
// which is what mis-served sibling agents inside one terminal (2026-11-17).
|
|
154
|
+
const key = sessionKey || `${NO_SESSION_PREFIX}${accountName}`;
|
|
155
|
+
if (intent?.previousMessageId) {
|
|
156
|
+
const holder = this.threads.get(intent.previousMessageId);
|
|
157
|
+
if (holder != null) {
|
|
158
|
+
this.noteThreadId(intent.previousMessageId, holder); // LRU bump: a live thread stays
|
|
159
|
+
if (holder === accountName) return false;
|
|
160
|
+
// A known non-holder is refused under the SAME per-session bound as an unknown
|
|
161
|
+
// owner. Past it we forward: the real 404 makes the client replay with
|
|
162
|
+
// thread:create and KEEP threading, where our 400 would downgrade that agent to
|
|
163
|
+
// stateless for the whole session (Claude Code 2.1.325) — unbounded refusals
|
|
164
|
+
// would downgrade every sibling after a terminal's account moves.
|
|
165
|
+
const entry = this._touch(key);
|
|
166
|
+
return !(entry && entry.refusals >= this.maxRefusals);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
110
169
|
// A request with NO session header is invisible to ownership tracking, and measured
|
|
111
170
|
// 2026-09-11 those are the majority of traffic — 14 of 20 consecutive /v1/messages
|
|
112
171
|
// lines carried no `[sess …]`. Skipping them left threaded turns reaching GLM and
|
|
113
172
|
// failing exactly as before the gate existed. We cannot know the owner, so treat it
|
|
114
173
|
// as an unknown session (refuse, the safe direction) and bucket the refusal COUNT
|
|
115
174
|
// per account so the storm bound still applies.
|
|
116
|
-
const key = sessionKey || `${NO_SESSION_PREFIX}${accountName}`;
|
|
117
175
|
const entry = this._touch(key);
|
|
118
176
|
if (entry && entry.owner === accountName) return false; // the account that holds it
|
|
119
177
|
if (entry && entry.refusals >= this.maxRefusals) return false; // bounded fail-open
|
|
@@ -129,10 +187,13 @@ export class ThreadOwners {
|
|
|
129
187
|
}
|
|
130
188
|
|
|
131
189
|
/** Record that `accountName` served a threaded turn for this session — it now holds
|
|
132
|
-
* the thread. Any refusal streak ends here.
|
|
133
|
-
|
|
190
|
+
* the thread. Any refusal streak ends here. The reply's own message id (when the
|
|
191
|
+
* caller captured it from the response) is recorded as the newest link of the
|
|
192
|
+
* thread, so the NEXT continue is judged by thread identity, not by terminal. */
|
|
193
|
+
noteServed(sessionKey, accountName, intent, replyMessageId = null) {
|
|
134
194
|
if (!accountName) return;
|
|
135
195
|
if (intent?.kind !== 'create' && intent?.kind !== 'continue') return;
|
|
196
|
+
if (replyMessageId) this.noteThreadId(replyMessageId, accountName);
|
|
136
197
|
// Without a session header there is no conversation to attribute ownership to; only
|
|
137
198
|
// clear the per-account refusal streak so a served turn re-arms the bound.
|
|
138
199
|
const key = sessionKey || `${NO_SESSION_PREFIX}${accountName}`;
|
package/src/tui.js
CHANGED
|
@@ -1180,7 +1180,16 @@ export class TUI {
|
|
|
1180
1180
|
/** EFFICIENCY line (2026-11-15), directly under Routing: is the prompt cache holding,
|
|
1181
1181
|
* and what are moves costing — Claude and GLM never blended. null with no traffic.
|
|
1182
1182
|
* Colours are WARN-only proposals until a one-day baseline exists; a red names the
|
|
1183
|
-
* cause and the account.
|
|
1183
|
+
* cause and the account.
|
|
1184
|
+
* Honest-efficiency rework (2026-11-17, audit findings 2-6):
|
|
1185
|
+
* - hit% never travels alone: writeShare (1.25·cc / 0.1·cr+1.25·cc+in) rides beside
|
|
1186
|
+
* it — a 91% hit line can carry writes worth half the spend, and that is the number
|
|
1187
|
+
* that hid the cold-hop waste. Yellow >40%, red >60% (Claude).
|
|
1188
|
+
* - moves split cross-family (the prompt re-wrote wholesale; drives the red alarm)
|
|
1189
|
+
* vs same-family (near-free: z.ai's cache is shared across its accounts).
|
|
1190
|
+
* - miss is per provider — GLM and Claude tokenizers differ ~1.6x, a blended sum is
|
|
1191
|
+
* unit-inconsistent.
|
|
1192
|
+
* - waits say what they count: busy-bounce waits only (queue holds book 0 here). */
|
|
1184
1193
|
_efficiencyLine(W = 80, now = Date.now()) {
|
|
1185
1194
|
const eff = this.am.efficiencyOverall?.('last15m', now);
|
|
1186
1195
|
if (!eff) return null;
|
|
@@ -1193,31 +1202,40 @@ export class TUI {
|
|
|
1193
1202
|
if (c.lowVolume && g.lowVolume) return truncLine(` Cache ${dim('— (little traffic)')}${since}`, W);
|
|
1194
1203
|
const pctTxt = x => `${Math.round(x * 100)}%`;
|
|
1195
1204
|
const parts = [];
|
|
1196
|
-
if (c.hit != null) parts.push(`Claude ${c.lowVolume ? dim(pctTxt(c.hit)) : pctTxt(c.hit)}`);
|
|
1205
|
+
if (c.hit != null) parts.push(`Claude ${c.lowVolume ? dim(pctTxt(c.hit)) : pctTxt(c.hit)}${c.writeShare != null ? ` (w/ ${pctTxt(c.writeShare)} of cost)` : ''}`);
|
|
1197
1206
|
if (g.hit != null) parts.push(`GLM ${g.lowVolume ? dim(pctTxt(g.hit)) : pctTxt(g.hit)}`);
|
|
1198
|
-
const
|
|
1199
|
-
|
|
1207
|
+
const missTxt = t => t >= 1e6 ? `${(t / 1e6).toFixed(1)}M` : `${Math.round(t / 1000)}k`;
|
|
1208
|
+
if (c.hit != null || c.missTokens) parts.push(`Claude miss ${missTxt(c.missTokens)} tok`);
|
|
1209
|
+
if (g.hit != null || g.missTokens) parts.push(`GLM miss ${missTxt(g.missTokens)} tok`);
|
|
1200
1210
|
const cause = k => c.movedByCause[k] + g.movedByCause[k];
|
|
1201
1211
|
const per100 = n => (served ? Math.round(n * 100 / served) : 0);
|
|
1202
|
-
|
|
1212
|
+
const cross = (c.movedCross || 0) + (g.movedCross || 0);
|
|
1213
|
+
// Causes list only what happened — a row of zeros is width the line cannot spare.
|
|
1214
|
+
const causeTxt = ['busy', 'balance', 'wall', 'link', 'other'].filter(k => cause(k) > 0).map(k => ` · ${k} ${cause(k)}`).join('');
|
|
1215
|
+
parts.push(`cross-moves ${per100(cross)}/100 (same ${(c.movedSame || 0) + (g.movedSame || 0)}${causeTxt})`);
|
|
1203
1216
|
const bounceShare = bounced / (served + bounced);
|
|
1204
1217
|
if (bounced) parts.push(`bounced ${pctTxt(bounceShare)}`);
|
|
1218
|
+
const glmBusy = g.movedByCause.busy || 0;
|
|
1219
|
+
if (glmBusy) parts.push(`glm busy-bounced ${glmBusy}`);
|
|
1205
1220
|
const waited = c.waited + g.waited;
|
|
1206
|
-
if (waited) parts.push(`
|
|
1221
|
+
if (waited) parts.push(`busy-bounce waits ${waited} (avg ${((c.waitMs + g.waitMs) / waited / 1000).toFixed(1)}s)`);
|
|
1207
1222
|
|
|
1208
1223
|
// RED — the first cause that applies, named with the account behind it.
|
|
1209
|
-
const forced = cause('busy') + cause('wall') + cause('link');
|
|
1210
1224
|
const wtm = c.waitThenMoved + g.waitThenMoved;
|
|
1211
1225
|
let red_ = null;
|
|
1212
1226
|
if (!c.lowVolume && c.hit != null && c.hit < 0.5) red_ = `Claude ${pctTxt(c.hit)} on ${this._worstCacheAccount() || '?'}`;
|
|
1213
|
-
else if (
|
|
1227
|
+
else if (!c.lowVolume && c.writeShare != null && c.writeShare > 0.6) red_ = `write share ${pctTxt(c.writeShare)} of cost on ${this._worstCacheAccount() || '?'}`;
|
|
1228
|
+
else if (per100(cross) > 2) red_ = `cross moves ${per100(cross)}/100 — most refusals on ${this._mostBouncedAccount(now) || '?'}`;
|
|
1214
1229
|
else if (bounceShare > 0.1) red_ = `${pctTxt(bounceShare)} bounced — most on ${this._mostBouncedAccount(now) || '?'}`;
|
|
1215
1230
|
else if (wtm * 100 / Math.max(1, served) > 0.5) red_ = `${wtm} waits still moved — on ${this._mostBouncedAccount(now) || '?'}`;
|
|
1216
1231
|
if (red_) return truncLine(` ${red('Cache')} ${red(`⚠ ${red_}`)} ${dim('·')} ${parts.join(' · ')} ${dim('[15m]')}${since}`, W);
|
|
1217
|
-
// YELLOW — Claude below 70%,
|
|
1232
|
+
// YELLOW — Claude below 70%, write share over 40% of cost, drifting 15+ points
|
|
1233
|
+
// under its hour, or waits are common.
|
|
1218
1234
|
const hour = this.am.efficiencyOverall('last1h', now).claude;
|
|
1219
1235
|
const drift = !c.lowVolume && !hour.lowVolume && c.hit != null && hour.hit != null && hour.hit - c.hit >= 0.15;
|
|
1220
|
-
const warn = (!c.lowVolume && c.hit != null && c.hit < 0.7)
|
|
1236
|
+
const warn = (!c.lowVolume && c.hit != null && c.hit < 0.7)
|
|
1237
|
+
|| (!c.lowVolume && c.writeShare != null && c.writeShare > 0.4)
|
|
1238
|
+
|| drift || waited > 0.25 * served;
|
|
1221
1239
|
return truncLine(` ${warn ? yellow('Cache') : green('Cache')} ${parts.join(' · ')} ${dim('[15m]')}${since}`, W);
|
|
1222
1240
|
}
|
|
1223
1241
|
|
|
@@ -3396,9 +3414,10 @@ export class TUI {
|
|
|
3396
3414
|
|
|
3397
3415
|
const merged = new Map();
|
|
3398
3416
|
const addWin = (name, w) => {
|
|
3399
|
-
const cur = merged.get(name) || { h24: 0, h24p: 0, d7: 0, d7p: 0, d30: 0, d30p: 0, total: 0, tok7: 0 };
|
|
3417
|
+
const cur = merged.get(name) || { h24: 0, h24p: 0, d7: 0, d7p: 0, d30: 0, d30p: 0, total: 0, tok7: 0, paid7: 0 };
|
|
3400
3418
|
for (const k of ['h24', 'h24p', 'd7', 'd7p', 'd30', 'd30p', 'total']) cur[k] += (w[k]?.req ?? 0);
|
|
3401
3419
|
cur.tok7 += (w.d7?.tok ?? 0);
|
|
3420
|
+
cur.paid7 += (w.d7?.paid ?? 0);
|
|
3402
3421
|
merged.set(name, cur);
|
|
3403
3422
|
};
|
|
3404
3423
|
for (const [n, w] of this.am.callerWindowStats?.() || []) addWin(n, w);
|
|
@@ -3422,22 +3441,23 @@ export class TUI {
|
|
|
3422
3441
|
};
|
|
3423
3442
|
const CW = 9;
|
|
3424
3443
|
const header = ' ' + 'person'.padEnd(10)
|
|
3425
|
-
+ ['24h', 'Δ24h', '7d', 'Δ7d', '30d', 'Δ30d', 'total', '~7d tok'].map(c => c.padStart(CW)).join('');
|
|
3444
|
+
+ ['24h', 'Δ24h', '7d', 'Δ7d', '30d', 'Δ30d', 'total', '~7d tok', '~7d paid'].map(c => c.padStart(CW)).join('');
|
|
3426
3445
|
out.push(dimUnderline(fitLine(header, W)));
|
|
3427
3446
|
|
|
3428
3447
|
const rows = [...merged.entries()].sort((a, b) => b[1].d7 - a[1].d7);
|
|
3429
|
-
const tot = { h24: 0, h24p: 0, d7: 0, d7p: 0, d30: 0, d30p: 0, total: 0, tok7: 0 };
|
|
3448
|
+
const tot = { h24: 0, h24p: 0, d7: 0, d7p: 0, d30: 0, d30p: 0, total: 0, tok7: 0, paid7: 0 };
|
|
3430
3449
|
for (const [name, w] of rows) {
|
|
3431
3450
|
for (const k of Object.keys(tot)) tot[k] += w[k];
|
|
3432
3451
|
out.push(' ' + name.slice(0, 10).padEnd(10)
|
|
3433
|
-
+ [fmt(w.h24), delta(w.h24, w.h24p), fmt(w.d7), delta(w.d7, w.d7p), fmt(w.d30), delta(w.d30, w.d30p), fmt(w.total), fmtTok(w.tok7)]
|
|
3452
|
+
+ [fmt(w.h24), delta(w.h24, w.h24p), fmt(w.d7), delta(w.d7, w.d7p), fmt(w.d30), delta(w.d30, w.d30p), fmt(w.total), fmtTok(w.tok7), fmtTok(w.paid7)]
|
|
3434
3453
|
.map(c => lpadVis(String(c), CW)).join(''));
|
|
3435
3454
|
}
|
|
3436
3455
|
out.push(dim(' ' + 'total'.padEnd(10)
|
|
3437
|
-
+ [fmt(tot.h24), delta(tot.h24, tot.h24p), fmt(tot.d7), delta(tot.d7, tot.d7p), fmt(tot.d30), delta(tot.d30, tot.d30p), fmt(tot.total), fmtTok(tot.tok7)]
|
|
3456
|
+
+ [fmt(tot.h24), delta(tot.h24, tot.h24p), fmt(tot.d7), delta(tot.d7, tot.d7p), fmt(tot.d30), delta(tot.d30, tot.d30p), fmt(tot.total), fmtTok(tot.tok7), fmtTok(tot.paid7)]
|
|
3438
3457
|
.map(c => lpadVis(String(c), CW)).join('')));
|
|
3439
3458
|
|
|
3440
3459
|
const link = (this.am.poolLinks || []).find(l => l.lastOkAt);
|
|
3460
|
+
out.push(dim(' tok = full prompt (input + cache read + cache write) + output, provider-native · paid = cache write + uncached input'));
|
|
3441
3461
|
out.push(dim(` rolling windows · restarts included${link ? ` · team pool synced ${new Date(link.lastOkAt).toLocaleTimeString()}` : ''}`));
|
|
3442
3462
|
return out;
|
|
3443
3463
|
}
|
|
@@ -3590,10 +3610,14 @@ export class TUI {
|
|
|
3590
3610
|
out.push('');
|
|
3591
3611
|
out.push(' ' + dim('Capacity = median of tokens ÷ % full across readings in each cycle. Med-3 = median of last 3 cycles, Med-all = every cycle.'));
|
|
3592
3612
|
out.push(' ' + dim('One row per account. Where = which side holds its history: cloud, laptop or both. ? = identity unknown.'));
|
|
3593
|
-
|
|
3613
|
+
// Axis honesty (audit 2026-11-17, finding 5): the figure counts processed tokens in
|
|
3614
|
+
// EACH provider's own tokenizer (GLM ≈ 0.6x Anthropic for the same prompt), and a
|
|
3615
|
+
// cache read (0.1x) weighs the same as a cache write (1.25x). So $/1M across a
|
|
3616
|
+
// Claude and a GLM row is not a like-for-like price per Claude-equivalent token.
|
|
3617
|
+
out.push(' ' + dim('Tokens = processed tokens (provider-native): input + cached + output, counted in each provider\'s own tokenizer.'));
|
|
3594
3618
|
out.push(' ' + dim('N = cycles in Med-all (Med-3 uses the newest 3). N* = includes a cycle both pools saw (merged) — estimated from close-time tokens, less precise.'));
|
|
3595
3619
|
out.push(' ' + dim('Current @% = live window, % full now · no-weekly weekly = avg 5h × 33.6.'));
|
|
3596
|
-
out.push(' ' + dim('$/1M = plan price ÷ measured weekly tank (monthly-normalized). -- = no price or tank yet.'));
|
|
3620
|
+
out.push(' ' + dim('$/1M = plan price ÷ measured weekly tank (monthly-normalized), per processed tokens (provider-native). -- = no price or tank yet.'));
|
|
3597
3621
|
return out;
|
|
3598
3622
|
}
|
|
3599
3623
|
|