maxpool 1.24.95 → 1.24.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -0
- package/package.json +1 -1
- package/src/account-manager.js +456 -29
- package/src/index.js +57 -4
- package/src/prober.js +11 -2
- package/src/server.js +266 -25
- package/src/tui.js +232 -61
- package/src/updater.js +7 -2
- package/src/usage-label.js +50 -0
package/src/index.js
CHANGED
|
@@ -20,7 +20,8 @@ import { setEventLogPath, installConsoleMirror, setConsoleStdoutSuppressed, setC
|
|
|
20
20
|
import { sendLoginToPool, planCliLogin, resolveUsablePools } from './cloud-login.js';
|
|
21
21
|
import { SleepGuard } from './sleep-guard.js';
|
|
22
22
|
import { controllingTerminalAlive } from './terminal-alive.js';
|
|
23
|
-
import { AccountManager } from './account-manager.js';
|
|
23
|
+
import { AccountManager, combineEfficiency } from './account-manager.js';
|
|
24
|
+
import { cacheUsageLabel } from './usage-label.js';
|
|
24
25
|
import { createProxyServer, REQUEST_IDLE_MAX_MS } from './server.js';
|
|
25
26
|
|
|
26
27
|
// ── Happy Eyeballs: stop racing a dead IPv6 leg ────────────────────────────────
|
|
@@ -1089,7 +1090,8 @@ async function serverWorkerCommand() {
|
|
|
1089
1090
|
// Headless (service / shared VM) pools have no TUI log buffer, so named-caller
|
|
1090
1091
|
// attribution would otherwise reach no log at all. Only named callers are
|
|
1091
1092
|
// logged here — a solo headless pool stays as quiet as before.
|
|
1092
|
-
|
|
1093
|
+
const cache = cacheUsageLabel(info.cacheUsage);
|
|
1094
|
+
appendEventLog(`[Maxpool] ${info.caller} · ${info.method} ${info.path} → ${info.account || '?'} (${info.status})${info.conversation ? ` [cv ${info.conversation.slice(0, 8)}]` : ''}${cache ? ` ${cache}` : ''}`);
|
|
1093
1095
|
}
|
|
1094
1096
|
},
|
|
1095
1097
|
};
|
|
@@ -1395,7 +1397,8 @@ async function serverWorkerCommand() {
|
|
|
1395
1397
|
// in memory but silently reverts on the next config write / restart. Merge
|
|
1396
1398
|
// onto the existing disk scheduler block so other scheduler keys survive.
|
|
1397
1399
|
if (config.scheduler?.crossProviderFallbackPolicy || config.scheduler?.providers
|
|
1398
|
-
|| config.scheduler?.weeklyAwareScoring !== undefined
|
|
1400
|
+
|| config.scheduler?.weeklyAwareScoring !== undefined
|
|
1401
|
+
|| config.scheduler?.stickyBusyBudgetMs !== undefined) {
|
|
1399
1402
|
diskConfig.scheduler = {
|
|
1400
1403
|
...diskConfig.scheduler,
|
|
1401
1404
|
...(config.scheduler.crossProviderFallbackPolicy
|
|
@@ -1408,6 +1411,9 @@ async function serverWorkerCommand() {
|
|
|
1408
1411
|
// default on the next restart.
|
|
1409
1412
|
...(config.scheduler.weeklyAwareScoring !== undefined
|
|
1410
1413
|
? { weeklyAwareScoring: config.scheduler.weeklyAwareScoring } : {}),
|
|
1414
|
+
// Wait for busy account (TUI routing b) — same reason as above.
|
|
1415
|
+
...(config.scheduler.stickyBusyBudgetMs !== undefined
|
|
1416
|
+
? { stickyBusyBudgetMs: config.scheduler.stickyBusyBudgetMs } : {}),
|
|
1411
1417
|
};
|
|
1412
1418
|
}
|
|
1413
1419
|
// Persist live-toggled automatic-update flags (the TUI 'u' Updates menu). Same
|
|
@@ -2230,13 +2236,25 @@ async function statusCommand() {
|
|
|
2230
2236
|
console.log(`Anthropic: temporarily throttled (${state}, queued ${queued}${oldest})\n`);
|
|
2231
2237
|
}
|
|
2232
2238
|
|
|
2239
|
+
for (const line of efficiencyLines(data)) console.log(line);
|
|
2240
|
+
|
|
2233
2241
|
for (const acct of data.accounts) {
|
|
2234
|
-
|
|
2242
|
+
// A benched duplicate exports quota: null (its local numbers are stale by design —
|
|
2243
|
+
// the cloud owns the live ones). {} keeps every quota read below safe.
|
|
2244
|
+
const q = acct.quota || {};
|
|
2235
2245
|
const current = acct.name === data.currentAccount ? ' *' : '';
|
|
2236
2246
|
|
|
2237
2247
|
console.log(` ${acct.name} (${acct.type})${current}`);
|
|
2238
2248
|
console.log(` Status: ${acct.enabled === false ? 'disabled' : acct.status}`);
|
|
2239
2249
|
console.log(` Load: current ${acct.load?.current?.inFlight || 0}/${acct.load?.current?.activeWeight || 0} weight, 15m ${formatLoadWindow(acct.load?.last15m)}, 1h ${formatLoadWindow(acct.load?.last1h)}`);
|
|
2250
|
+
if (acct.quotaSource === 'cloud-mirror') {
|
|
2251
|
+
console.log(' Quota: from team pool (this local copy is benched)');
|
|
2252
|
+
}
|
|
2253
|
+
// Cloud rows merge server-side: cache + served from the serving pool's export
|
|
2254
|
+
// (team-wide on that account), bounced from this pool's own refusals on it.
|
|
2255
|
+
const eff = acct.load?.last1h?.efficiency;
|
|
2256
|
+
const team = acct.remote && acct.teamEfficiency?.last1h ? ' (team)' : '';
|
|
2257
|
+
if (eff && (eff.served || eff.bounced)) console.log(` Cache: ${pctOrDash(eff.hit)} over 1h${team}, ${eff.served || 0} served, ${eff.bounced || 0} bounced`);
|
|
2240
2258
|
|
|
2241
2259
|
if (acct.type === 'provider') {
|
|
2242
2260
|
const last = acct.lastStatus ? `${acct.lastStatus} in ${formatDurationMs(acct.lastResponseMs)}` : '-';
|
|
@@ -2739,6 +2757,7 @@ function handleServerListenError(err, host, port) {
|
|
|
2739
2757
|
`[Maxpool] ${host}:${port} is already in use.`,
|
|
2740
2758
|
'Another Maxpool proxy may already be running.',
|
|
2741
2759
|
'Check the existing server with: maxpool status',
|
|
2760
|
+
'Dashboard: maxpool (attaches to the running service)',
|
|
2742
2761
|
`Find the listener with: lsof -nP -iTCP:${port} -sTCP:LISTEN`,
|
|
2743
2762
|
];
|
|
2744
2763
|
} else if (err.code === 'EACCES') {
|
|
@@ -2763,6 +2782,40 @@ function formatDurationMs(ms) {
|
|
|
2763
2782
|
return `${Math.floor(sec / 60)}m${String(Math.round(sec % 60)).padStart(2, '0')}s`;
|
|
2764
2783
|
}
|
|
2765
2784
|
|
|
2785
|
+
// A function declaration, not a const: commands dispatch before later consts initialise.
|
|
2786
|
+
function pctOrDash(x) { return x == null ? '—' : `${Math.round(x * 100)}%`; }
|
|
2787
|
+
|
|
2788
|
+
/** `maxpool status` Efficiency block. The server merges every row once (mirror rows:
|
|
2789
|
+
* cache + served from the serving pool's export, bounces / waits / moves / latency from
|
|
2790
|
+
* this pool's own events) — the same figures the TUI Cache line shows. With nothing
|
|
2791
|
+
* served or bounced in the window, rates read '—', never a measured-looking 0. */
|
|
2792
|
+
function efficiencyLines(data, key = 'last15m') {
|
|
2793
|
+
const accts = data.accounts || [];
|
|
2794
|
+
const overall = data.efficiency?.[key];
|
|
2795
|
+
if (!overall) return [];
|
|
2796
|
+
const claude = overall.claude || combineEfficiency([], 'claude');
|
|
2797
|
+
const glm = overall.glm || combineEfficiency([], 'glm');
|
|
2798
|
+
const served = claude.served + glm.served;
|
|
2799
|
+
const sum = (f) => (claude[f] || 0) + (glm[f] || 0);
|
|
2800
|
+
const cause = (c) => (claude.movedByCause?.[c] || 0) + (glm.movedByCause?.[c] || 0);
|
|
2801
|
+
const anyData = served + sum('bounced') > 0;
|
|
2802
|
+
const n = (x) => (anyData ? String(x) : '—');
|
|
2803
|
+
const failed = accts.reduce((t, a) => t + (a.load?.[key]?.failed || 0), 0);
|
|
2804
|
+
const linkDrops = accts.reduce((t, a) => t + (a.load?.[key]?.linkFailed || 0), 0);
|
|
2805
|
+
const co = data.clientOutcomes?.[key];
|
|
2806
|
+
const s = (ms) => (ms == null ? '—' : `${(ms / 1000).toFixed(1)}s`);
|
|
2807
|
+
return [
|
|
2808
|
+
`Efficiency (${key === 'last1h' ? '1h' : '15m'}):`,
|
|
2809
|
+
` Cache: Claude ${pctOrDash(claude.hit)} (miss ${(claude.missTokens / 1e6).toFixed(1)}M tok) · GLM ${pctOrDash(glm.hit)} (miss ${(glm.missTokens / 1e6).toFixed(1)}M tok)`,
|
|
2810
|
+
` Moves: ${served ? `${Math.round(sum('moved') * 100 / served)}/100 served` : '—'} (busy ${n(cause('busy'))} · balance ${n(cause('balance'))} · wall ${n(cause('wall'))} · link ${n(cause('link'))} · other ${n(cause('other'))})`,
|
|
2811
|
+
` Bounced: ${n(sum('bounced'))} (refused ${n(sum('refused'))}) · failed ${failed} · link drops ${linkDrops}`,
|
|
2812
|
+
` Waited: ${n(sum('waited'))}${sum('waited') ? ` (avg ${s(sum('waitMs') / sum('waited'))})` : ''} · waited then moved ${n(sum('waitThenMoved'))}`,
|
|
2813
|
+
` Client errors: ${co ? `${co.poolErrors + co.requestErrors} of ${co.total} (pool ${co.poolErrors} · request ${co.requestErrors})` : '—'}`,
|
|
2814
|
+
` First byte: p50 ${s(overall.e2eP50)} · p95 ${s(overall.e2eP95)} (this pool's clients, waits included)`,
|
|
2815
|
+
'',
|
|
2816
|
+
];
|
|
2817
|
+
}
|
|
2818
|
+
|
|
2766
2819
|
function formatLoadWindow(load = {}) {
|
|
2767
2820
|
const avg = load.avgMs != null ? ` avg ${formatDurationMs(load.avgMs)}` : '';
|
|
2768
2821
|
const failed = load.failed ? `, ${load.failed} failed` : '';
|
package/src/prober.js
CHANGED
|
@@ -109,9 +109,15 @@ export class Prober {
|
|
|
109
109
|
// a user often disables an account precisely BECAUSE it's exhausted, and still
|
|
110
110
|
// wants to see its quota recover — so keep refreshing its usage for visibility
|
|
111
111
|
// even though routing skips it. Do NOT add an enabled gate here.
|
|
112
|
-
// Skip
|
|
112
|
+
// Skip auth-dead accounts (dead refresh token): probing them just re-POSTs
|
|
113
113
|
// the rejected token every cycle — a 400 storm. They recover only on re-auth.
|
|
114
|
-
|
|
114
|
+
// Also skip BENCHED rows (`poolBenchedBy`): a local duplicate of an account the
|
|
115
|
+
// cloud team pool owns is never token-refreshed (account-manager
|
|
116
|
+
// ensureTokenFresh returns early so the cloud keeps the single-use refresh
|
|
117
|
+
// token), so its access token is dead and every probe fails — each failure
|
|
118
|
+
// widens the shared inter-probe gap and slows the HEALTHY rows. Its quota is
|
|
119
|
+
// shown from the cloud mirror (am.remoteRowFor), not from a local probe.
|
|
120
|
+
const oauth = this.am.accounts.filter(a => a.type === 'oauth' && a.credential && !a.refreshDead && !a.poolBenchedBy);
|
|
115
121
|
// Pool links are probed by their own PoolLink status poll, not here.
|
|
116
122
|
const providers = this.am.accounts.filter(a => a.type === 'provider' && a.credential); // 'pool' rows are excluded by type
|
|
117
123
|
|
|
@@ -413,6 +419,9 @@ export class Prober {
|
|
|
413
419
|
* than silently swallowed — a swallowed failing probe is what let a stale
|
|
414
420
|
* weekly look fresh. Never throws. */
|
|
415
421
|
async probeOne(account) {
|
|
422
|
+
// Benched duplicate: never refreshed locally (see the sweep filter in probeAll),
|
|
423
|
+
// so the token is dead — a probe can only fail. Quota comes from the cloud mirror.
|
|
424
|
+
if (account.poolBenchedBy) return { ok: false, status: null, skipped: 'benched' };
|
|
416
425
|
// Subscription latched org-disabled (account-manager recordProbeError): the quota
|
|
417
426
|
// endpoint 403s before answering anything, so probing is pure waste until the org
|
|
418
427
|
// accepts OAuth again. Skipping also stops the every-60s hammer that ran 700+ times
|
package/src/server.js
CHANGED
|
@@ -10,6 +10,7 @@ import { writeFile, mkdir } from 'node:fs/promises';
|
|
|
10
10
|
import { readFileSync } from 'node:fs';
|
|
11
11
|
import { join } from 'node:path';
|
|
12
12
|
import { modelFamily, organizationUnknownMatch, sameAccountIdentity, sameUser } from './oauth.js';
|
|
13
|
+
import { resolveStickyBusyBudget } from './account-manager.js';
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
const HOP_BY_HOP_HEADERS = new Set([
|
|
@@ -80,6 +81,18 @@ const posMsFromEnv = (name, dflt) => {
|
|
|
80
81
|
const n = Number(process.env[name]);
|
|
81
82
|
return Number.isFinite(n) && n > 0 ? n : dflt;
|
|
82
83
|
};
|
|
84
|
+
// STICKY BUSY WAIT (2026-11-15) — a warm Claude conversation whose cloud account says
|
|
85
|
+
// "busy" waits for it (bounded) instead of moving and re-writing its prompt cache.
|
|
86
|
+
const PIN_BUSY_MAX_RETRY_AFTER_S = 10; // a header-less 429/503 asking ≤ this is busy-class
|
|
87
|
+
const STICKY_BUSY_MAX_ATTEMPTS = posMsFromEnv('MAXPOOL_STICKY_BUSY_MAX_ATTEMPTS', 6);
|
|
88
|
+
const STICKY_BUSY_MIN_BYTES = posMsFromEnv('MAXPOOL_PIN_BUSY_MIN_BYTES', BIG_PROMPT_BYTES);
|
|
89
|
+
const STICKY_BUSY_WAIT_CAP_MS = 15_000; // per-wait cap for the growing waits (nextStickyBusyWaitMs)
|
|
90
|
+
/** The total wait budget, read LIVE per request (TUI-editable `stickyBusyBudgetMs`);
|
|
91
|
+
* env MAXPOOL_STICKY_BUSY_BUDGET_MS overrides it only when set. Same resolver as the
|
|
92
|
+
* TUI setting, so the screen always shows the value in force. */
|
|
93
|
+
function stickyBusyBudgetMs(accountManager) {
|
|
94
|
+
return resolveStickyBusyBudget(accountManager?.scheduler).ms;
|
|
95
|
+
}
|
|
83
96
|
/** Live knob values (env read per call so a two-pool test can inject a small cap). */
|
|
84
97
|
function deadlineKnobs() {
|
|
85
98
|
return {
|
|
@@ -717,6 +730,7 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
717
730
|
}
|
|
718
731
|
} finally {
|
|
719
732
|
clearInterval(reaperTimer);
|
|
733
|
+
accountManager.noteClientOutcome?.(requestInfo, ctx.status);
|
|
720
734
|
hooks.onRequestEnd?.(reqId, {
|
|
721
735
|
method: req.method, path: req.url,
|
|
722
736
|
account: ctx.account, status: ctx.status,
|
|
@@ -734,6 +748,10 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
734
748
|
// F9: only the message-creating call (POST /v1/messages) has a streaming shape — count_tokens and GETs
|
|
735
749
|
// must not read as "non-streaming" and swamp the share.
|
|
736
750
|
stream: requestInfo?.method === 'POST' && /^\/v1\/messages(?:\?|$)/.test(String(req.url || '')) ? Boolean(requestInfo.stream) : null,
|
|
751
|
+
// The served attempt's cache figures, for the `cache r… w… in…` line tag.
|
|
752
|
+
cacheUsage: requestInfo && !requestInfo.isCountTokens && (requestInfo._capacityInput || requestInfo._capacityCacheRead || requestInfo._capacityCacheCreation)
|
|
753
|
+
? { input_tokens: requestInfo._capacityInput || 0, cache_read_input_tokens: requestInfo._capacityCacheRead || 0, cache_creation_input_tokens: requestInfo._capacityCacheCreation || 0 }
|
|
754
|
+
: null,
|
|
737
755
|
});
|
|
738
756
|
}
|
|
739
757
|
} catch (err) {
|
|
@@ -919,8 +937,23 @@ async function forwardRequest(
|
|
|
919
937
|
// actually happened (acquireAccount consumes the one-shot hint either way).
|
|
920
938
|
const pendingSame = requestInfo.pendingRetrySame || null;
|
|
921
939
|
requestInfo.pendingRetrySame = null;
|
|
940
|
+
// A cloud row this request already gave up waiting on stays out while it is still
|
|
941
|
+
// busy — including after a queue resume, whose fresh exclusion set would re-pick it.
|
|
942
|
+
// (Gated on the same pinBusyUntil hasAvailableRoute reads, so the two never disagree.)
|
|
943
|
+
if (requestInfo._busyExcluded) {
|
|
944
|
+
const now = Date.now();
|
|
945
|
+
for (const a of accountManager.accounts) {
|
|
946
|
+
if (requestInfo._busyExcluded.has(a.name) && a.pinBusyUntil > now) excludedIndexes.add(a.index);
|
|
947
|
+
}
|
|
948
|
+
}
|
|
922
949
|
const lease = accountManager.acquireAccount(requestInfo, excludedIndexes);
|
|
923
950
|
const account = lease?.account;
|
|
951
|
+
requestInfo._hop = retryCount;
|
|
952
|
+
// The sticky-wait retry pin is good for exactly this one acquire.
|
|
953
|
+
if (requestInfo._stickyPin) {
|
|
954
|
+
requestInfo.pinnedAccountName = requestInfo._stickyPinPrev ?? null;
|
|
955
|
+
requestInfo._stickyPin = false;
|
|
956
|
+
}
|
|
924
957
|
if (pendingSame && account) {
|
|
925
958
|
if (account.name === pendingSame.account) logFailedAttempt(pendingSame.account, pendingSame.kind, pendingSame.detail, 'retry-same');
|
|
926
959
|
else logFailedAttempt(pendingSame.account, pendingSame.kind, `${pendingSame.detail}; same account refused by selection, served by "${account.name}"`, 'moved');
|
|
@@ -938,7 +971,16 @@ async function forwardRequest(
|
|
|
938
971
|
ctx.status = 503;
|
|
939
972
|
const busy = Boolean(requestInfo.pinRefusedBusy);
|
|
940
973
|
ctx.account = `(pinned ${requestInfo.pinAccount} ${busy ? 'busy' : 'unavailable'})`;
|
|
941
|
-
try {
|
|
974
|
+
try {
|
|
975
|
+
res.setHeader('retry-after', busy ? '2' : '30');
|
|
976
|
+
// Machine-readable cause + depth for the linked pool (additive; old pools ignore them).
|
|
977
|
+
res.setHeader('x-maxpool-pin-refusal', requestInfo.pinRefusal || 'unavailable');
|
|
978
|
+
const pinned = accountManager.accounts.find(a => !a.remote && a.name === requestInfo.pinAccount);
|
|
979
|
+
if (pinned) {
|
|
980
|
+
res.setHeader('x-maxpool-pin-inflight', String(pinned.inFlight || 0));
|
|
981
|
+
res.setHeader('x-maxpool-pin-limit', String(accountManager._pinBusyLimit(pinned)));
|
|
982
|
+
}
|
|
983
|
+
} catch { /* headers sent */ }
|
|
942
984
|
sendErrorResponse(res, requestInfo, 503, { type: 'error', error: {
|
|
943
985
|
type: busy ? 'pinned_account_busy' : 'pinned_account_unavailable',
|
|
944
986
|
message: `Account "${requestInfo.pinAccount}" is ${busy ? 'busy' : 'not available'} right now` } });
|
|
@@ -1214,14 +1256,66 @@ async function forwardRequest(
|
|
|
1214
1256
|
// streamed: cool THIS mirror row and fail over through the normal path to the
|
|
1215
1257
|
// next-best account, local or cloud — exactly like a local account's 429.
|
|
1216
1258
|
if (account.remote && (upRes.status === 429 || upRes.status === 503) && !clientHasRealBytes(res, requestInfo)) {
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1259
|
+
const refusal = await readPinRefusal(upRes);
|
|
1260
|
+
accountManager.noteRemoteDepth?.(account.name, refusal.inflight, refusal.limit);
|
|
1261
|
+
if (refusal.cause === 'busy') {
|
|
1262
|
+
// BUSY is seconds, not an account fault: a bounce (no cooldown, no failure
|
|
1263
|
+
// count), and the row repels new placements for a moment.
|
|
1264
|
+
const raMs = (refusal.retryAfterS ?? 1) * 1000;
|
|
1265
|
+
finishLink({ bounced: true, status: upRes.status, error: 'remote_pinned_busy', refusal: 'busy' });
|
|
1266
|
+
accountManager.markPinBusy(account.index, raMs);
|
|
1267
|
+
const waitMs = nextStickyBusyWaitMs(requestInfo._busyWaits || 0, raMs, requestInfo._busyWaitMs || 0, stickyBusyBudgetMs(accountManager));
|
|
1268
|
+
if (waitMs > 0 && stickyBusyWaitEligible(account, requestInfo, res, queueConfig, accountManager, waitMs)) {
|
|
1269
|
+
requestInfo._busyWaits = (requestInfo._busyWaits || 0) + 1;
|
|
1270
|
+
requestInfo._busyWaitMs = (requestInfo._busyWaitMs || 0) + waitMs;
|
|
1271
|
+
detachLinkClose();
|
|
1272
|
+
// KEEP-ALIVE through the wait (a wait can run 15 s+, the last one longer): re-arm the stream-forward
|
|
1273
|
+
// grace around EVERY sleep. Its deadline is anchored once per request (default 10 s
|
|
1274
|
+
// after arrival), so it fires mid-sleep and commits the SSE stream + the heartbeat
|
|
1275
|
+
// interval (a ping every queue.heartbeatMs, 10 s default) — the client's idle
|
|
1276
|
+
// watchdog is fed for the whole wait and every later hop, until the first real
|
|
1277
|
+
// chunk stops it. Only the grace TIMER is cleared after the sleep; a committed
|
|
1278
|
+
// heartbeat keeps running. No lease and no queue ticket is held while waiting.
|
|
1279
|
+
const grace = armStreamForwardGrace(res, requestInfo, queueConfig, accountManager);
|
|
1280
|
+
const stayed = await sleepUnlessClientGone(waitMs, req, res);
|
|
1281
|
+
if (grace) clearTimeout(grace);
|
|
1282
|
+
if (!stayed) {
|
|
1283
|
+
clearQueueHeartbeat(requestInfo);
|
|
1284
|
+
// The CLIENT cancelled mid-wait: not a pool error. ctx.status still holds
|
|
1285
|
+
// the refusal's 503 — 499 (client closed) keeps the request-end log honest.
|
|
1286
|
+
requestInfo._clientCancelled = true;
|
|
1287
|
+
ctx.status = 499;
|
|
1288
|
+
logFailedAttempt(account.name, `remote-${upRes.status}`, 'linked pool busy; client left during the wait', 'released-cancel');
|
|
1289
|
+
return;
|
|
1290
|
+
}
|
|
1291
|
+
if (resendAllowed(requestInfo)) {
|
|
1292
|
+
// Retry the SAME row: a hard one-shot pin (selection cannot outscore it).
|
|
1293
|
+
requestInfo._stickyPinPrev = requestInfo.pinnedAccountName ?? null;
|
|
1294
|
+
requestInfo.pinnedAccountName = account.name;
|
|
1295
|
+
requestInfo._stickyPin = true;
|
|
1296
|
+
requestInfo.pendingRetrySame = { account: account.name, kind: `remote-${upRes.status}`,
|
|
1297
|
+
detail: `linked pool busy; waited ${(waitMs / 1000).toFixed(1)}s for the account holding this conversation` };
|
|
1298
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
(requestInfo._busyExcluded ||= new Set()).add(account.name);
|
|
1302
|
+
requestInfo._moveCause = 'busy';
|
|
1303
|
+
} else if (refusal.cause === 'throttled') {
|
|
1304
|
+
// The linked pool's egress is held: every row behind it is out — one move.
|
|
1305
|
+
finishLink({ bounced: true, status: upRes.status, error: 'remote_pinned_throttled', refusal: 'throttled' });
|
|
1306
|
+
const cooled = accountManager.markLinkMirrorsCooldown(account, Math.min((refusal.retryAfterS || 5) * 1000, 30_000));
|
|
1307
|
+
for (const a of cooled) excludedIndexes.add(a.index);
|
|
1308
|
+
requestInfo._moveCause = 'wall';
|
|
1309
|
+
} else {
|
|
1310
|
+
const ra = refusal.retryAfterS;
|
|
1311
|
+
// "unavailable"/429 cools this row; booked as a bounce, not a failure.
|
|
1312
|
+
account.cooldownUntil = Date.now() + (ra != null ? Math.min(ra, 300) * 1000 : 30_000);
|
|
1313
|
+
finishLink({ bounced: true, status: upRes.status, error: 'remote_pinned_unavailable', refusal: 'unavailable' });
|
|
1314
|
+
requestInfo._moveCause = 'wall';
|
|
1315
|
+
}
|
|
1222
1316
|
excludedIndexes.add(account.index);
|
|
1223
1317
|
if (retryCount + 1 < maxAttempts && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
1224
|
-
logFailedAttempt(account.name, `remote-${upRes.status}`, `linked pool refused before any byte`, 'moved');
|
|
1318
|
+
logFailedAttempt(account.name, `remote-${upRes.status}`, `linked pool refused before any byte (${refusal.cause})`, 'moved');
|
|
1225
1319
|
detachLinkClose();
|
|
1226
1320
|
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
|
|
1227
1321
|
}
|
|
@@ -1317,9 +1411,16 @@ async function forwardRequest(
|
|
|
1317
1411
|
// a conditional clear let pings land inside the real stream when the grace
|
|
1318
1412
|
// timer had already fired).
|
|
1319
1413
|
if (requestInfo.queueHeartbeatActive) clearQueueHeartbeat(requestInfo);
|
|
1414
|
+
if (upRes.status < 300) {
|
|
1415
|
+
// The conversation now lives on this row (first REAL byte), and its usage
|
|
1416
|
+
// is read off the relayed bytes — figures only, never a booking.
|
|
1417
|
+
accountManager.noteConversationServed?.(requestInfo, account);
|
|
1418
|
+
tapLinkUsage(value, requestInfo);
|
|
1419
|
+
}
|
|
1320
1420
|
res.write(Buffer.from(value));
|
|
1321
1421
|
}
|
|
1322
1422
|
} finally { try { await reader.cancel(); } catch { /* already done */ } }
|
|
1423
|
+
if (upRes.status < 300) tapLinkUsage(null, requestInfo); // flush the non-stream tail
|
|
1323
1424
|
}
|
|
1324
1425
|
res.end(); // an empty 2xx body: the heartbeat reaps itself on res.writableEnded
|
|
1325
1426
|
finishLink({ success: ctx.status < 500, status: ctx.status });
|
|
@@ -1338,6 +1439,7 @@ async function forwardRequest(
|
|
|
1338
1439
|
if (linkClientGone && !clientHasRealBytes(res, requestInfo)) {
|
|
1339
1440
|
finishLink({ success: false, neutral: true, error: 'client_cancelled' });
|
|
1340
1441
|
clearQueueHeartbeat(requestInfo);
|
|
1442
|
+
requestInfo._clientCancelled = true;
|
|
1341
1443
|
console.log(`[Maxpool] Client left after ${heldS}s on "${account.name}" (link, still waiting) — released, not retried`);
|
|
1342
1444
|
logFailedAttempt(account.name, 'client_cancelled', `client left after ${heldS}s, nothing sent`, 'released-cancel');
|
|
1343
1445
|
return;
|
|
@@ -1370,6 +1472,7 @@ async function forwardRequest(
|
|
|
1370
1472
|
finishLink({ status: 502, error: linkKind, scope: 'request', linkFault: true });
|
|
1371
1473
|
}
|
|
1372
1474
|
excludedIndexes.add(account.index); // P4b: never re-pick a row that just failed
|
|
1475
|
+
if (!linkClientGone) requestInfo._moveCause = 'link';
|
|
1373
1476
|
const streamTag = requestInfo.stream ? 'streaming' : 'non-streaming';
|
|
1374
1477
|
// A string code (UND_ERR_SOCKET, ECONNRESET) names the fault; DOMException
|
|
1375
1478
|
// codes are numbers (AbortError = 20), so fall back to the cause, then the name.
|
|
@@ -1517,6 +1620,8 @@ async function forwardRequest(
|
|
|
1517
1620
|
// Log-only: no setting reads it.
|
|
1518
1621
|
requestInfo._upstreamForwardStartAt = upstreamForwardStartAt;
|
|
1519
1622
|
requestInfo._firstUpstreamByteAt = Date.now();
|
|
1623
|
+
// A 2xx answer: the conversation now lives on this account (never on a refusal).
|
|
1624
|
+
if (upstreamRes.status < 300) accountManager.noteConversationServed?.(requestInfo, account);
|
|
1520
1625
|
// Response arrived — the pre-response leak window is over. Stop guarding for
|
|
1521
1626
|
// client-disconnect via abort (streamResponse handles mid-stream disconnects).
|
|
1522
1627
|
res.off('close', onClientClose);
|
|
@@ -1620,6 +1725,9 @@ async function forwardRequest(
|
|
|
1620
1725
|
&& !clientHasRealBytes(res, requestInfo)
|
|
1621
1726
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1622
1727
|
) {
|
|
1728
|
+
// Forced move (review 2026-11-16, 5b): label it, so the TUI breakdown and the
|
|
1729
|
+
// forced-moves signal see it instead of an unlabelled 'other'.
|
|
1730
|
+
requestInfo._moveCause = 'wall';
|
|
1623
1731
|
logFailedAttempt(account.name, 'upstream_429', `server-side throttle, retry-after ${retryAfter}s`, 'moved');
|
|
1624
1732
|
return forwardRequest(
|
|
1625
1733
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
@@ -1738,6 +1846,7 @@ async function forwardRequest(
|
|
|
1738
1846
|
!clientHasRealBytes(res, requestInfo) &&
|
|
1739
1847
|
hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1740
1848
|
) {
|
|
1849
|
+
requestInfo._moveCause = 'wall';
|
|
1741
1850
|
logFailedAttempt(account.name, rlKind, `HTTP 429, benched ${retryAfter}s`, 'moved');
|
|
1742
1851
|
return forwardRequest(
|
|
1743
1852
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
@@ -1845,6 +1954,7 @@ async function forwardRequest(
|
|
|
1845
1954
|
&& !clientHasRealBytes(res, requestInfo)
|
|
1846
1955
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1847
1956
|
) {
|
|
1957
|
+
requestInfo._moveCause = 'wall';
|
|
1848
1958
|
logFailedAttempt(account.name, 'upstream_529', `overloaded, retry-after ${retryAfter}s`, 'moved');
|
|
1849
1959
|
return forwardRequest(
|
|
1850
1960
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
@@ -1901,6 +2011,7 @@ async function forwardRequest(
|
|
|
1901
2011
|
|
|
1902
2012
|
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
1903
2013
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
2014
|
+
requestInfo._moveCause = 'link';
|
|
1904
2015
|
logFailedAttempt(account.name, `upstream_${upstreamRes.status}`, 'retriable upstream status, cooling down', 'moved');
|
|
1905
2016
|
return forwardRequest(
|
|
1906
2017
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
@@ -2545,6 +2656,7 @@ async function forwardRequest(
|
|
|
2545
2656
|
console.log(`[Maxpool] Client left after ${(heldMs / 1000).toFixed(1)}s on "${account.name}" `
|
|
2546
2657
|
+ `(${gotOutput ? 'mid-response — output already sent' : 'still waiting, nothing sent'})`);
|
|
2547
2658
|
logFailedAttempt(account.name, 'client_cancelled', `client left after ${(heldMs / 1000).toFixed(1)}s (${gotOutput ? 'mid-response' : 'still waiting'})`, 'released-cancel');
|
|
2659
|
+
requestInfo._clientCancelled = true;
|
|
2548
2660
|
releaseOnClientGone();
|
|
2549
2661
|
return;
|
|
2550
2662
|
}
|
|
@@ -2554,6 +2666,7 @@ async function forwardRequest(
|
|
|
2554
2666
|
const heldMs = Date.now() - (requestInfo.startedAt || Date.now());
|
|
2555
2667
|
console.log(`[Maxpool] Client left after ${(heldMs / 1000).toFixed(1)}s on "${account.name}" (during the response body read) — released, not retried`);
|
|
2556
2668
|
logFailedAttempt(account.name, 'client_cancelled', `client left after ${(heldMs / 1000).toFixed(1)}s (body read)`, 'released-cancel');
|
|
2669
|
+
requestInfo._clientCancelled = true;
|
|
2557
2670
|
accountManager.relinquishUpstreamProbe?.(lease);
|
|
2558
2671
|
accountManager.releaseAccount(lease, { neutral: true, error: 'client_cancelled', kind: 'client_cancelled' });
|
|
2559
2672
|
clearQueueHeartbeat(requestInfo);
|
|
@@ -2670,6 +2783,7 @@ async function forwardRequest(
|
|
|
2670
2783
|
// answer with a misleading quota message).
|
|
2671
2784
|
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2672
2785
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
2786
|
+
requestInfo._moveCause = 'link';
|
|
2673
2787
|
logFailedAttempt(account.name, transientKind, `transient after ${heldSeconds(lease)}s`, 'moved');
|
|
2674
2788
|
return forwardRequest(
|
|
2675
2789
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
@@ -2720,6 +2834,7 @@ async function forwardRequest(
|
|
|
2720
2834
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
2721
2835
|
account.status = 'error';
|
|
2722
2836
|
excludedIndexes.add(account.index);
|
|
2837
|
+
requestInfo._moveCause = 'link';
|
|
2723
2838
|
logFailedAttempt(account.name, (err.code || err.message || 'error').toString().slice(0, 40), err.message, 'moved');
|
|
2724
2839
|
return forwardRequest(
|
|
2725
2840
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
@@ -2801,6 +2916,76 @@ function retrySameEligible(account, requestInfo, res, queueConfig = {}) {
|
|
|
2801
2916
|
return !clientHasRealBytes(res, requestInfo);
|
|
2802
2917
|
}
|
|
2803
2918
|
|
|
2919
|
+
/** The next sticky wait (ms) for 0-based attempt `attempt`, or 0 = stop waiting and move.
|
|
2920
|
+
* The 6-attempt cap bounds body re-uploads, so the waits GROW to still cover the budget:
|
|
2921
|
+
* min(15 s, retry-after × 2^attempt) × full jitter (0.5-1.5), and the LAST attempt waits
|
|
2922
|
+
* whatever is left. Every wait is clipped to the budget left, so the total equals the
|
|
2923
|
+
* setting and never exceeds it. At retry-after 2 s, jitter midpoint:
|
|
2924
|
+
* 15 s → 2+4+8+1 30 s → 2+4+8+15+1 60 s → 2+4+8+15+15+16 120 s → 2+4+8+15+15+76
|
|
2925
|
+
* Budget 0 ('off') → never waits (moves at once, as before). */
|
|
2926
|
+
function nextStickyBusyWaitMs(attempt, retryAfterMs, spentMs, budgetMs, rand = Math.random) {
|
|
2927
|
+
const remaining = budgetMs - spentMs;
|
|
2928
|
+
if (attempt >= STICKY_BUSY_MAX_ATTEMPTS || remaining <= 0) return 0;
|
|
2929
|
+
if (attempt === STICKY_BUSY_MAX_ATTEMPTS - 1) return remaining;
|
|
2930
|
+
return Math.min(remaining, Math.min(STICKY_BUSY_WAIT_CAP_MS, retryAfterMs * 2 ** attempt) * (0.5 + rand()));
|
|
2931
|
+
}
|
|
2932
|
+
|
|
2933
|
+
/** R3: may a busy refusal on this cloud row WAIT and retry the same row? Only a warm
|
|
2934
|
+
* Claude conversation (its cache is on this row), a big prompt, nothing sent yet, and
|
|
2935
|
+
* inside the client's patience (the attempt cap and the live budget: nextStickyBusyWaitMs). */
|
|
2936
|
+
function stickyBusyWaitEligible(account, requestInfo, res, queueConfig, accountManager, waitMs) {
|
|
2937
|
+
if (requestInfo.isCountTokens || clientHasRealBytes(res, requestInfo)) return false;
|
|
2938
|
+
if ((requestInfo.bodyBytes || 0) < STICKY_BUSY_MIN_BYTES) return false;
|
|
2939
|
+
// Non-streaming waits only with a real deadline that still leaves time to move after.
|
|
2940
|
+
if (!requestInfo.stream && (requestInfo.deadlineAt == null
|
|
2941
|
+
|| nonStreamRemainingMs(requestInfo) - waitMs < deadlineKnobs().NONSTREAM_MIN_MOVE_MS)) return false;
|
|
2942
|
+
if (!accountManager.holdsWarmConversation?.(account, requestInfo)) return false;
|
|
2943
|
+
const ceilingMs = clientWaitCeilingMs(requestInfo, queueConfig);
|
|
2944
|
+
if (ceilingMs != null && Date.now() - (requestInfo.startedAt || Date.now()) + waitMs > ceilingMs) return false;
|
|
2945
|
+
return true;
|
|
2946
|
+
}
|
|
2947
|
+
|
|
2948
|
+
/** R1: classify a linked pool's 429/503 before any byte. Header cause first, then the
|
|
2949
|
+
* 503 body type (sent by every version), then a short retry-after = busy-class;
|
|
2950
|
+
* anything else is "unavailable" (never guessed busy). Reads ≤ 4 KB of the body. */
|
|
2951
|
+
async function readPinRefusal(upRes) {
|
|
2952
|
+
const num = (v) => (v == null || v === '' || !Number.isFinite(Number(v)) ? null : Number(v));
|
|
2953
|
+
const ra = num(upRes.headers.get('retry-after'));
|
|
2954
|
+
const retryAfterS = ra != null && ra > 0 ? ra : null;
|
|
2955
|
+
let bodyType = null;
|
|
2956
|
+
if (upRes.body) {
|
|
2957
|
+
try {
|
|
2958
|
+
const reader = upRes.body.getReader();
|
|
2959
|
+
const r = await Promise.race([reader.read(), sleep(2000).then(() => null)]);
|
|
2960
|
+
if (r?.value) {
|
|
2961
|
+
try { bodyType = JSON.parse(Buffer.from(r.value).subarray(0, 4096).toString())?.error?.type ?? null; } catch { /* not JSON */ }
|
|
2962
|
+
}
|
|
2963
|
+
reader.cancel().catch(() => {});
|
|
2964
|
+
} catch { /* drained */ }
|
|
2965
|
+
}
|
|
2966
|
+
const header = String(upRes.headers.get('x-maxpool-pin-refusal') || '').toLowerCase();
|
|
2967
|
+
const cause = ['busy', 'unavailable', 'throttled'].includes(header) ? header
|
|
2968
|
+
: bodyType === 'pinned_account_busy' ? 'busy'
|
|
2969
|
+
: bodyType === 'pinned_account_unavailable' ? 'unavailable'
|
|
2970
|
+
: retryAfterS != null && retryAfterS <= PIN_BUSY_MAX_RETRY_AFTER_S ? 'busy'
|
|
2971
|
+
: 'unavailable';
|
|
2972
|
+
return { cause, retryAfterS, bodyType,
|
|
2973
|
+
inflight: num(upRes.headers.get('x-maxpool-pin-inflight')), limit: num(upRes.headers.get('x-maxpool-pin-limit')) };
|
|
2974
|
+
}
|
|
2975
|
+
|
|
2976
|
+
/** Sleep `ms` unless the client leaves first. true = slept the whole time. */
|
|
2977
|
+
function sleepUnlessClientGone(ms, req, res) {
|
|
2978
|
+
if (res.destroyed || res.writableEnded) return Promise.resolve(false);
|
|
2979
|
+
return new Promise((resolve) => {
|
|
2980
|
+
let timer = null;
|
|
2981
|
+
const done = (stayed) => { clearTimeout(timer); req.off('aborted', gone); res.off('close', gone); resolve(stayed); };
|
|
2982
|
+
const gone = () => done(false);
|
|
2983
|
+
req.once('aborted', gone);
|
|
2984
|
+
res.once('close', gone);
|
|
2985
|
+
timer = setTimeout(() => done(!res.destroyed), ms);
|
|
2986
|
+
});
|
|
2987
|
+
}
|
|
2988
|
+
|
|
2804
2989
|
/**
|
|
2805
2990
|
* D5: one line per FAILED attempt, naming the account, the error class and what the
|
|
2806
2991
|
* pool did next — so "why did this move" is answerable from the log without a forensic
|
|
@@ -3069,7 +3254,7 @@ function isContextLengthError(errorBody) {
|
|
|
3069
3254
|
return /exceeded model token limit|maximum context length|context length exceeded|context window (?:size )?(?:exceeded|too)|prompt is too long|input is too long|reduce the length of|too many (?:input )?tokens|request too large/i.test(errorBody);
|
|
3070
3255
|
}
|
|
3071
3256
|
|
|
3072
|
-
export const __serverTest = { hasEligibleRoute, resendAllowed, directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, parsePatience, computeDeadline, computeLinkHop, nonStreamFirstByteWait, nonStreamBodyWait, nonStreamRemainingMs, deadlineKnobs, deadlineErrorPayload, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
3257
|
+
export const __serverTest = { nextStickyBusyWaitMs, readPinRefusal, sleepUnlessClientGone, stickyBusyWaitEligible, stickyBusyBudgetMs, usageFromSseData, usageFromBodyTail, tapLinkUsage, stickyKnobs: () => ({ PIN_BUSY_MAX_RETRY_AFTER_S, STICKY_BUSY_MAX_ATTEMPTS, STICKY_BUSY_MIN_BYTES, STICKY_BUSY_WAIT_CAP_MS }), hasEligibleRoute, resendAllowed, directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, parsePatience, computeDeadline, computeLinkHop, nonStreamFirstByteWait, nonStreamBodyWait, nonStreamRemainingMs, deadlineKnobs, deadlineErrorPayload, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
3073
3258
|
|
|
3074
3259
|
async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
|
|
3075
3260
|
if (!upstreamRes.body) return '';
|
|
@@ -5031,6 +5216,71 @@ function noteCapacityCache(requestInfo, usage) {
|
|
|
5031
5216
|
requestInfo._capacityCacheCreation = Math.max(requestInfo._capacityCacheCreation || 0, usage?.cache_creation_input_tokens || 0);
|
|
5032
5217
|
}
|
|
5033
5218
|
|
|
5219
|
+
/** Pure: the usage object one parsed SSE event carries — message_start's
|
|
5220
|
+
* `message.usage` or message_delta's `usage` — else null. No side effects. */
|
|
5221
|
+
function usageFromSseData(data) {
|
|
5222
|
+
if (data?.type === 'message_start' && data.message?.usage) return data.message.usage;
|
|
5223
|
+
if (data?.type === 'message_delta' && data.usage) return data.usage;
|
|
5224
|
+
return null;
|
|
5225
|
+
}
|
|
5226
|
+
|
|
5227
|
+
/** Running-max one event's usage into the request's capacity figures. Output only from
|
|
5228
|
+
* message_delta (Anthropic's interim delta usage is CUMULATIVE — M3). Providers differ
|
|
5229
|
+
* in WHICH event carries the cache/input counters (z.ai: all of them on message_delta,
|
|
5230
|
+
* message_start is zeros — captured 2026-11-06, pinned by test/capacity-seam.test.js +
|
|
5231
|
+
* test/fixtures/zai-sse-usage.txt), so input/cache are a running max over BOTH events. */
|
|
5232
|
+
function noteCapacityUsage(requestInfo, usage, withOutput) {
|
|
5233
|
+
requestInfo._capacityInput = Math.max(requestInfo._capacityInput || 0, usage?.input_tokens || 0);
|
|
5234
|
+
if (withOutput) requestInfo._capacityOutput = Math.max(requestInfo._capacityOutput || 0, usage?.output_tokens || 0);
|
|
5235
|
+
noteCapacityCache(requestInfo, usage);
|
|
5236
|
+
}
|
|
5237
|
+
|
|
5238
|
+
/** Pure: the usage object at the END of a JSON message body, from its tail (the body
|
|
5239
|
+
* may be cut at the front). Finds the last `"usage":{…}` and parses that object. */
|
|
5240
|
+
function usageFromBodyTail(text) {
|
|
5241
|
+
const at = String(text || '').lastIndexOf('"usage"');
|
|
5242
|
+
if (at < 0) return null;
|
|
5243
|
+
const open = text.indexOf('{', at);
|
|
5244
|
+
if (open < 0) return null;
|
|
5245
|
+
let depth = 0;
|
|
5246
|
+
for (let i = open; i < text.length; i++) {
|
|
5247
|
+
if (text[i] === '{') depth++;
|
|
5248
|
+
else if (text[i] === '}' && --depth === 0) {
|
|
5249
|
+
try { return JSON.parse(text.slice(open, i + 1)); } catch { return null; }
|
|
5250
|
+
}
|
|
5251
|
+
}
|
|
5252
|
+
return null;
|
|
5253
|
+
}
|
|
5254
|
+
|
|
5255
|
+
/** LINK USAGE TAP: read the usage a linked pool relays (it is the SERVING pool and books
|
|
5256
|
+
* the tokens itself), into this request's capacity figures ONLY — never a ledger,
|
|
5257
|
+
* usage or caller booking. Streaming: SSE events split across chunks are re-joined.
|
|
5258
|
+
* Non-streaming: keeps the last 64 KB, parsed when called with null at stream end. */
|
|
5259
|
+
function tapLinkUsage(chunk, requestInfo) {
|
|
5260
|
+
if (requestInfo.isCountTokens) return;
|
|
5261
|
+
if (!requestInfo.stream) {
|
|
5262
|
+
if (chunk) {
|
|
5263
|
+
requestInfo._tapBuf = ((requestInfo._tapBuf || '') + Buffer.from(chunk).toString()).slice(-64 * 1024);
|
|
5264
|
+
return;
|
|
5265
|
+
}
|
|
5266
|
+
const u = usageFromBodyTail(requestInfo._tapBuf);
|
|
5267
|
+
requestInfo._tapBuf = '';
|
|
5268
|
+
if (u) noteCapacityUsage(requestInfo, u, true);
|
|
5269
|
+
return;
|
|
5270
|
+
}
|
|
5271
|
+
if (!chunk) return;
|
|
5272
|
+
const parts = ((requestInfo._tapBuf || '') + Buffer.from(chunk).toString()).split('\n\n');
|
|
5273
|
+
requestInfo._tapBuf = parts.pop().slice(-64 * 1024);
|
|
5274
|
+
for (const event of parts) {
|
|
5275
|
+
const dataLine = event.split('\n').find(l => l.startsWith('data: '));
|
|
5276
|
+
if (!dataLine) continue;
|
|
5277
|
+
let data;
|
|
5278
|
+
try { data = JSON.parse(dataLine.slice(6)); } catch { continue; }
|
|
5279
|
+
const u = usageFromSseData(data);
|
|
5280
|
+
if (u) noteCapacityUsage(requestInfo, u, data.type === 'message_delta');
|
|
5281
|
+
}
|
|
5282
|
+
}
|
|
5283
|
+
|
|
5034
5284
|
function parseSSEEvent(event, accountIndex, accountManager, requestInfo = {}) {
|
|
5035
5285
|
const dataLine = event.split('\n').find(l => l.startsWith('data: '));
|
|
5036
5286
|
if (!dataLine) return;
|
|
@@ -5041,23 +5291,13 @@ function parseSSEEvent(event, accountIndex, accountManager, requestInfo = {}) {
|
|
|
5041
5291
|
// M3: Anthropic interim message_delta usage is CUMULATIVE — add-semantics inflates
|
|
5042
5292
|
// output on long generations. Track a per-stream RUNNING MAX, accrue once at end.
|
|
5043
5293
|
// M4: count_tokens responses are prompt-size echoes, not delivered work — skip.
|
|
5044
|
-
|
|
5045
|
-
|
|
5046
|
-
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
accountManager.updateUsage(accountIndex, 0, data.usage.output_tokens);
|
|
5052
|
-
accountManager.countCallerTokens?.(requestInfo.caller, data.usage.output_tokens);
|
|
5053
|
-
requestInfo._capacityOutput = Math.max(requestInfo._capacityOutput || 0, data.usage.output_tokens || 0);
|
|
5054
|
-
// Providers differ in WHICH event carries the cache/input counters (z.ai: all of
|
|
5055
|
-
// them on message_delta, message_start is zeros — captured 2026-11-06, pinned by
|
|
5056
|
-
// test/capacity-seam.test.js + test/fixtures/zai-sse-usage.txt), so every field is
|
|
5057
|
-
// a running max over BOTH events.
|
|
5058
|
-
requestInfo._capacityInput = Math.max(requestInfo._capacityInput || 0, data.usage.input_tokens || 0);
|
|
5059
|
-
noteCapacityCache(requestInfo, data.usage);
|
|
5060
|
-
}
|
|
5294
|
+
const usage = requestInfo.isCountTokens ? null : usageFromSseData(data);
|
|
5295
|
+
if (usage) {
|
|
5296
|
+
const isDelta = data.type === 'message_delta';
|
|
5297
|
+
const n = isDelta ? usage.output_tokens : usage.input_tokens;
|
|
5298
|
+
accountManager.updateUsage(accountIndex, isDelta ? 0 : n, isDelta ? n : 0);
|
|
5299
|
+
accountManager.countCallerTokens?.(requestInfo.caller, n);
|
|
5300
|
+
noteCapacityUsage(requestInfo, usage, isDelta);
|
|
5061
5301
|
}
|
|
5062
5302
|
if (sseEventContainsThinking(data)) {
|
|
5063
5303
|
accountManager.markSessionThinkingProtected?.(requestInfo.sessionKey, requestInfo.model);
|
|
@@ -5114,6 +5354,7 @@ function extractUsageFromBody(buffer, accountIndex, accountManager, requestInfo
|
|
|
5114
5354
|
// work delivered. Claude Code calls it constantly; counting it would inflate every
|
|
5115
5355
|
// cycle by whole prompt sizes.
|
|
5116
5356
|
if (!requestInfo.isCountTokens) {
|
|
5357
|
+
noteCapacityUsage(requestInfo, json.usage, true); // efficiency figures (no booking)
|
|
5117
5358
|
accountManager.accrueCapacity?.(accountIndex, {
|
|
5118
5359
|
input: json.usage.input_tokens || 0,
|
|
5119
5360
|
output: json.usage.output_tokens || 0,
|