maxpool 1.24.80 → 1.24.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +725 -35
- package/src/claude-reset-grants.js +124 -24
- package/src/index.js +1 -0
- package/src/prober.js +162 -14
- package/src/reset-policy.js +130 -25
- package/src/reset-timing.js +27 -0
- package/src/server.js +486 -93
- package/src/tui.js +111 -28
- package/src/zai-reset-cards.js +20 -6
package/src/server.js
CHANGED
|
@@ -31,6 +31,25 @@ const DEFAULT_RETRY = {
|
|
|
31
31
|
// serves EVERY provider (Anthropic/GLM/Kimi), so the idle gap is generous enough
|
|
32
32
|
// that a legitimately-slow-but-alive stream is never cut (each chunk resets it).
|
|
33
33
|
const UPSTREAM_TTFB_MS = Math.max(5_000, Number(process.env.MAXPOOL_TTFB_MS) || 120_000); // headers must arrive within this
|
|
34
|
+
// D2 (2026-11-09): the first-byte wait scales with the prompt. A 650KB conversation
|
|
35
|
+
// legitimately needs longer than a 5KB one before its first byte (measured p95 300-350s
|
|
36
|
+
// on 400-650KB turns); the flat 120s was the engine of the hop loop — each timeout moved
|
|
37
|
+
// the turn to a cold account that re-read the whole prompt and timed out again.
|
|
38
|
+
// wait = max(floor, ceil(bodyBytes × msPerByte)), clamped at the client's patience.
|
|
39
|
+
// An explicit 0 disables scaling (flat floor, the pre-2026-11-09 behaviour).
|
|
40
|
+
const msPerByteFromEnv = (name, dflt) => {
|
|
41
|
+
const raw = process.env[name];
|
|
42
|
+
if (raw == null || raw === '') return dflt;
|
|
43
|
+
const n = Number(raw);
|
|
44
|
+
return Number.isFinite(n) && n >= 0 ? n : dflt;
|
|
45
|
+
};
|
|
46
|
+
const TTFB_MS_PER_BYTE = msPerByteFromEnv('MAXPOOL_TTFB_MS_PER_BYTE', 0.6);
|
|
47
|
+
// D3: at/over this size a first-byte timeout retries the SAME account once (its stored
|
|
48
|
+
// copy is warm there) before the request is allowed to move.
|
|
49
|
+
const BIG_PROMPT_BYTES = (() => {
|
|
50
|
+
const n = Number(process.env.MAXPOOL_BIG_PROMPT_BYTES);
|
|
51
|
+
return Number.isFinite(n) && n > 0 ? n : 250_000;
|
|
52
|
+
})();
|
|
34
53
|
// A real SSE EVENT, not a `:` comment. Claude Code's stall watchdog is reset only when its
|
|
35
54
|
// SSE iterator YIELDS an event; per the spec a comment line is discarded by the parser and
|
|
36
55
|
// never yields, so a comment keepalive resets nothing. That is why held requests died at
|
|
@@ -454,6 +473,8 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
454
473
|
return;
|
|
455
474
|
}
|
|
456
475
|
|
|
476
|
+
markFirstByte(res, () => hooks.onRequestFirstByte?.(reqId));
|
|
477
|
+
|
|
457
478
|
// The request is now TRACKED (onRequestStart accepted → tui.active +
|
|
458
479
|
// restartController.activeRequests both hold it). EVERYTHING that can throw —
|
|
459
480
|
// including body buffering — must run inside the try whose finally fires
|
|
@@ -477,6 +498,11 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
477
498
|
const queueConfig = { ...DEFAULT_QUEUE, ...(config.queue || {}) };
|
|
478
499
|
const canRetryBufferedBody = body.length <= retryConfig.maxRetryBufferBytes;
|
|
479
500
|
requestInfo = describeRequest(req, body);
|
|
501
|
+
// Request clock: the direct client-gone line ("Client left after Xs" — request
|
|
502
|
+
// age) and the retry-same patience rule read it. (The link path reports time on
|
|
503
|
+
// THIS account, from lease.startedAt.) Was never set, so every direct-path
|
|
504
|
+
// client-gone line read 0.0s.
|
|
505
|
+
requestInfo.startedAt = Date.now();
|
|
480
506
|
const maxQueuedBodyBytes = queueConfig.maxQueuedBodyBytes == null
|
|
481
507
|
? Infinity
|
|
482
508
|
: Math.max(0, Number(queueConfig.maxQueuedBodyBytes) || 0);
|
|
@@ -548,6 +574,11 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
548
574
|
// belonged to, so a turn's account is exact instead of a timestamp guess.
|
|
549
575
|
conversation: requestInfo?.conversationHash || null,
|
|
550
576
|
sessionKey: headerValue(req.headers, 'x-maxpool-session') || null,
|
|
577
|
+
// D (2026-11-10): the first REAL upstream byte (the fetch resolve), for
|
|
578
|
+
// the TUI's ttfb= tag. Prefer this over the client-write stamp: a queued
|
|
579
|
+
// request's heartbeat commits writes LONG before upstream answers.
|
|
580
|
+
firstUpstreamByteAt: requestInfo?._firstUpstreamByteAt ?? null,
|
|
581
|
+
upstreamForwardStartAt: requestInfo?._upstreamForwardStartAt ?? null,
|
|
551
582
|
});
|
|
552
583
|
}
|
|
553
584
|
} catch (err) {
|
|
@@ -712,13 +743,22 @@ async function forwardRequest(
|
|
|
712
743
|
}
|
|
713
744
|
|
|
714
745
|
// Select account
|
|
746
|
+
// D3/D5: a retry-same attempt's line waits for THIS acquire — the token is what
|
|
747
|
+
// actually happened (acquireAccount consumes the one-shot hint either way).
|
|
748
|
+
const pendingSame = requestInfo.pendingRetrySame || null;
|
|
749
|
+
requestInfo.pendingRetrySame = null;
|
|
715
750
|
const lease = accountManager.acquireAccount(requestInfo, excludedIndexes);
|
|
716
751
|
const account = lease?.account;
|
|
752
|
+
if (pendingSame && account) {
|
|
753
|
+
if (account.name === pendingSame.account) logFailedAttempt(pendingSame.account, pendingSame.kind, pendingSame.detail, 'retry-same');
|
|
754
|
+
else logFailedAttempt(pendingSame.account, pendingSame.kind, `${pendingSame.detail}; same account refused by selection, served by "${account.name}"`, 'moved');
|
|
755
|
+
}
|
|
717
756
|
if (!account) {
|
|
718
757
|
const queued = await queueAndRetry(
|
|
719
758
|
'no eligible account/provider currently available',
|
|
720
759
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
721
760
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'quota',
|
|
761
|
+
...(pendingSame ? [{ attempt: [pendingSame.account, pendingSame.kind, `${pendingSame.detail}; same account refused by selection, no route free`] }] : []),
|
|
722
762
|
);
|
|
723
763
|
if (queued) return;
|
|
724
764
|
|
|
@@ -801,7 +841,7 @@ async function forwardRequest(
|
|
|
801
841
|
// first events would be pure latency. See streamResponse + classifyHeldStreamPrefix.
|
|
802
842
|
requestInfo._refusalRerouteEligible = account.type !== 'provider'
|
|
803
843
|
&& canRetryBufferedBody
|
|
804
|
-
&& !res
|
|
844
|
+
&& !clientHasRealBytes(res, requestInfo);
|
|
805
845
|
|
|
806
846
|
// Refresh OAuth token if needed
|
|
807
847
|
const tokenReady = await accountManager.ensureTokenFresh(account.index);
|
|
@@ -822,6 +862,7 @@ async function forwardRequest(
|
|
|
822
862
|
retryCount + 1 < maxAttempts &&
|
|
823
863
|
hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
824
864
|
) {
|
|
865
|
+
logFailedAttempt(account.name, 'token_refresh', 'OAuth token refresh unavailable', 'moved');
|
|
825
866
|
return forwardRequest(
|
|
826
867
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
827
868
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -832,6 +873,7 @@ async function forwardRequest(
|
|
|
832
873
|
`OAuth token refresh unavailable for "${account.name}"`,
|
|
833
874
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
834
875
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'quota',
|
|
876
|
+
{ attempt: [account.name, 'token_refresh', 'OAuth token refresh unavailable; no other route'] },
|
|
835
877
|
);
|
|
836
878
|
if (queued) return;
|
|
837
879
|
|
|
@@ -908,6 +950,18 @@ async function forwardRequest(
|
|
|
908
950
|
// listener (calling releaseOnClientGone would release the lease itself).
|
|
909
951
|
res.off('close', onClientClose);
|
|
910
952
|
let linkDone = false;
|
|
953
|
+
// Set by the res 'close' handler BEFORE it aborts the link fetch, so the catch can
|
|
954
|
+
// tell "the client left" apart from the TTFB timer or a real link error. Mirrors
|
|
955
|
+
// the direct path's clientGone handling (catch below, "Client left after").
|
|
956
|
+
let linkClientGone = false;
|
|
957
|
+
// The linked pool's status once headers arrived (null = no answer yet). Local, not
|
|
958
|
+
// ctx.status: an earlier hop's 429 would otherwise leak into this hop's catch.
|
|
959
|
+
let linkStatus = null;
|
|
960
|
+
let linkGraceTimer = null;
|
|
961
|
+
// One 'close' listener per link hop, detached at every exit (review r2 F) — like the
|
|
962
|
+
// direct path's onClientClose, so a failover / queue-resume chain never stacks them.
|
|
963
|
+
let onLinkClose = null;
|
|
964
|
+
const detachLinkClose = () => { if (onLinkClose) res.off('close', onLinkClose); onLinkClose = null; };
|
|
911
965
|
const finishLink = (outcome) => {
|
|
912
966
|
if (linkDone) return; linkDone = true;
|
|
913
967
|
accountManager.releaseAccount(lease, outcome);
|
|
@@ -918,14 +972,27 @@ async function forwardRequest(
|
|
|
918
972
|
// thinking turns regularly sit silent 20-60s before headers, so the link
|
|
919
973
|
// TTFB must match the direct path's UPSTREAM_TTFB_MS (120s default), not a
|
|
920
974
|
// flat 15s (owner report 2026-10-30: every pinned GLM turn 502-aborted).
|
|
975
|
+
// FLAT on purpose — no D2 size scaling and no D3 retry-same here (red-team r1,
|
|
976
|
+
// 2026-11-09): the linked pool commits its own keep-alive headers ~1s in, which
|
|
977
|
+
// clears this timer for every HEALTHY link, so a scaled wait only ever ran against
|
|
978
|
+
// a dead/blackholed tunnel — stretching its detection from 120s to minutes. The
|
|
979
|
+
// big-prompt first-byte timeouts measured were all on the direct path.
|
|
921
980
|
const linkTtfbMs = Math.max(15_000, Number(process.env.MAXPOOL_LINK_TTFB_MS) || UPSTREAM_TTFB_MS);
|
|
922
981
|
const linkCtl = new AbortController();
|
|
923
982
|
const linkTtfb = setTimeout(() => linkCtl.abort(), linkTtfbMs);
|
|
924
|
-
|
|
983
|
+
// P2: the link path had NO pre-headers keep-alive — a streaming client sat on a
|
|
984
|
+
// silent socket for the whole link wait (up to the 120s floor). Arm the same
|
|
985
|
+
// stream-forward grace the direct path uses: after streamForwardGraceMs with no
|
|
986
|
+
// headers, commit the SSE stream + heartbeat. The post-headers branch below
|
|
987
|
+
// (queueHeartbeatActive) already clears it and frames a remote error correctly.
|
|
988
|
+
linkGraceTimer = armStreamForwardGrace(res, requestInfo, queueConfig, accountManager);
|
|
989
|
+
onLinkClose = () => { if (!res.writableEnded) linkClientGone = true; linkCtl.abort(); };
|
|
990
|
+
res.once('close', onLinkClose);
|
|
925
991
|
let upRes;
|
|
926
992
|
try {
|
|
927
993
|
upRes = await fetch(linkUrl, { method: req.method, headers: linkHeaders, body: ['GET', 'HEAD'].includes(req.method) ? undefined : body, duplex: 'half', signal: linkCtl.signal });
|
|
928
|
-
} finally { clearTimeout(linkTtfb); }
|
|
994
|
+
} finally { clearTimeout(linkTtfb); if (linkGraceTimer) clearTimeout(linkGraceTimer); }
|
|
995
|
+
linkStatus = upRes.status;
|
|
929
996
|
ctx.status = upRes.status;
|
|
930
997
|
// The linked pool refused the request's SHAPE (not Claude Code). The cloud
|
|
931
998
|
// subscription accounts are healthy — exclude them (this mirror + every local
|
|
@@ -939,13 +1006,18 @@ async function forwardRequest(
|
|
|
939
1006
|
for (const a of accountManager.accounts) {
|
|
940
1007
|
if (a.type === 'oauth') excludedIndexes.add(a.index);
|
|
941
1008
|
}
|
|
1009
|
+
// !clientHasRealBytes, not !headersSent: after the P2 keep-alive commits, headers
|
|
1010
|
+
// are out with only pings behind them — a slow-then-refused turn must still fail over.
|
|
942
1011
|
if (
|
|
943
1012
|
retryCount + 1 < maxAttempts
|
|
944
|
-
&& !res
|
|
1013
|
+
&& !clientHasRealBytes(res, requestInfo)
|
|
945
1014
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
946
1015
|
) {
|
|
1016
|
+
logFailedAttempt(account.name, 'request_shape_refused', 'linked pool refused the request shape (not Claude Code)', 'moved');
|
|
1017
|
+
detachLinkClose();
|
|
947
1018
|
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
|
|
948
1019
|
}
|
|
1020
|
+
logFailedAttempt(account.name, 'request_shape_refused', 'linked pool refused the request shape; no route serves it', 'surfaced');
|
|
949
1021
|
sendErrorBody(res, requestInfo, 429, errorText, upRes.headers);
|
|
950
1022
|
return;
|
|
951
1023
|
}
|
|
@@ -960,14 +1032,18 @@ async function forwardRequest(
|
|
|
960
1032
|
finishLink({ status: upRes.status, error: 'remote_pinned_unavailable' });
|
|
961
1033
|
excludedIndexes.add(account.index);
|
|
962
1034
|
if (retryCount + 1 < maxAttempts && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
1035
|
+
logFailedAttempt(account.name, `remote-${upRes.status}`, `linked pool refused before any byte`, 'moved');
|
|
1036
|
+
detachLinkClose();
|
|
963
1037
|
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
|
|
964
1038
|
}
|
|
965
1039
|
// Every route refused: that is capacity, so it waits here. The body was cancelled
|
|
966
1040
|
// above, so falling through handed the client a 503 with no body.
|
|
1041
|
+
detachLinkClose();
|
|
967
1042
|
const queued = await queueAndRetry(
|
|
968
1043
|
`all routes failed after ${upRes.status} from "${account.name}"`,
|
|
969
1044
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
970
1045
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'capacity',
|
|
1046
|
+
{ attempt: [account.name, `remote-${upRes.status}`, 'linked pool refused before any byte; no other route'] },
|
|
971
1047
|
);
|
|
972
1048
|
if (queued) return;
|
|
973
1049
|
ctx.status = upRes.status;
|
|
@@ -975,30 +1051,72 @@ async function forwardRequest(
|
|
|
975
1051
|
return;
|
|
976
1052
|
}
|
|
977
1053
|
// A request released from the queue already holds a committed keep-alive stream.
|
|
978
|
-
|
|
1054
|
+
// An error on it is framed as an SSE error; a 2xx keeps the heartbeat until the
|
|
1055
|
+
// first real chunk (below).
|
|
1056
|
+
if (requestInfo.queueHeartbeatActive && upRes.status >= 400) {
|
|
979
1057
|
clearQueueHeartbeat(requestInfo);
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
}
|
|
1058
|
+
const errorText = await readErrorBody(upRes);
|
|
1059
|
+
logFailedAttempt(account.name, `remote-${upRes.status}`, 'linked pool answered an error on a held stream', 'surfaced');
|
|
1060
|
+
sendErrorBody(res, requestInfo, upRes.status, errorText, upRes.headers);
|
|
1061
|
+
finishLink({ success: upRes.status < 500, status: upRes.status });
|
|
1062
|
+
return;
|
|
986
1063
|
}
|
|
987
1064
|
const outHeaders = {};
|
|
988
1065
|
upRes.headers.forEach((v, k) => { if (!HOP_BY_HOP_HEADERS.has(k) && k !== 'content-length' && k !== 'transfer-encoding') outHeaders[k] = v; });
|
|
1066
|
+
// Headers committed exactly as before (review r4: the batch-7 deferral is REVERTED
|
|
1067
|
+
// — it let a stale grace timer commit keep-alive mid-failover, ERR_HTTP_HEADERS_SENT
|
|
1068
|
+
// on the next hop, and leaked pings into the real stream). Consequence, accepted:
|
|
1069
|
+
// once the link's own headers are written, a later stall can only SURFACE — the
|
|
1070
|
+
// idle guard below bounds the wait instead of undici's ~300s bodyTimeout. Failover
|
|
1071
|
+
// stays possible while only OUR keep-alive pings have been sent (no real chunk yet).
|
|
989
1072
|
if (!res.headersSent) res.writeHead(upRes.status, outHeaders);
|
|
1073
|
+
// r4: headers are committed but NO real byte has been written — keep the keep-alive
|
|
1074
|
+
// heartbeating through the link body wait and any failover hop (grace arm refuses
|
|
1075
|
+
// an already-committed response). clientHasRealBytes stays false until the first
|
|
1076
|
+
// real chunk, so a silent/stalled link can still fail over.
|
|
1077
|
+
if (requestInfo.stream && upRes.status < 300 && !res.destroyed && !res.writableEnded
|
|
1078
|
+
&& !requestInfo.queueHeartbeatActive) {
|
|
1079
|
+
ensureQueueHeartbeatInterval(res, requestInfo, Math.max(1000, Number(queueConfig?.heartbeatMs) || 10_000));
|
|
1080
|
+
}
|
|
990
1081
|
if (upRes.body) {
|
|
991
1082
|
const reader = upRes.body.getReader();
|
|
1083
|
+
// R3-3/r4: bound the link body like the direct path bounds its stream
|
|
1084
|
+
// (MAXPOOL_LINK_IDLE_MS, default STREAM_IDLE_MS). Without this, a link that
|
|
1085
|
+
// answers headers then stalls is masked until undici's ~300s bodyTimeout — longer
|
|
1086
|
+
// than the direct path's bound, and the reaper exempts the link lease. The catch
|
|
1087
|
+
// below fails over while no real byte reached the client; after that it surfaces
|
|
1088
|
+
// honestly and releases the lease request-scoped.
|
|
1089
|
+
const linkIdleMs = Math.max(1_000, Number(process.env.MAXPOOL_LINK_IDLE_MS) || STREAM_IDLE_MS);
|
|
992
1090
|
try {
|
|
993
1091
|
for (;;) {
|
|
994
|
-
const
|
|
1092
|
+
const readP = reader.read();
|
|
1093
|
+
readP.catch(() => {});
|
|
1094
|
+
let idleTimer;
|
|
1095
|
+
const idle = new Promise((_, reject) => {
|
|
1096
|
+
idleTimer = setTimeout(
|
|
1097
|
+
() => reject(Object.assign(new Error('link body idle timeout'), { code: 'UPSTREAM_IDLE', linkBody: true })),
|
|
1098
|
+
linkIdleMs,
|
|
1099
|
+
);
|
|
1100
|
+
idleTimer.unref?.();
|
|
1101
|
+
});
|
|
1102
|
+
let r;
|
|
1103
|
+
try {
|
|
1104
|
+
r = await Promise.race([readP, idle]);
|
|
1105
|
+
} finally {
|
|
1106
|
+
clearTimeout(idleTimer);
|
|
1107
|
+
}
|
|
1108
|
+
const { done, value } = r;
|
|
995
1109
|
if (done) break;
|
|
996
1110
|
if (res.destroyed) { try { await reader.cancel(); } catch { } break; }
|
|
1111
|
+
// First REAL byte stops the keep-alive (review r2 D — UNCONDITIONAL, r4 M1:
|
|
1112
|
+
// a conditional clear let pings land inside the real stream when the grace
|
|
1113
|
+
// timer had already fired).
|
|
1114
|
+
if (requestInfo.queueHeartbeatActive) clearQueueHeartbeat(requestInfo);
|
|
997
1115
|
res.write(Buffer.from(value));
|
|
998
1116
|
}
|
|
999
1117
|
} finally { try { await reader.cancel(); } catch { /* already done */ } }
|
|
1000
1118
|
}
|
|
1001
|
-
res.end();
|
|
1119
|
+
res.end(); // an empty 2xx body: the heartbeat reaps itself on res.writableEnded
|
|
1002
1120
|
finishLink({ success: ctx.status < 500, status: ctx.status });
|
|
1003
1121
|
// SC22 no-double-count: this request was counted at auth; the serving pool
|
|
1004
1122
|
// counted it too. Undo the local one.
|
|
@@ -1008,21 +1126,49 @@ async function forwardRequest(
|
|
|
1008
1126
|
}
|
|
1009
1127
|
hooks.onRequestRouted?.(reqId, { account: account.name });
|
|
1010
1128
|
} catch (err) {
|
|
1011
|
-
|
|
1129
|
+
const heldS = heldSeconds(lease);
|
|
1130
|
+
// D1: the client left before anything reached it. The account did nothing wrong
|
|
1131
|
+
// and the turn is abandoned — release neutral, do not exclude, never re-send.
|
|
1132
|
+
// (A cancel AFTER bytes flowed keeps the path below: a mid-response failure.)
|
|
1133
|
+
if (linkClientGone && !clientHasRealBytes(res, requestInfo)) {
|
|
1134
|
+
finishLink({ success: false, neutral: true, error: 'client_cancelled' });
|
|
1135
|
+
clearQueueHeartbeat(requestInfo);
|
|
1136
|
+
console.log(`[Maxpool] Client left after ${heldS}s on "${account.name}" (link, still waiting) — released, not retried`);
|
|
1137
|
+
logFailedAttempt(account.name, 'client_cancelled', `client left after ${heldS}s, nothing sent`, 'released-cancel');
|
|
1138
|
+
return;
|
|
1139
|
+
}
|
|
1140
|
+
// Everything that reaches this catch is request-scoped (D4): a remote 429/503 is
|
|
1141
|
+
// handled — account-scoped — by its own branch above before anything can throw, so
|
|
1142
|
+
// only a TTFB abort, a mid-stream cancel or an unknown link error lands here. It
|
|
1143
|
+
// must not release every other conversation held on this row.
|
|
1144
|
+
const linkKind = linkClientGone ? 'client_cancelled'
|
|
1145
|
+
: err?.linkBody ? 'link_idle'
|
|
1146
|
+
: err?.name === 'AbortError' ? 'link_ttfb'
|
|
1147
|
+
: 'pool_link_unreachable';
|
|
1148
|
+
finishLink({ status: 502, error: 'pool_link_unreachable', scope: 'request' });
|
|
1012
1149
|
excludedIndexes.add(account.index); // P4b: never re-pick a row that just failed
|
|
1150
|
+
const linkDetail = `${linkStatus == null ? 'no answer' : `HTTP ${linkStatus}`} after ${heldS}s (${err?.message || err})`;
|
|
1013
1151
|
if (!clientHasRealBytes(res, requestInfo)) {
|
|
1014
1152
|
if (retryCount + 1 < maxAttempts && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
1153
|
+
logFailedAttempt(account.name, linkKind, linkDetail, 'moved');
|
|
1154
|
+
detachLinkClose();
|
|
1015
1155
|
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
|
|
1016
1156
|
}
|
|
1157
|
+
detachLinkClose();
|
|
1017
1158
|
const queued = await queueAndRetry(
|
|
1018
1159
|
`all routes failed after a link error on "${account.name}"`,
|
|
1019
1160
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1020
1161
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'network',
|
|
1162
|
+
{ attempt: [account.name, linkKind, linkDetail] },
|
|
1021
1163
|
);
|
|
1022
1164
|
if (queued) return;
|
|
1165
|
+
} else {
|
|
1166
|
+
logFailedAttempt(account.name, linkKind, `${linkDetail}, output already sent`, 'surfaced');
|
|
1023
1167
|
}
|
|
1024
1168
|
ctx.status = 502;
|
|
1025
1169
|
sendErrorResponse(res, requestInfo, 502, { type: 'error', error: { type: 'api_error', message: `Team pool "${account.name}" unreachable: ${err?.message || err}` } });
|
|
1170
|
+
} finally {
|
|
1171
|
+
detachLinkClose();
|
|
1026
1172
|
}
|
|
1027
1173
|
return;
|
|
1028
1174
|
}
|
|
@@ -1081,9 +1227,12 @@ async function forwardRequest(
|
|
|
1081
1227
|
// arrive — the streaming body is then governed by the per-chunk idle guard, never
|
|
1082
1228
|
// this fixed clock, so a long healthy stream is never cut.
|
|
1083
1229
|
const ttfbController = new AbortController();
|
|
1230
|
+
const ttfbCeilingMs = clientWaitCeilingMs(requestInfo, queueConfig);
|
|
1231
|
+
const ttfbMs = scaledTtfbMs(UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, requestInfo.bodyBytes, ttfbCeilingMs);
|
|
1232
|
+
noteScaledTtfb(requestInfo, account.name, 'direct', ttfbMs, UPSTREAM_TTFB_MS, ttfbCeilingMs);
|
|
1084
1233
|
const ttfbTimer = setTimeout(
|
|
1085
1234
|
() => ttfbController.abort(Object.assign(new Error('upstream TTFB timeout'), { code: 'UPSTREAM_TTFB' })),
|
|
1086
|
-
|
|
1235
|
+
ttfbMs,
|
|
1087
1236
|
);
|
|
1088
1237
|
ttfbTimer.unref?.();
|
|
1089
1238
|
// Proactive streaming keep-alive. UPSTREAM_TTFB_MS (120s) is FAR above the
|
|
@@ -1095,21 +1244,13 @@ async function forwardRequest(
|
|
|
1095
1244
|
// grace, commit the SSE stream + heartbeat so the client stays alive. The
|
|
1096
1245
|
// deadline is anchored ONCE per request (??=) so it also bounds CUMULATIVE
|
|
1097
1246
|
// failover time across re-forwards, not each attempt independently.
|
|
1098
|
-
const
|
|
1099
|
-
? 10000
|
|
1100
|
-
: Math.max(0, Number(queueConfig.streamForwardGraceMs) || 0);
|
|
1101
|
-
let streamGraceTimer = null;
|
|
1102
|
-
if (requestInfo.stream && !res.headersSent) {
|
|
1103
|
-
requestInfo.streamGraceDeadline ??= Date.now() + streamForwardGraceMs;
|
|
1104
|
-
const graceDelay = Math.max(0, requestInfo.streamGraceDeadline - Date.now());
|
|
1105
|
-
streamGraceTimer = setTimeout(
|
|
1106
|
-
() => commitStreamGraceHeartbeat(res, requestInfo, queueConfig, accountManager),
|
|
1107
|
-
graceDelay,
|
|
1108
|
-
);
|
|
1109
|
-
streamGraceTimer.unref?.();
|
|
1110
|
-
}
|
|
1247
|
+
const streamGraceTimer = armStreamForwardGrace(res, requestInfo, queueConfig, accountManager);
|
|
1111
1248
|
try {
|
|
1112
1249
|
let upstreamRes;
|
|
1250
|
+
// D (finding 5): THIS attempt's upstream forward start. ttfb is measured from
|
|
1251
|
+
// here — never from the request's arrival, which includes queue wait and the
|
|
1252
|
+
// earlier attempts of a failover.
|
|
1253
|
+
const upstreamForwardStartAt = Date.now();
|
|
1113
1254
|
try {
|
|
1114
1255
|
upstreamRes = await fetch(upstreamUrl, {
|
|
1115
1256
|
method,
|
|
@@ -1124,6 +1265,14 @@ async function forwardRequest(
|
|
|
1124
1265
|
// a re-forward (failover) re-arms for the REMAINING time to the shared deadline.
|
|
1125
1266
|
if (streamGraceTimer) clearTimeout(streamGraceTimer);
|
|
1126
1267
|
}
|
|
1268
|
+
// D (2026-11-10, finding 5): the upstream ANSWERED (fetch resolved — a failed
|
|
1269
|
+
// or timed-out fetch threw above and never reaches here). Stamped on EVERY
|
|
1270
|
+
// attempt together with that attempt's forward start, so after a failover the
|
|
1271
|
+
// pair describes the attempt that actually served (a quick 429 then a slow
|
|
1272
|
+
// second account reports the slow one). The queue heartbeat never stamps it.
|
|
1273
|
+
// Log-only: no setting reads it.
|
|
1274
|
+
requestInfo._upstreamForwardStartAt = upstreamForwardStartAt;
|
|
1275
|
+
requestInfo._firstUpstreamByteAt = Date.now();
|
|
1127
1276
|
// Response arrived — the pre-response leak window is over. Stop guarding for
|
|
1128
1277
|
// client-disconnect via abort (streamResponse handles mid-stream disconnects).
|
|
1129
1278
|
res.off('close', onClientClose);
|
|
@@ -1183,16 +1332,18 @@ async function forwardRequest(
|
|
|
1183
1332
|
if (
|
|
1184
1333
|
canRetryBufferedBody
|
|
1185
1334
|
&& retryCount + 1 < maxAttempts
|
|
1186
|
-
&& !res
|
|
1335
|
+
&& !clientHasRealBytes(res, requestInfo)
|
|
1187
1336
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1188
1337
|
) {
|
|
1189
1338
|
if (logDir) writeRequestLog(logDir, reqId, logSections);
|
|
1339
|
+
logFailedAttempt(account.name, 'request_shape_refused', 'subscription refused the request shape (not Claude Code)', 'moved');
|
|
1190
1340
|
return forwardRequest(
|
|
1191
1341
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1192
1342
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
1193
1343
|
);
|
|
1194
1344
|
}
|
|
1195
1345
|
if (logDir) writeRequestLog(logDir, reqId, logSections);
|
|
1346
|
+
logFailedAttempt(account.name, 'request_shape_refused', 'subscription refused the request shape; no route serves it', 'surfaced');
|
|
1196
1347
|
ctx.status = 429;
|
|
1197
1348
|
sendErrorResponse(res, requestInfo, 429, {
|
|
1198
1349
|
type: 'error',
|
|
@@ -1222,9 +1373,10 @@ async function forwardRequest(
|
|
|
1222
1373
|
if (
|
|
1223
1374
|
canRetryBufferedBody
|
|
1224
1375
|
&& retryCount + 1 < maxAttempts
|
|
1225
|
-
&& !res
|
|
1376
|
+
&& !clientHasRealBytes(res, requestInfo)
|
|
1226
1377
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1227
1378
|
) {
|
|
1379
|
+
logFailedAttempt(account.name, 'upstream_429', `server-side throttle, retry-after ${retryAfter}s`, 'moved');
|
|
1228
1380
|
return forwardRequest(
|
|
1229
1381
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1230
1382
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -1243,6 +1395,7 @@ async function forwardRequest(
|
|
|
1243
1395
|
'Anthropic upstream is temporarily limiting requests',
|
|
1244
1396
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1245
1397
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'upstream_throttle',
|
|
1398
|
+
{ attempt: [account.name, 'upstream_429', 'request-wide server-side throttle'] },
|
|
1246
1399
|
);
|
|
1247
1400
|
if (queued) return;
|
|
1248
1401
|
|
|
@@ -1261,6 +1414,7 @@ async function forwardRequest(
|
|
|
1261
1414
|
`all routes failed after server-side 429 from "${account.name}"`,
|
|
1262
1415
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1263
1416
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'capacity',
|
|
1417
|
+
{ attempt: [account.name, 'upstream_429', 'server-side throttle; no other route'] },
|
|
1264
1418
|
);
|
|
1265
1419
|
if (queued) return;
|
|
1266
1420
|
|
|
@@ -1288,6 +1442,7 @@ async function forwardRequest(
|
|
|
1288
1442
|
'Anthropic upstream is temporarily limiting requests',
|
|
1289
1443
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1290
1444
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'upstream_throttle',
|
|
1445
|
+
{ attempt: [account.name, 'upstream_429', 'matching ambiguous 429s — shared throttle opened'] },
|
|
1291
1446
|
);
|
|
1292
1447
|
if (queued) return;
|
|
1293
1448
|
|
|
@@ -1302,6 +1457,27 @@ async function forwardRequest(
|
|
|
1302
1457
|
fingerprint: rateLimit.scope === 'unknown' ? rateLimit.fingerprint : null,
|
|
1303
1458
|
modelScope: rateLimit.modelScope || null,
|
|
1304
1459
|
});
|
|
1460
|
+
// C2 REACTIVE CLAIM (2026-11-10): a 429 on the serving lease claims a held
|
|
1461
|
+
// reset AT ONCE, not after a full prober sweep. Only on the lease's LOCAL
|
|
1462
|
+
// account holding the credential (never a mirror row — its 429 is the
|
|
1463
|
+
// cloud's, and the cloud claims on its own side). A weekly-CLASS 429 (the
|
|
1464
|
+
// 7d header itself ≥ the wall) claims a general grant; a scoped 429 claims
|
|
1465
|
+
// only if the probe MEASURED that model's bucket at the wall. Uses the
|
|
1466
|
+
// current token; a 401 defers to the sweep.
|
|
1467
|
+
// FIRE-AND-FORGET (finding 4): the claim (profile GET + POST) runs OFF the
|
|
1468
|
+
// user's request — this request fails over at once; the claim unbenches the
|
|
1469
|
+
// account for the NEXT request when it lands. Never awaited here.
|
|
1470
|
+
if (accountManager.tryReactiveClaim && !account.remote) {
|
|
1471
|
+
const weeklyHdr = Number(rateLimitHeaders['anthropic-ratelimit-unified-7d-utilization']);
|
|
1472
|
+
const weeklyWall = Number.isFinite(weeklyHdr) && weeklyHdr >= 0.999;
|
|
1473
|
+
const kind = weeklyWall ? 'weekly' : (rateLimit.modelScope || null);
|
|
1474
|
+
if (kind) {
|
|
1475
|
+
try {
|
|
1476
|
+
Promise.resolve(accountManager.tryReactiveClaim(account, { kind, weeklyUtil: weeklyHdr }))
|
|
1477
|
+
.catch(() => { /* best-effort; the sweep retries */ });
|
|
1478
|
+
} catch { /* a synchronous throw must not break the failover below */ }
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1305
1481
|
accountManager.releaseAccount(lease, { status: 429, error: rateLimit.modelScope ? 'model_rate_limited' : 'rate_limited' });
|
|
1306
1482
|
|
|
1307
1483
|
if (logDir) {
|
|
@@ -1310,13 +1486,15 @@ async function forwardRequest(
|
|
|
1310
1486
|
}
|
|
1311
1487
|
console.log(`[Maxpool] 429 on "${account.name}" — failing over before first byte`);
|
|
1312
1488
|
excludedIndexes.add(account.index);
|
|
1489
|
+
const rlKind = rateLimit.modelScope ? 'model_rate_limited' : 'rate_limited';
|
|
1313
1490
|
|
|
1314
1491
|
if (
|
|
1315
1492
|
canRetryBufferedBody &&
|
|
1316
1493
|
retryCount + 1 < maxAttempts &&
|
|
1317
|
-
!res
|
|
1494
|
+
!clientHasRealBytes(res, requestInfo) &&
|
|
1318
1495
|
hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1319
1496
|
) {
|
|
1497
|
+
logFailedAttempt(account.name, rlKind, `HTTP 429, benched ${retryAfter}s`, 'moved');
|
|
1320
1498
|
return forwardRequest(
|
|
1321
1499
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1322
1500
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -1327,6 +1505,7 @@ async function forwardRequest(
|
|
|
1327
1505
|
`all routes failed after 429 from "${account.name}"`,
|
|
1328
1506
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1329
1507
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canRetryBufferedBody, 'quota',
|
|
1508
|
+
{ attempt: [account.name, rlKind, `HTTP 429, benched ${retryAfter}s; no other route`] },
|
|
1330
1509
|
);
|
|
1331
1510
|
if (queued) return;
|
|
1332
1511
|
|
|
@@ -1369,19 +1548,22 @@ async function forwardRequest(
|
|
|
1369
1548
|
if (errorBody) logSections.push(`=== ERROR BODY ===\n${errorBody}`);
|
|
1370
1549
|
}
|
|
1371
1550
|
console.log(`[Maxpool] ${upstreamRes.status} on provider "${account.name}" — ${forbidden ? `cooled down ${PROVIDER_FORBIDDEN_COOLDOWN_SEC}s (recoverable)` : 'disabled'} and failing over before first byte`);
|
|
1551
|
+
const authKind = forbidden ? 'provider_403' : 'auth_failed';
|
|
1372
1552
|
|
|
1373
1553
|
if (
|
|
1374
1554
|
canRetryBufferedBody &&
|
|
1375
1555
|
retryCount + 1 < maxAttempts &&
|
|
1376
|
-
!res
|
|
1556
|
+
!clientHasRealBytes(res, requestInfo) &&
|
|
1377
1557
|
hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1378
1558
|
) {
|
|
1559
|
+
logFailedAttempt(account.name, authKind, `HTTP ${upstreamRes.status}`, 'moved');
|
|
1379
1560
|
return forwardRequest(
|
|
1380
1561
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1381
1562
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
1382
1563
|
);
|
|
1383
1564
|
}
|
|
1384
1565
|
|
|
1566
|
+
logFailedAttempt(account.name, authKind, `HTTP ${upstreamRes.status}; no other route`, 'surfaced');
|
|
1385
1567
|
ctx.status = 502;
|
|
1386
1568
|
if (logDir) writeRequestLog(logDir, reqId, logSections);
|
|
1387
1569
|
sendErrorResponse(res, requestInfo, 502, {
|
|
@@ -1416,9 +1598,10 @@ async function forwardRequest(
|
|
|
1416
1598
|
if (
|
|
1417
1599
|
canRetryBufferedBody
|
|
1418
1600
|
&& retryCount + 1 < maxAttempts
|
|
1419
|
-
&& !res
|
|
1601
|
+
&& !clientHasRealBytes(res, requestInfo)
|
|
1420
1602
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1421
1603
|
) {
|
|
1604
|
+
logFailedAttempt(account.name, 'upstream_529', `overloaded, retry-after ${retryAfter}s`, 'moved');
|
|
1422
1605
|
return forwardRequest(
|
|
1423
1606
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1424
1607
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -1434,6 +1617,7 @@ async function forwardRequest(
|
|
|
1434
1617
|
'Anthropic upstream is overloaded',
|
|
1435
1618
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1436
1619
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'upstream_throttle',
|
|
1620
|
+
{ attempt: [account.name, 'upstream_529', 'request-wide overload — shared throttle opened'] },
|
|
1437
1621
|
);
|
|
1438
1622
|
if (queued) return;
|
|
1439
1623
|
|
|
@@ -1452,6 +1636,7 @@ async function forwardRequest(
|
|
|
1452
1636
|
`all routes failed after HTTP 529 from "${account.name}"`,
|
|
1453
1637
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1454
1638
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'capacity',
|
|
1639
|
+
{ attempt: [account.name, 'upstream_529', 'overloaded; no other route'] },
|
|
1455
1640
|
);
|
|
1456
1641
|
if (queued) return;
|
|
1457
1642
|
|
|
@@ -1471,6 +1656,7 @@ async function forwardRequest(
|
|
|
1471
1656
|
}
|
|
1472
1657
|
|
|
1473
1658
|
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
|
|
1659
|
+
logFailedAttempt(account.name, `upstream_${upstreamRes.status}`, 'retriable upstream status, cooling down', 'moved');
|
|
1474
1660
|
return forwardRequest(
|
|
1475
1661
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1476
1662
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -1481,6 +1667,7 @@ async function forwardRequest(
|
|
|
1481
1667
|
`all routes failed after ${upstreamRes.status} from "${account.name}"`,
|
|
1482
1668
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1483
1669
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canRetryBufferedBody, 'capacity',
|
|
1670
|
+
{ attempt: [account.name, `upstream_${upstreamRes.status}`, 'retriable upstream status; no other route'] },
|
|
1484
1671
|
);
|
|
1485
1672
|
if (queued) return;
|
|
1486
1673
|
|
|
@@ -1631,6 +1818,7 @@ async function forwardRequest(
|
|
|
1631
1818
|
accountManager.revertSessionBinding(requestInfo.sessionKey, lease.migratedFromName);
|
|
1632
1819
|
excludedIndexes.add(account.index);
|
|
1633
1820
|
console.log(`[Maxpool] Thinking-signature fail-safe: reverting session to issuer "${lease.migratedFromName}" and retrying`);
|
|
1821
|
+
logFailedAttempt(account.name, 'invalid_thinking_signature', `migrated replay rejected; back to issuer "${lease.migratedFromName}"`, 'moved');
|
|
1634
1822
|
return forwardRequest(
|
|
1635
1823
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1636
1824
|
retryConfig, queueConfig, { ...requestInfo, pinnedAccountName: lease.migratedFromName },
|
|
@@ -1783,42 +1971,27 @@ async function forwardRequest(
|
|
|
1783
1971
|
// session) and retry PROVIDER-only, rather than surfacing the 400. Only worth
|
|
1784
1972
|
// it when a provider can actually serve it (profile=all with a GLM/Kimi token).
|
|
1785
1973
|
//
|
|
1786
|
-
//
|
|
1787
|
-
//
|
|
1788
|
-
//
|
|
1789
|
-
//
|
|
1790
|
-
//
|
|
1791
|
-
//
|
|
1792
|
-
//
|
|
1793
|
-
// converted body replays 200 (verified live, count_tokens, 2026-11-06).
|
|
1974
|
+
// NO post-rejection conversion retry (review r2, 2026-11-09): detectTranscriptOrigin
|
|
1975
|
+
// flags exactly the ids the converter can convert, so the up-front strip has
|
|
1976
|
+
// already converted every curable block before the first send (pinned by
|
|
1977
|
+
// glm-mcp-two-turn F2). A rejection that reaches here has nothing convertible —
|
|
1978
|
+
// re-sending it was a guaranteed identical 400, and its latch stripped valid Claude
|
|
1979
|
+
// thinking from every later turn. Move to GLM/Kimi when one is available; else
|
|
1980
|
+
// the original 400 surfaces (below).
|
|
1794
1981
|
const providerAvailable = accountManager.accounts?.some(a => a.type === 'provider' && a.enabled !== false);
|
|
1795
|
-
// Exact-shape gate (red-team M2)
|
|
1796
|
-
// fires on unrelated unknown-tag 400s (its
|
|
1797
|
-
// server_tool_use)
|
|
1798
|
-
// deterministic server-tool-id wording, never the fuzzy predicate.
|
|
1799
|
-
const
|
|
1982
|
+
// Exact-shape gate (red-team M2) for the surfaced attempt line: the loose
|
|
1983
|
+
// isAnthropicIncompatBody matcher also fires on unrelated unknown-tag 400s (its
|
|
1984
|
+
// expected-tags list mentions server_tool_use), so key on the two MCP wordings or
|
|
1985
|
+
// the deterministic server-tool-id wording, never the fuzzy predicate.
|
|
1986
|
+
const incompatTagRejection = /server_tool_use\.id: String should match pattern/i.test(errorBody)
|
|
1800
1987
|
|| /mcp_tool_(use|result)/.test(errorBody)
|
|
1801
1988
|
&& /(does not match any of the expected tags|without a corresponding)/i.test(errorBody);
|
|
1802
|
-
if (anthropicIncompat && incompatRepairable && !requestInfo.incompatRepairTried && canRepairBody) {
|
|
1803
|
-
const repaired = stripForeignThinkingBlocks(body);
|
|
1804
|
-
if (repaired.body) {
|
|
1805
|
-
// Latch the pre-strip sticky so the NEXT turn repairs up front instead of
|
|
1806
|
-
// re-paying this rejected round-trip every turn (the client resends the whole
|
|
1807
|
-
// poisoned history each turn — same cache-churn the audit is about).
|
|
1808
|
-
accountManager.markSessionThinkingContaminated?.(requestInfo.sessionKey);
|
|
1809
|
-
console.log(`[Maxpool] Recovering session on Claude: incompatibility repair removed ${repaired.removed} thinking block(s), converted ${repaired.converted} provider tool block(s) — retrying on Claude before pinning`);
|
|
1810
|
-
return forwardRequest(
|
|
1811
|
-
req, res, repaired.body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1812
|
-
retryConfig, queueConfig, { ...requestInfo, incompatRepairTried: true, anthropicIncompatible: false, repairCount: repairCount + 1 },
|
|
1813
|
-
repaired.body.length <= retryConfig.maxRetryBufferBytes, canQueueBufferedBody, excludedIndexes,
|
|
1814
|
-
);
|
|
1815
|
-
}
|
|
1816
|
-
}
|
|
1817
1989
|
if (anthropicIncompat && requestInfo.sessionKey && providerAvailable
|
|
1818
1990
|
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
|
|
1819
1991
|
accountManager.markSessionIncompatible?.(requestInfo.sessionKey, requestInfo.homeProvider);
|
|
1820
1992
|
excludedIndexes.add(account.index);
|
|
1821
1993
|
console.log(`[Maxpool] Anthropic rejected this transcript (server_tool_use/thinking); pinning session to GLM/Kimi and retrying`);
|
|
1994
|
+
logFailedAttempt(account.name, 'transcript_rejected', 'Anthropic rejected the transcript; pinning to GLM/Kimi', 'moved');
|
|
1822
1995
|
return forwardRequest(
|
|
1823
1996
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1824
1997
|
retryConfig, queueConfig, { ...requestInfo, anthropicIncompatible: true },
|
|
@@ -1846,6 +2019,7 @@ async function forwardRequest(
|
|
|
1846
2019
|
if (a.type === 'provider') excludedIndexes.add(a.index);
|
|
1847
2020
|
}
|
|
1848
2021
|
console.log(`[Maxpool] Provider "${account.name}" context too small for this request; pinning session to Claude and retrying`);
|
|
2022
|
+
logFailedAttempt(account.name, 'provider_context_too_small', 'request exceeds the provider context; pinning to Claude', 'moved');
|
|
1849
2023
|
return forwardRequest(
|
|
1850
2024
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1851
2025
|
retryConfig, queueConfig, { ...requestInfo, largeContext: true },
|
|
@@ -1863,6 +2037,7 @@ async function forwardRequest(
|
|
|
1863
2037
|
if (a.type === 'provider') excludedIndexes.add(a.index);
|
|
1864
2038
|
}
|
|
1865
2039
|
console.log(`[Maxpool] Provider "${account.name}" rejected this request's shape (${String(errorBody).slice(0, 120)}); retrying on Claude`);
|
|
2040
|
+
logFailedAttempt(account.name, 'provider_shape_rejected', 'provider will not take this body; retrying on Claude', 'moved');
|
|
1866
2041
|
return forwardRequest(
|
|
1867
2042
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1868
2043
|
retryConfig, queueConfig, requestInfo,
|
|
@@ -1918,6 +2093,9 @@ async function forwardRequest(
|
|
|
1918
2093
|
return;
|
|
1919
2094
|
}
|
|
1920
2095
|
|
|
2096
|
+
if (anthropicIncompat && incompatTagRejection) {
|
|
2097
|
+
logFailedAttempt(account.name, 'transcript_rejected', 'Anthropic rejected the transcript; nothing convertible to repair', 'surfaced');
|
|
2098
|
+
}
|
|
1921
2099
|
ctx.status = upstreamRes.status;
|
|
1922
2100
|
sendErrorBody(res, requestInfo, upstreamRes.status, errorBody, upstreamRes.headers);
|
|
1923
2101
|
return;
|
|
@@ -2082,8 +2260,11 @@ async function forwardRequest(
|
|
|
2082
2260
|
// was still waiting" — and the elapsed time is what exposes a client watchdog firing
|
|
2083
2261
|
// at its floor while maxpool was still happily holding the request.
|
|
2084
2262
|
const heldMs = Date.now() - (requestInfo.startedAt || Date.now());
|
|
2263
|
+
// clientHasRealBytes, not headersSent: the keep-alive commits headers with only pings.
|
|
2264
|
+
const gotOutput = clientHasRealBytes(res, requestInfo);
|
|
2085
2265
|
console.log(`[Maxpool] Client left after ${(heldMs / 1000).toFixed(1)}s on "${account.name}" `
|
|
2086
|
-
+ `(${
|
|
2266
|
+
+ `(${gotOutput ? 'mid-response — output already sent' : 'still waiting, nothing sent'})`);
|
|
2267
|
+
logFailedAttempt(account.name, 'client_cancelled', `client left after ${(heldMs / 1000).toFixed(1)}s (${gotOutput ? 'mid-response' : 'still waiting'})`, 'released-cancel');
|
|
2087
2268
|
releaseOnClientGone();
|
|
2088
2269
|
return;
|
|
2089
2270
|
}
|
|
@@ -2103,7 +2284,9 @@ async function forwardRequest(
|
|
|
2103
2284
|
// provider. If no provider is available,
|
|
2104
2285
|
// fall through to the normal error path — the client sees the same message it
|
|
2105
2286
|
// would have seen before this feature existed.
|
|
2106
|
-
|
|
2287
|
+
// !clientHasRealBytes, not !headersSent: a slow turn whose keep-alive committed
|
|
2288
|
+
// (pings only) is just as reroutable as a fast one (review r2 B).
|
|
2289
|
+
if (err.code === 'REFUSAL_RETRY' && canRetryBufferedBody && !clientHasRealBytes(res, requestInfo)) {
|
|
2107
2290
|
const category = err.refusalCategory || 'unknown';
|
|
2108
2291
|
const anthropicIndexes = accountManager.accounts
|
|
2109
2292
|
.filter(a => a.type !== 'provider')
|
|
@@ -2115,6 +2298,7 @@ async function forwardRequest(
|
|
|
2115
2298
|
// 2026-09-14: removing a latch changed nothing on any constructible path.
|
|
2116
2299
|
if (providerAvailable && retryCount + 1 <= maxAttempts) {
|
|
2117
2300
|
console.log(`[Maxpool] Anthropic safeguard refusal (${category}) on "${account.name}" — rerouting turn to a provider account`);
|
|
2301
|
+
logFailedAttempt(account.name, 'safeguard_refusal', `${category}; rerouting to a provider`, 'moved');
|
|
2118
2302
|
accountManager.releaseAccount(lease, { success: true, status: 200, refusal: category });
|
|
2119
2303
|
return forwardRequest(
|
|
2120
2304
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
@@ -2146,10 +2330,41 @@ async function forwardRequest(
|
|
|
2146
2330
|
// Network-class (ECONNRESET / fetch failed / timeout / terminated): short fixed
|
|
2147
2331
|
// cooldown, no exponential escalation — the fleet auto-recovers seconds after
|
|
2148
2332
|
// connectivity returns instead of being benched for up to 15 min.
|
|
2149
|
-
|
|
2333
|
+
const transientKind = err.code || (err.message || 'network_error').slice(0, 40);
|
|
2334
|
+
// D3 retry-same: a first-byte timeout on a BIG prompt goes back to the SAME
|
|
2335
|
+
// account once — no cooldown (it would make the account unavailable to its own
|
|
2336
|
+
// retry), no exclusion, through the normal acquire path via a one-hop hint (P1).
|
|
2337
|
+
// The second timeout falls through to the move path below.
|
|
2338
|
+
if (err.code === 'UPSTREAM_TTFB' && canRetryBufferedBody && retrySameEligible(account, requestInfo, res, queueConfig)) {
|
|
2339
|
+
requestInfo.sameAccountRetried = true;
|
|
2340
|
+
requestInfo.retrySameAccountName = account.name;
|
|
2341
|
+
accountManager.releaseAccount(lease, { error: 'upstream_ttfb', scope: 'request' });
|
|
2342
|
+
requestInfo.pendingRetrySame = { account: account.name, kind: transientKind,
|
|
2343
|
+
detail: `no first byte after ${heldSeconds(lease)}s on a ${Math.round((requestInfo.bodyBytes || 0) / 1000)}KB prompt` };
|
|
2344
|
+
// retryCount NOT bumped: maxAttempts is sized by ACCOUNT COUNT (failover budget);
|
|
2345
|
+
// a same-account retry is not a failover and must not eat the move it may still
|
|
2346
|
+
// need (2 accounts: retry-same + move = 2 hops). sameAccountRetried bounds it.
|
|
2347
|
+
return forwardRequest(
|
|
2348
|
+
req, res, body, accountManager, upstream, retryCount, hooks, reqId, ctx, logDir,
|
|
2349
|
+
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
2350
|
+
);
|
|
2351
|
+
}
|
|
2352
|
+
// A first-byte / idle / stalled-body timeout is REQUEST-scoped (D4, review r2 A): it soft-cools the
|
|
2353
|
+
// account for NEW placements only — conversations held on it keep it — and only 3
|
|
2354
|
+
// in a row escalate to an account fault. Every other network class (reset, DNS,
|
|
2355
|
+
// fetch failed) is fleet-wide connectivity: the short fixed account cooldown. The
|
|
2356
|
+
// release carries no error/status, so it moves no counter itself.
|
|
2357
|
+
const requestScoped = isRequestScopedTimeout(err);
|
|
2358
|
+
accountManager.markTransientFailure(account.index, err.code || err.message || 'network_error',
|
|
2359
|
+
{ network: true, scope: requestScoped ? 'request' : 'account' });
|
|
2150
2360
|
accountManager.releaseAccount(lease);
|
|
2151
2361
|
excludedIndexes.add(account.index);
|
|
2152
|
-
|
|
2362
|
+
// hasEligibleRoute like every sibling gate: with no route left the network-cause
|
|
2363
|
+
// hold below is the honest path (a move would land in the no-account queue and
|
|
2364
|
+
// answer with a misleading quota message).
|
|
2365
|
+
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2366
|
+
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
2367
|
+
logFailedAttempt(account.name, transientKind, `transient after ${heldSeconds(lease)}s`, 'moved');
|
|
2153
2368
|
return forwardRequest(
|
|
2154
2369
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
2155
2370
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -2159,6 +2374,7 @@ async function forwardRequest(
|
|
|
2159
2374
|
`all routes failed after network error from "${account.name}"`,
|
|
2160
2375
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
2161
2376
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canRetryBufferedBody, 'network',
|
|
2377
|
+
{ attempt: [account.name, transientKind, 'transient; no other route'] },
|
|
2162
2378
|
);
|
|
2163
2379
|
if (queued) return;
|
|
2164
2380
|
ctx.status = 503;
|
|
@@ -2188,6 +2404,7 @@ async function forwardRequest(
|
|
|
2188
2404
|
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
|
|
2189
2405
|
account.status = 'error';
|
|
2190
2406
|
excludedIndexes.add(account.index);
|
|
2407
|
+
logFailedAttempt(account.name, (err.code || err.message || 'error').toString().slice(0, 40), err.message, 'moved');
|
|
2191
2408
|
return forwardRequest(
|
|
2192
2409
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
2193
2410
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -2197,6 +2414,7 @@ async function forwardRequest(
|
|
|
2197
2414
|
`all routes failed after proxy error from "${account.name}"`,
|
|
2198
2415
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
2199
2416
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canRetryBufferedBody, 'proxy',
|
|
2417
|
+
{ attempt: [account.name, (err.code || err.message || 'error').toString().slice(0, 40), `no other route after ${err.message}`] },
|
|
2200
2418
|
);
|
|
2201
2419
|
if (queued) return;
|
|
2202
2420
|
ctx.status = 502;
|
|
@@ -2208,6 +2426,83 @@ async function forwardRequest(
|
|
|
2208
2426
|
}
|
|
2209
2427
|
}
|
|
2210
2428
|
|
|
2429
|
+
/**
|
|
2430
|
+
* The longest this request may sit waiting for a first byte before the CLIENT gives up.
|
|
2431
|
+
* Streaming: the client's own tolerance (x-maxpool-client-stream-idle-ms × 0.8, else the
|
|
2432
|
+
* conservative streamClientToleranceMs = 240s for a bare `claude`), never past the CLI's
|
|
2433
|
+
* absolute 30-min stream ceiling. The pre-headers keep-alive (streamForwardGraceMs) is
|
|
2434
|
+
* what keeps the client's idle watchdog fed during that wait, on both paths.
|
|
2435
|
+
* Non-streaming: nothing keeps the client alive pre-headers, so no scaling at all — the
|
|
2436
|
+
* ceiling IS the flat floor (null = "use the floor").
|
|
2437
|
+
*/
|
|
2438
|
+
function clientWaitCeilingMs(requestInfo = {}, queueConfig = {}) {
|
|
2439
|
+
if (!requestInfo.stream) return null;
|
|
2440
|
+
const tolerance = Number.isFinite(requestInfo.clientToleranceMs)
|
|
2441
|
+
? requestInfo.clientToleranceMs
|
|
2442
|
+
: Math.max(0, Number(queueConfig?.streamClientToleranceMs ?? DEFAULT_QUEUE.streamClientToleranceMs) || 0);
|
|
2443
|
+
return Math.min(tolerance, CLI_STREAM_CEILING_MS);
|
|
2444
|
+
}
|
|
2445
|
+
|
|
2446
|
+
/** D2: first-byte wait for one attempt. Never below `floorMs`; scaled part clamped at `ceilingMs`. */
|
|
2447
|
+
function scaledTtfbMs(floorMs, msPerByte, bodyBytes, ceilingMs) {
|
|
2448
|
+
if (ceilingMs == null) return floorMs; // 0 ms/byte needs no branch: max(floor, 0) = floor
|
|
2449
|
+
const scaled = Math.ceil((Number(bodyBytes) || 0) * msPerByte);
|
|
2450
|
+
return Math.max(floorMs, Math.min(scaled, ceilingMs));
|
|
2451
|
+
}
|
|
2452
|
+
|
|
2453
|
+
/** Log the chosen first-byte wait once per request, only when it exceeds the floor. */
|
|
2454
|
+
function noteScaledTtfb(requestInfo, accountName, path, waitMs, floorMs, ceilingMs) {
|
|
2455
|
+
if (waitMs <= floorMs || requestInfo._ttfbWaitLogged) return;
|
|
2456
|
+
requestInfo._ttfbWaitLogged = true;
|
|
2457
|
+
console.log(`[Maxpool] first-byte wait ${Math.round(waitMs / 1000)}s for ${Math.round((requestInfo.bodyBytes || 0) / 1000)}KB prompt on "${accountName}" (${path}; floor ${Math.round(floorMs / 1000)}s${waitMs >= ceilingMs ? ', clamped at client patience' : ''})`);
|
|
2458
|
+
}
|
|
2459
|
+
|
|
2460
|
+
/**
|
|
2461
|
+
* D3: may this first-byte timeout retry the SAME account instead of moving?
|
|
2462
|
+
* Only for a big prompt, once per request, on an account that is not throttled /
|
|
2463
|
+
* auth-dead / cooling, while nothing real has reached the client, and only when the
|
|
2464
|
+
* second wait can still end inside the client's patience. Direct path only.
|
|
2465
|
+
*/
|
|
2466
|
+
function retrySameEligible(account, requestInfo, res, queueConfig = {}) {
|
|
2467
|
+
if (requestInfo.sameAccountRetried) return false;
|
|
2468
|
+
if ((requestInfo.bodyBytes || 0) < BIG_PROMPT_BYTES) return false;
|
|
2469
|
+
if (!account || account.status === 'throttled' || account.status === 'error'
|
|
2470
|
+
|| account.subscriptionGone || account.refreshDead) return false;
|
|
2471
|
+
if (account.cooldownUntil && Date.now() < account.cooldownUntil) return false;
|
|
2472
|
+
// The second attempt must be able to FINISH inside the client's patience: a bare
|
|
2473
|
+
// `claude` (240s ceiling, hard 300s death) whose first wait already ran 240s cannot
|
|
2474
|
+
// get a first byte from a 240s retry — that is guaranteed-wasted load; move instead.
|
|
2475
|
+
// Non-streaming has no ceiling (null): its wait is the flat floor either way.
|
|
2476
|
+
const ceilingMs = clientWaitCeilingMs(requestInfo, queueConfig);
|
|
2477
|
+
if (ceilingMs != null) {
|
|
2478
|
+
const nextWaitMs = scaledTtfbMs(UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, requestInfo.bodyBytes, ceilingMs);
|
|
2479
|
+
const elapsedMs = Date.now() - (requestInfo.startedAt || Date.now());
|
|
2480
|
+
if (elapsedMs + nextWaitMs > ceilingMs) return false;
|
|
2481
|
+
}
|
|
2482
|
+
return !clientHasRealBytes(res, requestInfo);
|
|
2483
|
+
}
|
|
2484
|
+
|
|
2485
|
+
/**
|
|
2486
|
+
* D5: one line per FAILED attempt, naming the account, the error class and what the
|
|
2487
|
+
* pool did next — so "why did this move" is answerable from the log without a forensic
|
|
2488
|
+
* dig (the Mac pool logged failed attempts not at all before this). nextAction tokens:
|
|
2489
|
+
* released-cancel | retry-same | moved | queued | surfaced.
|
|
2490
|
+
*/
|
|
2491
|
+
function logFailedAttempt(account, kind, detail, nextAction) {
|
|
2492
|
+
console.log(`[Maxpool] attempt failed (${kind}) on "${account}" — ${detail} → ${nextAction}`);
|
|
2493
|
+
}
|
|
2494
|
+
|
|
2495
|
+
/** A first-byte, mid-stream idle or stalled non-streaming body timeout concerns ONE
|
|
2496
|
+
* request, not the account (review r2 A / r3 R3-1). Resets, DNS, fetch failures do not. */
|
|
2497
|
+
function isRequestScopedTimeout(err) {
|
|
2498
|
+
return err?.code === 'UPSTREAM_TTFB' || err?.code === 'UPSTREAM_IDLE' || err?.code === 'UPSTREAM_BODY';
|
|
2499
|
+
}
|
|
2500
|
+
|
|
2501
|
+
/** Seconds this lease has been held on its account, one decimal ("12.3"). */
|
|
2502
|
+
function heldSeconds(lease) {
|
|
2503
|
+
return ((Date.now() - (lease.startedAt || Date.now())) / 1000).toFixed(1);
|
|
2504
|
+
}
|
|
2505
|
+
|
|
2211
2506
|
function parseRetryAfter(value) {
|
|
2212
2507
|
if (value == null) return null;
|
|
2213
2508
|
const n = parseInt(value, 10);
|
|
@@ -2432,7 +2727,7 @@ function isContextLengthError(errorBody) {
|
|
|
2432
2727
|
return /exceeded model token limit|maximum context length|context length exceeded|context window (?:size )?(?:exceeded|too)|prompt is too long|input is too long|reduce the length of|too many (?:input )?tokens|request too large/i.test(errorBody);
|
|
2433
2728
|
}
|
|
2434
2729
|
|
|
2435
|
-
export const __serverTest = { directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
2730
|
+
export const __serverTest = { directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
2436
2731
|
|
|
2437
2732
|
async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
|
|
2438
2733
|
if (!upstreamRes.body) return '';
|
|
@@ -3142,10 +3437,40 @@ function clientHasRealBytes(res, requestInfo) {
|
|
|
3142
3437
|
return res.headersSent && !requestInfo.queueHeartbeatActive;
|
|
3143
3438
|
}
|
|
3144
3439
|
|
|
3145
|
-
async function queueAndRetry(
|
|
3440
|
+
async function queueAndRetry(...args) {
|
|
3441
|
+
// A call WITHOUT an attempt descriptor is exactly 16 args (through `cause`) — never
|
|
3442
|
+
// strip that. A 17th `{ attempt: [account, kind, detail] }` arg makes the queue print
|
|
3443
|
+
// the failed attempt's ONE line once its outcome is known (D5): `→ queued` when the
|
|
3444
|
+
// hold commits, `→ surfaced` when the queue declines without holding. A hold that
|
|
3445
|
+
// later gives up prints a separate `hold gave up` line — the attempt was queued.
|
|
3446
|
+
const desc = args.length > 16 ? args[args.length - 1] : null;
|
|
3447
|
+
if (!desc || !Array.isArray(desc.attempt)) return queueAndRetryInner(...args, null, null);
|
|
3448
|
+
const inner = args.slice(0, -1);
|
|
3449
|
+
const [acct, kind, detail] = desc.attempt;
|
|
3450
|
+
let held = false;
|
|
3451
|
+
let finalized = false;
|
|
3452
|
+
const onHold = () => {
|
|
3453
|
+
if (held) return;
|
|
3454
|
+
held = true;
|
|
3455
|
+
logFailedAttempt(acct, kind, detail, 'queued');
|
|
3456
|
+
};
|
|
3457
|
+
const onGiveUp = () => {
|
|
3458
|
+
if (finalized) return;
|
|
3459
|
+
finalized = true;
|
|
3460
|
+
if (held) console.log(`[Maxpool] hold gave up for the (${kind}) attempt on "${acct}" → surfaced`);
|
|
3461
|
+
else logFailedAttempt(acct, kind, `${detail}; not held`, 'surfaced');
|
|
3462
|
+
};
|
|
3463
|
+
const result = await queueAndRetryInner(...inner, onGiveUp, onHold);
|
|
3464
|
+
if (!result) onGiveUp();
|
|
3465
|
+
return result;
|
|
3466
|
+
}
|
|
3467
|
+
|
|
3468
|
+
async function queueAndRetryInner(
|
|
3146
3469
|
reason, req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
3147
3470
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody = canRetryBufferedBody, cause = 'quota',
|
|
3471
|
+
onGiveUp = null, onHold = null,
|
|
3148
3472
|
) {
|
|
3473
|
+
const giveUp = (msg) => { onGiveUp?.(); return finishQueuedStreamIfNeeded(res, requestInfo, msg); };
|
|
3149
3474
|
if (!queueConfig.enabled || !canQueueBufferedBody || (res.headersSent && !requestInfo.queueHeartbeatActive) || res.destroyed) {
|
|
3150
3475
|
if (queueConfig.enabled && !canQueueBufferedBody && requestInfo.queueBlockedReason) {
|
|
3151
3476
|
console.log(`[Maxpool] Not queueing request: ${requestInfo.queueBlockedReason}`);
|
|
@@ -3231,7 +3556,7 @@ async function queueAndRetry(
|
|
|
3231
3556
|
// Weekly-capped but the reset time is unknown (cold start / probe failure):
|
|
3232
3557
|
// we can't estimate a wait, so don't pretend to — error honestly now.
|
|
3233
3558
|
if (retryPlan.cause === 'weekly_reset_unknown') {
|
|
3234
|
-
return
|
|
3559
|
+
return giveUp(honestMessage);
|
|
3235
3560
|
}
|
|
3236
3561
|
|
|
3237
3562
|
const streamHoldMaxMs = queueConfig.streamHoldMaxMs == null
|
|
@@ -3254,7 +3579,7 @@ async function queueAndRetry(
|
|
|
3254
3579
|
countTokensMaxWaitMs,
|
|
3255
3580
|
});
|
|
3256
3581
|
|
|
3257
|
-
if (queueWindowMs <= 0) return
|
|
3582
|
+
if (queueWindowMs <= 0) return giveUp(honestMessage);
|
|
3258
3583
|
|
|
3259
3584
|
// HOLD-vs-ERROR oracle (from nextRetryForRequest): HOLD only when a TEMPORARY
|
|
3260
3585
|
// cause has a finite real reset within the ceiling. ERROR FAST for permanent /
|
|
@@ -3294,16 +3619,16 @@ async function queueAndRetry(
|
|
|
3294
3619
|
if (ticket === null) {
|
|
3295
3620
|
// Backpressure: too many requests already waiting / too many bytes buffered.
|
|
3296
3621
|
// Reject honestly instead of growing the heap unbounded.
|
|
3297
|
-
return
|
|
3298
|
-
'Maxpool queue is full — too many requests are already waiting for capacity. Try again shortly.');
|
|
3622
|
+
return giveUp('Maxpool queue is full — too many requests are already waiting for capacity. Try again shortly.');
|
|
3299
3623
|
}
|
|
3300
3624
|
const elapsed = Date.now() - requestInfo.queueStartedAt;
|
|
3301
3625
|
const remaining = queueWindowMs - elapsed;
|
|
3302
3626
|
if (remaining <= 0) {
|
|
3303
3627
|
accountManager.removeQueuedRequest?.(requestInfo);
|
|
3304
|
-
return
|
|
3628
|
+
return giveUp(honestMessage);
|
|
3305
3629
|
}
|
|
3306
3630
|
|
|
3631
|
+
onHold?.(); // the hold is committed: the attempt that led here is `→ queued`
|
|
3307
3632
|
ctx.account = '(queued)';
|
|
3308
3633
|
hooks.onRequestRouted?.(reqId, { account: '(queued)' });
|
|
3309
3634
|
// Hold-chain note: when this hold expires still-full (queueHoldChain), the
|
|
@@ -3315,7 +3640,7 @@ async function queueAndRetry(
|
|
|
3315
3640
|
const available = await waitForAvailableRoute(req, res, accountManager, requestInfo, queueConfig, remaining);
|
|
3316
3641
|
if (!available) {
|
|
3317
3642
|
accountManager.removeQueuedRequest?.(requestInfo);
|
|
3318
|
-
return
|
|
3643
|
+
return giveUp(honestMessage);
|
|
3319
3644
|
}
|
|
3320
3645
|
|
|
3321
3646
|
// NOTE: the heartbeat is deliberately NOT cleared here. It must stay alive
|
|
@@ -3334,6 +3659,25 @@ async function queueAndRetry(
|
|
|
3334
3659
|
).then(() => true);
|
|
3335
3660
|
}
|
|
3336
3661
|
|
|
3662
|
+
// Arms the stream-forward grace timer. Shared by the direct and link paths. The
|
|
3663
|
+
// deadline is anchored ONCE per request (??=) so it bounds CUMULATIVE pre-headers time
|
|
3664
|
+
// across re-forwards; returns the timer (or null) for the caller to clear the moment
|
|
3665
|
+
// headers arrive.
|
|
3666
|
+
function armStreamForwardGrace(res, requestInfo, queueConfig, accountManager) {
|
|
3667
|
+
if (!requestInfo.stream || res.headersSent) return null;
|
|
3668
|
+
const streamForwardGraceMs = queueConfig?.streamForwardGraceMs == null
|
|
3669
|
+
? 10000
|
|
3670
|
+
: Math.max(0, Number(queueConfig.streamForwardGraceMs) || 0);
|
|
3671
|
+
requestInfo.streamGraceDeadline ??= Date.now() + streamForwardGraceMs;
|
|
3672
|
+
const graceDelay = Math.max(0, requestInfo.streamGraceDeadline - Date.now());
|
|
3673
|
+
const t = setTimeout(
|
|
3674
|
+
() => commitStreamGraceHeartbeat(res, requestInfo, queueConfig, accountManager),
|
|
3675
|
+
graceDelay,
|
|
3676
|
+
);
|
|
3677
|
+
t.unref?.();
|
|
3678
|
+
return t;
|
|
3679
|
+
}
|
|
3680
|
+
|
|
3337
3681
|
// The stream-forward grace-timer callback (extracted so its logic is unit-testable
|
|
3338
3682
|
// in isolation). Fires when a STREAMING request's upstream is slow to first byte:
|
|
3339
3683
|
// commits the SSE stream + heartbeat so the client never idle-times-out. Client-
|
|
@@ -3383,16 +3727,25 @@ function ensureQueueHeartbeat(res, requestInfo, queueConfig, accountManager) {
|
|
|
3383
3727
|
reapDead();
|
|
3384
3728
|
return;
|
|
3385
3729
|
}
|
|
3386
|
-
requestInfo
|
|
3387
|
-
|
|
3388
|
-
|
|
3730
|
+
ensureQueueHeartbeatInterval(res, requestInfo, heartbeatMs, () => reapDead());
|
|
3731
|
+
}
|
|
3732
|
+
|
|
3733
|
+
/** The keep-alive interval half of the queue heartbeat, for a response whose headers
|
|
3734
|
+
* are ALREADY committed (the link pass-through writes the link's own headers, r4):
|
|
3735
|
+
* pings every heartbeatMs until clearQueueHeartbeat, so the client stays alive across
|
|
3736
|
+
* the link body wait and any failover hop while clientHasRealBytes stays false. */
|
|
3737
|
+
function ensureQueueHeartbeatInterval(res, requestInfo, heartbeatMs, onDead = null) {
|
|
3738
|
+
const timer = setInterval(() => {
|
|
3739
|
+
if (res.destroyed || res.writableEnded) { clearQueueHeartbeat(requestInfo); onDead?.(); return; }
|
|
3389
3740
|
try {
|
|
3390
3741
|
res.write(QUEUE_KEEPALIVE);
|
|
3391
3742
|
} catch {
|
|
3392
|
-
|
|
3743
|
+
clearQueueHeartbeat(requestInfo); onDead?.();
|
|
3393
3744
|
}
|
|
3394
3745
|
}, heartbeatMs);
|
|
3395
|
-
|
|
3746
|
+
timer.unref?.();
|
|
3747
|
+
requestInfo.queueHeartbeatActive = true;
|
|
3748
|
+
requestInfo.queueHeartbeatTimer = timer;
|
|
3396
3749
|
}
|
|
3397
3750
|
|
|
3398
3751
|
function clearQueueHeartbeat(requestInfo) {
|
|
@@ -3732,6 +4085,28 @@ function prepareRuntimeProviders(accountManager, headers) {
|
|
|
3732
4085
|
}
|
|
3733
4086
|
}
|
|
3734
4087
|
|
|
4088
|
+
/**
|
|
4089
|
+
* Call `onFirst` once, the first time anything is flushed to the client (headers or
|
|
4090
|
+
* body). This is the moment a waiting client first sees life: Claude Code reports
|
|
4091
|
+
* "Waiting for API response" when nothing arrives for ~20s, so the request log carries
|
|
4092
|
+
* time-to-first-byte separately from the total duration. Pure observer: the wrapped
|
|
4093
|
+
* methods behave identically, and a throwing callback never affects the response.
|
|
4094
|
+
*/
|
|
4095
|
+
export function markFirstByte(res, onFirst) {
|
|
4096
|
+
let fired = false;
|
|
4097
|
+
const fire = () => { if (fired) return; fired = true; try { onFirst(); } catch { /* observer only */ } };
|
|
4098
|
+
for (const m of ['write', 'end', 'flushHeaders']) {
|
|
4099
|
+
const orig = res[m];
|
|
4100
|
+
if (typeof orig !== 'function') continue;
|
|
4101
|
+
res[m] = function (...args) {
|
|
4102
|
+
// end() with no body and write('') send no bytes of their own, but end() still
|
|
4103
|
+
// flushes the headers, so any call to these three means bytes are on the way.
|
|
4104
|
+
fire();
|
|
4105
|
+
return orig.apply(this, args);
|
|
4106
|
+
};
|
|
4107
|
+
}
|
|
4108
|
+
}
|
|
4109
|
+
|
|
3735
4110
|
function headerValue(headers, name) {
|
|
3736
4111
|
const lname = name.toLowerCase();
|
|
3737
4112
|
let value = headers[lname];
|
|
@@ -3967,13 +4342,15 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
3967
4342
|
let readFailed = false;
|
|
3968
4343
|
|
|
3969
4344
|
|
|
3970
|
-
//
|
|
3971
|
-
//
|
|
3972
|
-
//
|
|
3973
|
-
// the
|
|
3974
|
-
//
|
|
3975
|
-
|
|
3976
|
-
|
|
4345
|
+
// The queue/keep-alive heartbeat keeps running until the FIRST REAL BYTE goes to the
|
|
4346
|
+
// client (review r2 D): until then the client has only pings, so a failure between
|
|
4347
|
+
// upstream headers and the first body chunk can still fail over (clientHasRealBytes
|
|
4348
|
+
// stays false) and the client's idle watchdog stays fed. realWrite stops it right
|
|
4349
|
+
// before the first real write, so no ping can land between live SSE events (Bug A).
|
|
4350
|
+
const realWrite = (chunk) => {
|
|
4351
|
+
if (requestInfo.queueHeartbeatActive) clearQueueHeartbeat(requestInfo);
|
|
4352
|
+
return res.write(chunk);
|
|
4353
|
+
};
|
|
3977
4354
|
|
|
3978
4355
|
// Client left mid-stream: the caller drops its disconnect-abort once headers
|
|
3979
4356
|
// arrive, and the loop below only checks res.destroyed AFTER a read resolves — so
|
|
@@ -3986,6 +4363,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
3986
4363
|
// MODEL ECHO NORMALIZATION state (rationale at the write site).
|
|
3987
4364
|
let modelEchoPending = true;
|
|
3988
4365
|
let modelEchoBuffer = null;
|
|
4366
|
+
let modelEchoHoldStart = null; // r4: the echo cap counts from the FIRST held byte, not stream start
|
|
3989
4367
|
|
|
3990
4368
|
// SAFEGUARD-REFUSAL REROUTE (2026-09-14). Anthropic's server-side classifier can
|
|
3991
4369
|
// terminate a turn with HTTP 200 + `stop_reason:"refusal"` + stop_details.category
|
|
@@ -4004,7 +4382,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4004
4382
|
let refusalRerouting = false;
|
|
4005
4383
|
const refusalWatch = requestInfo?._refusalRerouteEligible === true;
|
|
4006
4384
|
let refusalHoldBuffer = refusalWatch ? [] : null;
|
|
4007
|
-
|
|
4385
|
+
let holdByteAt = null; // r4/r5: when the FIRST byte entered the (refusal or echo) hold — ONE 30s budget from there
|
|
4008
4386
|
|
|
4009
4387
|
try {
|
|
4010
4388
|
while (true) {
|
|
@@ -4041,6 +4419,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4041
4419
|
// rerouted turn leaves the client socket completely untouched — the caller can
|
|
4042
4420
|
// then re-dispatch onto a provider as if the Anthropic attempt never happened.
|
|
4043
4421
|
if (refusalHoldBuffer) {
|
|
4422
|
+
holdByteAt ??= Date.now();
|
|
4044
4423
|
refusalHoldBuffer.push(value);
|
|
4045
4424
|
const held = decoder.decode(concatUint8(refusalHoldBuffer));
|
|
4046
4425
|
const verdict = classifyHeldStreamPrefix(held);
|
|
@@ -4052,13 +4431,15 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4052
4431
|
}
|
|
4053
4432
|
if (verdict.decision === 'hold'
|
|
4054
4433
|
&& totalLen(refusalHoldBuffer) <= REFUSAL_HOLD_MAX_BYTES
|
|
4055
|
-
&& Date.now() -
|
|
4434
|
+
&& holdByteAt != null && Date.now() - holdByteAt <= REFUSAL_HOLD_MAX_MS) { // r5: ONE 30s budget from the first held byte
|
|
4056
4435
|
continue; // nothing decisive yet — keep holding, write nothing
|
|
4057
4436
|
}
|
|
4058
4437
|
// Decisive content, or either cap hit → fail open: release the held bytes as
|
|
4059
|
-
// this chunk and never hold again on this stream.
|
|
4438
|
+
// this chunk and never hold again on this stream. The echo hold (below) inherits
|
|
4439
|
+
// when the FIRST byte was held, so the two stacked holds share ONE 30s budget.
|
|
4060
4440
|
value = concatUint8(refusalHoldBuffer);
|
|
4061
4441
|
refusalHoldBuffer = null;
|
|
4442
|
+
modelEchoHoldStart = holdByteAt;
|
|
4062
4443
|
}
|
|
4063
4444
|
|
|
4064
4445
|
if (!committed) {
|
|
@@ -4079,6 +4460,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4079
4460
|
// concat with decoder-safe helpers.
|
|
4080
4461
|
let out = value;
|
|
4081
4462
|
if (requestInfo?.model && modelEchoPending) {
|
|
4463
|
+
modelEchoHoldStart ??= Date.now();
|
|
4082
4464
|
modelEchoBuffer = modelEchoBuffer ? [...modelEchoBuffer, value] : [value];
|
|
4083
4465
|
const s = decoder.decode(concatUint8(modelEchoBuffer));
|
|
4084
4466
|
if (s.includes('"model"') && s.includes('\n\n')) {
|
|
@@ -4100,6 +4482,16 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4100
4482
|
out = Buffer.from(concatUint8(modelEchoBuffer));
|
|
4101
4483
|
modelEchoBuffer = null;
|
|
4102
4484
|
modelEchoPending = false;
|
|
4485
|
+
} else if (modelEchoHoldStart != null && Date.now() - modelEchoHoldStart > REFUSAL_HOLD_MAX_MS) {
|
|
4486
|
+
// Same fail-open cap as the refusal hold (review r3 R3-4), anchored at the
|
|
4487
|
+
// first held byte (r4): a split model-bearing event arriving >30s after stream
|
|
4488
|
+
// START must still be rewritten, not released verbatim to leak the provider id.
|
|
4489
|
+
// trickles events without ever emitting a model-bearing one must not hold the
|
|
4490
|
+
// stream forever — release the held bytes verbatim (a missed rewrite warning
|
|
4491
|
+
// beats a dead session) so the idle guard and abort paths can act.
|
|
4492
|
+
out = Buffer.from(concatUint8(modelEchoBuffer));
|
|
4493
|
+
modelEchoBuffer = null;
|
|
4494
|
+
modelEchoPending = false;
|
|
4103
4495
|
} else {
|
|
4104
4496
|
continue; // hold until the model-bearing event is complete
|
|
4105
4497
|
}
|
|
@@ -4111,7 +4503,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4111
4503
|
// single decoded text feeds the scrubber (client lane) and the accounting lane.
|
|
4112
4504
|
const chunkText = decoder.decode(out, { stream: true });
|
|
4113
4505
|
const writeText = scrubber ? scrubber.push(chunkText) : null;
|
|
4114
|
-
const ok = scrubber ? (writeText ?
|
|
4506
|
+
const ok = scrubber ? (writeText ? realWrite(writeText) : true) : realWrite(out);
|
|
4115
4507
|
|
|
4116
4508
|
const text = chunkText;
|
|
4117
4509
|
|
|
@@ -4196,7 +4588,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4196
4588
|
if (!readFailed && !res.destroyed) {
|
|
4197
4589
|
try {
|
|
4198
4590
|
if (!committed && !res.headersSent) { res.writeHead(status, responseHeaders); committed = true; }
|
|
4199
|
-
|
|
4591
|
+
realWrite(held);
|
|
4200
4592
|
} catch { /* client already gone */ }
|
|
4201
4593
|
}
|
|
4202
4594
|
}
|
|
@@ -4210,7 +4602,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4210
4602
|
const held = Buffer.from(concatUint8(modelEchoBuffer));
|
|
4211
4603
|
modelEchoBuffer = null;
|
|
4212
4604
|
try {
|
|
4213
|
-
if (!readFailed && !res.destroyed)
|
|
4605
|
+
if (!readFailed && !res.destroyed) realWrite(held);
|
|
4214
4606
|
} catch { /* client already gone */ }
|
|
4215
4607
|
const heldText = held.toString('utf8');
|
|
4216
4608
|
if (streamLog) streamLog.push(heldText);
|
|
@@ -4224,6 +4616,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4224
4616
|
// Cancel upstream reader to stop consuming data nobody needs
|
|
4225
4617
|
reader.cancel().catch(() => {});
|
|
4226
4618
|
if (!readFailed) {
|
|
4619
|
+
clearQueueHeartbeat(requestInfo); // finishing: no pings after this point
|
|
4227
4620
|
if (!committed && !res.headersSent) res.writeHead(status, responseHeaders);
|
|
4228
4621
|
// Truncation sentinel FIRST (before end()) — a write after end() throws and
|
|
4229
4622
|
// would be swallowed, silently producing the truncated stream this guards.
|
|
@@ -4239,11 +4632,11 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4239
4632
|
// client a corrupt frame ahead of the error.
|
|
4240
4633
|
if (scrubber && !truncatedMidEvent && !res.destroyed && !res.writableEnded) {
|
|
4241
4634
|
const tail = scrubber.flush();
|
|
4242
|
-
if (tail) { try {
|
|
4635
|
+
if (tail) { try { realWrite(tail); } catch { /* client gone */ } }
|
|
4243
4636
|
}
|
|
4244
4637
|
if ((committed || res.headersSent) && truncatedMidEvent && !res.destroyed && !res.writableEnded) {
|
|
4245
4638
|
try {
|
|
4246
|
-
|
|
4639
|
+
realWrite(`event: error\ndata: ${JSON.stringify({
|
|
4247
4640
|
type: 'error',
|
|
4248
4641
|
error: { type: 'api_error', message: 'Upstream closed the stream before the response completed. Retry the message.' },
|
|
4249
4642
|
})}\n\n`);
|