maxpool 1.24.79 → 1.24.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +330 -38
- package/src/server.js +424 -93
package/src/server.js
CHANGED
|
@@ -31,6 +31,25 @@ const DEFAULT_RETRY = {
|
|
|
31
31
|
// serves EVERY provider (Anthropic/GLM/Kimi), so the idle gap is generous enough
|
|
32
32
|
// that a legitimately-slow-but-alive stream is never cut (each chunk resets it).
|
|
33
33
|
const UPSTREAM_TTFB_MS = Math.max(5_000, Number(process.env.MAXPOOL_TTFB_MS) || 120_000); // headers must arrive within this
|
|
34
|
+
// D2 (2026-11-09): the first-byte wait scales with the prompt. A 650KB conversation
|
|
35
|
+
// legitimately needs longer than a 5KB one before its first byte (measured p95 300-350s
|
|
36
|
+
// on 400-650KB turns); the flat 120s was the engine of the hop loop — each timeout moved
|
|
37
|
+
// the turn to a cold account that re-read the whole prompt and timed out again.
|
|
38
|
+
// wait = max(floor, ceil(bodyBytes × msPerByte)), clamped at the client's patience.
|
|
39
|
+
// An explicit 0 disables scaling (flat floor, the pre-2026-11-09 behaviour).
|
|
40
|
+
const msPerByteFromEnv = (name, dflt) => {
|
|
41
|
+
const raw = process.env[name];
|
|
42
|
+
if (raw == null || raw === '') return dflt;
|
|
43
|
+
const n = Number(raw);
|
|
44
|
+
return Number.isFinite(n) && n >= 0 ? n : dflt;
|
|
45
|
+
};
|
|
46
|
+
const TTFB_MS_PER_BYTE = msPerByteFromEnv('MAXPOOL_TTFB_MS_PER_BYTE', 0.6);
|
|
47
|
+
// D3: at/over this size a first-byte timeout retries the SAME account once (its stored
|
|
48
|
+
// copy is warm there) before the request is allowed to move.
|
|
49
|
+
const BIG_PROMPT_BYTES = (() => {
|
|
50
|
+
const n = Number(process.env.MAXPOOL_BIG_PROMPT_BYTES);
|
|
51
|
+
return Number.isFinite(n) && n > 0 ? n : 250_000;
|
|
52
|
+
})();
|
|
34
53
|
// A real SSE EVENT, not a `:` comment. Claude Code's stall watchdog is reset only when its
|
|
35
54
|
// SSE iterator YIELDS an event; per the spec a comment line is discarded by the parser and
|
|
36
55
|
// never yields, so a comment keepalive resets nothing. That is why held requests died at
|
|
@@ -477,6 +496,11 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
477
496
|
const queueConfig = { ...DEFAULT_QUEUE, ...(config.queue || {}) };
|
|
478
497
|
const canRetryBufferedBody = body.length <= retryConfig.maxRetryBufferBytes;
|
|
479
498
|
requestInfo = describeRequest(req, body);
|
|
499
|
+
// Request clock: the direct client-gone line ("Client left after Xs" — request
|
|
500
|
+
// age) and the retry-same patience rule read it. (The link path reports time on
|
|
501
|
+
// THIS account, from lease.startedAt.) Was never set, so every direct-path
|
|
502
|
+
// client-gone line read 0.0s.
|
|
503
|
+
requestInfo.startedAt = Date.now();
|
|
480
504
|
const maxQueuedBodyBytes = queueConfig.maxQueuedBodyBytes == null
|
|
481
505
|
? Infinity
|
|
482
506
|
: Math.max(0, Number(queueConfig.maxQueuedBodyBytes) || 0);
|
|
@@ -712,13 +736,22 @@ async function forwardRequest(
|
|
|
712
736
|
}
|
|
713
737
|
|
|
714
738
|
// Select account
|
|
739
|
+
// D3/D5: a retry-same attempt's line waits for THIS acquire — the token is what
|
|
740
|
+
// actually happened (acquireAccount consumes the one-shot hint either way).
|
|
741
|
+
const pendingSame = requestInfo.pendingRetrySame || null;
|
|
742
|
+
requestInfo.pendingRetrySame = null;
|
|
715
743
|
const lease = accountManager.acquireAccount(requestInfo, excludedIndexes);
|
|
716
744
|
const account = lease?.account;
|
|
745
|
+
if (pendingSame && account) {
|
|
746
|
+
if (account.name === pendingSame.account) logFailedAttempt(pendingSame.account, pendingSame.kind, pendingSame.detail, 'retry-same');
|
|
747
|
+
else logFailedAttempt(pendingSame.account, pendingSame.kind, `${pendingSame.detail}; same account refused by selection, served by "${account.name}"`, 'moved');
|
|
748
|
+
}
|
|
717
749
|
if (!account) {
|
|
718
750
|
const queued = await queueAndRetry(
|
|
719
751
|
'no eligible account/provider currently available',
|
|
720
752
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
721
753
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'quota',
|
|
754
|
+
...(pendingSame ? [{ attempt: [pendingSame.account, pendingSame.kind, `${pendingSame.detail}; same account refused by selection, no route free`] }] : []),
|
|
722
755
|
);
|
|
723
756
|
if (queued) return;
|
|
724
757
|
|
|
@@ -801,7 +834,7 @@ async function forwardRequest(
|
|
|
801
834
|
// first events would be pure latency. See streamResponse + classifyHeldStreamPrefix.
|
|
802
835
|
requestInfo._refusalRerouteEligible = account.type !== 'provider'
|
|
803
836
|
&& canRetryBufferedBody
|
|
804
|
-
&& !res
|
|
837
|
+
&& !clientHasRealBytes(res, requestInfo);
|
|
805
838
|
|
|
806
839
|
// Refresh OAuth token if needed
|
|
807
840
|
const tokenReady = await accountManager.ensureTokenFresh(account.index);
|
|
@@ -822,6 +855,7 @@ async function forwardRequest(
|
|
|
822
855
|
retryCount + 1 < maxAttempts &&
|
|
823
856
|
hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
824
857
|
) {
|
|
858
|
+
logFailedAttempt(account.name, 'token_refresh', 'OAuth token refresh unavailable', 'moved');
|
|
825
859
|
return forwardRequest(
|
|
826
860
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
827
861
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -832,6 +866,7 @@ async function forwardRequest(
|
|
|
832
866
|
`OAuth token refresh unavailable for "${account.name}"`,
|
|
833
867
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
834
868
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'quota',
|
|
869
|
+
{ attempt: [account.name, 'token_refresh', 'OAuth token refresh unavailable; no other route'] },
|
|
835
870
|
);
|
|
836
871
|
if (queued) return;
|
|
837
872
|
|
|
@@ -908,6 +943,18 @@ async function forwardRequest(
|
|
|
908
943
|
// listener (calling releaseOnClientGone would release the lease itself).
|
|
909
944
|
res.off('close', onClientClose);
|
|
910
945
|
let linkDone = false;
|
|
946
|
+
// Set by the res 'close' handler BEFORE it aborts the link fetch, so the catch can
|
|
947
|
+
// tell "the client left" apart from the TTFB timer or a real link error. Mirrors
|
|
948
|
+
// the direct path's clientGone handling (catch below, "Client left after").
|
|
949
|
+
let linkClientGone = false;
|
|
950
|
+
// The linked pool's status once headers arrived (null = no answer yet). Local, not
|
|
951
|
+
// ctx.status: an earlier hop's 429 would otherwise leak into this hop's catch.
|
|
952
|
+
let linkStatus = null;
|
|
953
|
+
let linkGraceTimer = null;
|
|
954
|
+
// One 'close' listener per link hop, detached at every exit (review r2 F) — like the
|
|
955
|
+
// direct path's onClientClose, so a failover / queue-resume chain never stacks them.
|
|
956
|
+
let onLinkClose = null;
|
|
957
|
+
const detachLinkClose = () => { if (onLinkClose) res.off('close', onLinkClose); onLinkClose = null; };
|
|
911
958
|
const finishLink = (outcome) => {
|
|
912
959
|
if (linkDone) return; linkDone = true;
|
|
913
960
|
accountManager.releaseAccount(lease, outcome);
|
|
@@ -918,14 +965,27 @@ async function forwardRequest(
|
|
|
918
965
|
// thinking turns regularly sit silent 20-60s before headers, so the link
|
|
919
966
|
// TTFB must match the direct path's UPSTREAM_TTFB_MS (120s default), not a
|
|
920
967
|
// flat 15s (owner report 2026-10-30: every pinned GLM turn 502-aborted).
|
|
968
|
+
// FLAT on purpose — no D2 size scaling and no D3 retry-same here (red-team r1,
|
|
969
|
+
// 2026-11-09): the linked pool commits its own keep-alive headers ~1s in, which
|
|
970
|
+
// clears this timer for every HEALTHY link, so a scaled wait only ever ran against
|
|
971
|
+
// a dead/blackholed tunnel — stretching its detection from 120s to minutes. The
|
|
972
|
+
// big-prompt first-byte timeouts measured were all on the direct path.
|
|
921
973
|
const linkTtfbMs = Math.max(15_000, Number(process.env.MAXPOOL_LINK_TTFB_MS) || UPSTREAM_TTFB_MS);
|
|
922
974
|
const linkCtl = new AbortController();
|
|
923
975
|
const linkTtfb = setTimeout(() => linkCtl.abort(), linkTtfbMs);
|
|
924
|
-
|
|
976
|
+
// P2: the link path had NO pre-headers keep-alive — a streaming client sat on a
|
|
977
|
+
// silent socket for the whole link wait (up to the 120s floor). Arm the same
|
|
978
|
+
// stream-forward grace the direct path uses: after streamForwardGraceMs with no
|
|
979
|
+
// headers, commit the SSE stream + heartbeat. The post-headers branch below
|
|
980
|
+
// (queueHeartbeatActive) already clears it and frames a remote error correctly.
|
|
981
|
+
linkGraceTimer = armStreamForwardGrace(res, requestInfo, queueConfig, accountManager);
|
|
982
|
+
onLinkClose = () => { if (!res.writableEnded) linkClientGone = true; linkCtl.abort(); };
|
|
983
|
+
res.once('close', onLinkClose);
|
|
925
984
|
let upRes;
|
|
926
985
|
try {
|
|
927
986
|
upRes = await fetch(linkUrl, { method: req.method, headers: linkHeaders, body: ['GET', 'HEAD'].includes(req.method) ? undefined : body, duplex: 'half', signal: linkCtl.signal });
|
|
928
|
-
} finally { clearTimeout(linkTtfb); }
|
|
987
|
+
} finally { clearTimeout(linkTtfb); if (linkGraceTimer) clearTimeout(linkGraceTimer); }
|
|
988
|
+
linkStatus = upRes.status;
|
|
929
989
|
ctx.status = upRes.status;
|
|
930
990
|
// The linked pool refused the request's SHAPE (not Claude Code). The cloud
|
|
931
991
|
// subscription accounts are healthy — exclude them (this mirror + every local
|
|
@@ -939,13 +999,18 @@ async function forwardRequest(
|
|
|
939
999
|
for (const a of accountManager.accounts) {
|
|
940
1000
|
if (a.type === 'oauth') excludedIndexes.add(a.index);
|
|
941
1001
|
}
|
|
1002
|
+
// !clientHasRealBytes, not !headersSent: after the P2 keep-alive commits, headers
|
|
1003
|
+
// are out with only pings behind them — a slow-then-refused turn must still fail over.
|
|
942
1004
|
if (
|
|
943
1005
|
retryCount + 1 < maxAttempts
|
|
944
|
-
&& !res
|
|
1006
|
+
&& !clientHasRealBytes(res, requestInfo)
|
|
945
1007
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
946
1008
|
) {
|
|
1009
|
+
logFailedAttempt(account.name, 'request_shape_refused', 'linked pool refused the request shape (not Claude Code)', 'moved');
|
|
1010
|
+
detachLinkClose();
|
|
947
1011
|
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
|
|
948
1012
|
}
|
|
1013
|
+
logFailedAttempt(account.name, 'request_shape_refused', 'linked pool refused the request shape; no route serves it', 'surfaced');
|
|
949
1014
|
sendErrorBody(res, requestInfo, 429, errorText, upRes.headers);
|
|
950
1015
|
return;
|
|
951
1016
|
}
|
|
@@ -960,14 +1025,18 @@ async function forwardRequest(
|
|
|
960
1025
|
finishLink({ status: upRes.status, error: 'remote_pinned_unavailable' });
|
|
961
1026
|
excludedIndexes.add(account.index);
|
|
962
1027
|
if (retryCount + 1 < maxAttempts && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
1028
|
+
logFailedAttempt(account.name, `remote-${upRes.status}`, `linked pool refused before any byte`, 'moved');
|
|
1029
|
+
detachLinkClose();
|
|
963
1030
|
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
|
|
964
1031
|
}
|
|
965
1032
|
// Every route refused: that is capacity, so it waits here. The body was cancelled
|
|
966
1033
|
// above, so falling through handed the client a 503 with no body.
|
|
1034
|
+
detachLinkClose();
|
|
967
1035
|
const queued = await queueAndRetry(
|
|
968
1036
|
`all routes failed after ${upRes.status} from "${account.name}"`,
|
|
969
1037
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
970
1038
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'capacity',
|
|
1039
|
+
{ attempt: [account.name, `remote-${upRes.status}`, 'linked pool refused before any byte; no other route'] },
|
|
971
1040
|
);
|
|
972
1041
|
if (queued) return;
|
|
973
1042
|
ctx.status = upRes.status;
|
|
@@ -975,30 +1044,72 @@ async function forwardRequest(
|
|
|
975
1044
|
return;
|
|
976
1045
|
}
|
|
977
1046
|
// A request released from the queue already holds a committed keep-alive stream.
|
|
978
|
-
|
|
1047
|
+
// An error on it is framed as an SSE error; a 2xx keeps the heartbeat until the
|
|
1048
|
+
// first real chunk (below).
|
|
1049
|
+
if (requestInfo.queueHeartbeatActive && upRes.status >= 400) {
|
|
979
1050
|
clearQueueHeartbeat(requestInfo);
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
}
|
|
1051
|
+
const errorText = await readErrorBody(upRes);
|
|
1052
|
+
logFailedAttempt(account.name, `remote-${upRes.status}`, 'linked pool answered an error on a held stream', 'surfaced');
|
|
1053
|
+
sendErrorBody(res, requestInfo, upRes.status, errorText, upRes.headers);
|
|
1054
|
+
finishLink({ success: upRes.status < 500, status: upRes.status });
|
|
1055
|
+
return;
|
|
986
1056
|
}
|
|
987
1057
|
const outHeaders = {};
|
|
988
1058
|
upRes.headers.forEach((v, k) => { if (!HOP_BY_HOP_HEADERS.has(k) && k !== 'content-length' && k !== 'transfer-encoding') outHeaders[k] = v; });
|
|
1059
|
+
// Headers committed exactly as before (review r4: the batch-7 deferral is REVERTED
|
|
1060
|
+
// — it let a stale grace timer commit keep-alive mid-failover, ERR_HTTP_HEADERS_SENT
|
|
1061
|
+
// on the next hop, and leaked pings into the real stream). Consequence, accepted:
|
|
1062
|
+
// once the link's own headers are written, a later stall can only SURFACE — the
|
|
1063
|
+
// idle guard below bounds the wait instead of undici's ~300s bodyTimeout. Failover
|
|
1064
|
+
// stays possible while only OUR keep-alive pings have been sent (no real chunk yet).
|
|
989
1065
|
if (!res.headersSent) res.writeHead(upRes.status, outHeaders);
|
|
1066
|
+
// r4: headers are committed but NO real byte has been written — keep the keep-alive
|
|
1067
|
+
// heartbeating through the link body wait and any failover hop (grace arm refuses
|
|
1068
|
+
// an already-committed response). clientHasRealBytes stays false until the first
|
|
1069
|
+
// real chunk, so a silent/stalled link can still fail over.
|
|
1070
|
+
if (requestInfo.stream && upRes.status < 300 && !res.destroyed && !res.writableEnded
|
|
1071
|
+
&& !requestInfo.queueHeartbeatActive) {
|
|
1072
|
+
ensureQueueHeartbeatInterval(res, requestInfo, Math.max(1000, Number(queueConfig?.heartbeatMs) || 10_000));
|
|
1073
|
+
}
|
|
990
1074
|
if (upRes.body) {
|
|
991
1075
|
const reader = upRes.body.getReader();
|
|
1076
|
+
// R3-3/r4: bound the link body like the direct path bounds its stream
|
|
1077
|
+
// (MAXPOOL_LINK_IDLE_MS, default STREAM_IDLE_MS). Without this, a link that
|
|
1078
|
+
// answers headers then stalls is masked until undici's ~300s bodyTimeout — longer
|
|
1079
|
+
// than the direct path's bound, and the reaper exempts the link lease. The catch
|
|
1080
|
+
// below fails over while no real byte reached the client; after that it surfaces
|
|
1081
|
+
// honestly and releases the lease request-scoped.
|
|
1082
|
+
const linkIdleMs = Math.max(1_000, Number(process.env.MAXPOOL_LINK_IDLE_MS) || STREAM_IDLE_MS);
|
|
992
1083
|
try {
|
|
993
1084
|
for (;;) {
|
|
994
|
-
const
|
|
1085
|
+
const readP = reader.read();
|
|
1086
|
+
readP.catch(() => {});
|
|
1087
|
+
let idleTimer;
|
|
1088
|
+
const idle = new Promise((_, reject) => {
|
|
1089
|
+
idleTimer = setTimeout(
|
|
1090
|
+
() => reject(Object.assign(new Error('link body idle timeout'), { code: 'UPSTREAM_IDLE', linkBody: true })),
|
|
1091
|
+
linkIdleMs,
|
|
1092
|
+
);
|
|
1093
|
+
idleTimer.unref?.();
|
|
1094
|
+
});
|
|
1095
|
+
let r;
|
|
1096
|
+
try {
|
|
1097
|
+
r = await Promise.race([readP, idle]);
|
|
1098
|
+
} finally {
|
|
1099
|
+
clearTimeout(idleTimer);
|
|
1100
|
+
}
|
|
1101
|
+
const { done, value } = r;
|
|
995
1102
|
if (done) break;
|
|
996
1103
|
if (res.destroyed) { try { await reader.cancel(); } catch { } break; }
|
|
1104
|
+
// First REAL byte stops the keep-alive (review r2 D — UNCONDITIONAL, r4 M1:
|
|
1105
|
+
// a conditional clear let pings land inside the real stream when the grace
|
|
1106
|
+
// timer had already fired).
|
|
1107
|
+
if (requestInfo.queueHeartbeatActive) clearQueueHeartbeat(requestInfo);
|
|
997
1108
|
res.write(Buffer.from(value));
|
|
998
1109
|
}
|
|
999
1110
|
} finally { try { await reader.cancel(); } catch { /* already done */ } }
|
|
1000
1111
|
}
|
|
1001
|
-
res.end();
|
|
1112
|
+
res.end(); // an empty 2xx body: the heartbeat reaps itself on res.writableEnded
|
|
1002
1113
|
finishLink({ success: ctx.status < 500, status: ctx.status });
|
|
1003
1114
|
// SC22 no-double-count: this request was counted at auth; the serving pool
|
|
1004
1115
|
// counted it too. Undo the local one.
|
|
@@ -1008,21 +1119,49 @@ async function forwardRequest(
|
|
|
1008
1119
|
}
|
|
1009
1120
|
hooks.onRequestRouted?.(reqId, { account: account.name });
|
|
1010
1121
|
} catch (err) {
|
|
1011
|
-
|
|
1122
|
+
const heldS = heldSeconds(lease);
|
|
1123
|
+
// D1: the client left before anything reached it. The account did nothing wrong
|
|
1124
|
+
// and the turn is abandoned — release neutral, do not exclude, never re-send.
|
|
1125
|
+
// (A cancel AFTER bytes flowed keeps the path below: a mid-response failure.)
|
|
1126
|
+
if (linkClientGone && !clientHasRealBytes(res, requestInfo)) {
|
|
1127
|
+
finishLink({ success: false, neutral: true, error: 'client_cancelled' });
|
|
1128
|
+
clearQueueHeartbeat(requestInfo);
|
|
1129
|
+
console.log(`[Maxpool] Client left after ${heldS}s on "${account.name}" (link, still waiting) — released, not retried`);
|
|
1130
|
+
logFailedAttempt(account.name, 'client_cancelled', `client left after ${heldS}s, nothing sent`, 'released-cancel');
|
|
1131
|
+
return;
|
|
1132
|
+
}
|
|
1133
|
+
// Everything that reaches this catch is request-scoped (D4): a remote 429/503 is
|
|
1134
|
+
// handled — account-scoped — by its own branch above before anything can throw, so
|
|
1135
|
+
// only a TTFB abort, a mid-stream cancel or an unknown link error lands here. It
|
|
1136
|
+
// must not release every other conversation held on this row.
|
|
1137
|
+
const linkKind = linkClientGone ? 'client_cancelled'
|
|
1138
|
+
: err?.linkBody ? 'link_idle'
|
|
1139
|
+
: err?.name === 'AbortError' ? 'link_ttfb'
|
|
1140
|
+
: 'pool_link_unreachable';
|
|
1141
|
+
finishLink({ status: 502, error: 'pool_link_unreachable', scope: 'request' });
|
|
1012
1142
|
excludedIndexes.add(account.index); // P4b: never re-pick a row that just failed
|
|
1143
|
+
const linkDetail = `${linkStatus == null ? 'no answer' : `HTTP ${linkStatus}`} after ${heldS}s (${err?.message || err})`;
|
|
1013
1144
|
if (!clientHasRealBytes(res, requestInfo)) {
|
|
1014
1145
|
if (retryCount + 1 < maxAttempts && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
1146
|
+
logFailedAttempt(account.name, linkKind, linkDetail, 'moved');
|
|
1147
|
+
detachLinkClose();
|
|
1015
1148
|
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
|
|
1016
1149
|
}
|
|
1150
|
+
detachLinkClose();
|
|
1017
1151
|
const queued = await queueAndRetry(
|
|
1018
1152
|
`all routes failed after a link error on "${account.name}"`,
|
|
1019
1153
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1020
1154
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'network',
|
|
1155
|
+
{ attempt: [account.name, linkKind, linkDetail] },
|
|
1021
1156
|
);
|
|
1022
1157
|
if (queued) return;
|
|
1158
|
+
} else {
|
|
1159
|
+
logFailedAttempt(account.name, linkKind, `${linkDetail}, output already sent`, 'surfaced');
|
|
1023
1160
|
}
|
|
1024
1161
|
ctx.status = 502;
|
|
1025
1162
|
sendErrorResponse(res, requestInfo, 502, { type: 'error', error: { type: 'api_error', message: `Team pool "${account.name}" unreachable: ${err?.message || err}` } });
|
|
1163
|
+
} finally {
|
|
1164
|
+
detachLinkClose();
|
|
1026
1165
|
}
|
|
1027
1166
|
return;
|
|
1028
1167
|
}
|
|
@@ -1081,9 +1220,12 @@ async function forwardRequest(
|
|
|
1081
1220
|
// arrive — the streaming body is then governed by the per-chunk idle guard, never
|
|
1082
1221
|
// this fixed clock, so a long healthy stream is never cut.
|
|
1083
1222
|
const ttfbController = new AbortController();
|
|
1223
|
+
const ttfbCeilingMs = clientWaitCeilingMs(requestInfo, queueConfig);
|
|
1224
|
+
const ttfbMs = scaledTtfbMs(UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, requestInfo.bodyBytes, ttfbCeilingMs);
|
|
1225
|
+
noteScaledTtfb(requestInfo, account.name, 'direct', ttfbMs, UPSTREAM_TTFB_MS, ttfbCeilingMs);
|
|
1084
1226
|
const ttfbTimer = setTimeout(
|
|
1085
1227
|
() => ttfbController.abort(Object.assign(new Error('upstream TTFB timeout'), { code: 'UPSTREAM_TTFB' })),
|
|
1086
|
-
|
|
1228
|
+
ttfbMs,
|
|
1087
1229
|
);
|
|
1088
1230
|
ttfbTimer.unref?.();
|
|
1089
1231
|
// Proactive streaming keep-alive. UPSTREAM_TTFB_MS (120s) is FAR above the
|
|
@@ -1095,19 +1237,7 @@ async function forwardRequest(
|
|
|
1095
1237
|
// grace, commit the SSE stream + heartbeat so the client stays alive. The
|
|
1096
1238
|
// deadline is anchored ONCE per request (??=) so it also bounds CUMULATIVE
|
|
1097
1239
|
// failover time across re-forwards, not each attempt independently.
|
|
1098
|
-
const
|
|
1099
|
-
? 10000
|
|
1100
|
-
: Math.max(0, Number(queueConfig.streamForwardGraceMs) || 0);
|
|
1101
|
-
let streamGraceTimer = null;
|
|
1102
|
-
if (requestInfo.stream && !res.headersSent) {
|
|
1103
|
-
requestInfo.streamGraceDeadline ??= Date.now() + streamForwardGraceMs;
|
|
1104
|
-
const graceDelay = Math.max(0, requestInfo.streamGraceDeadline - Date.now());
|
|
1105
|
-
streamGraceTimer = setTimeout(
|
|
1106
|
-
() => commitStreamGraceHeartbeat(res, requestInfo, queueConfig, accountManager),
|
|
1107
|
-
graceDelay,
|
|
1108
|
-
);
|
|
1109
|
-
streamGraceTimer.unref?.();
|
|
1110
|
-
}
|
|
1240
|
+
const streamGraceTimer = armStreamForwardGrace(res, requestInfo, queueConfig, accountManager);
|
|
1111
1241
|
try {
|
|
1112
1242
|
let upstreamRes;
|
|
1113
1243
|
try {
|
|
@@ -1183,16 +1313,18 @@ async function forwardRequest(
|
|
|
1183
1313
|
if (
|
|
1184
1314
|
canRetryBufferedBody
|
|
1185
1315
|
&& retryCount + 1 < maxAttempts
|
|
1186
|
-
&& !res
|
|
1316
|
+
&& !clientHasRealBytes(res, requestInfo)
|
|
1187
1317
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1188
1318
|
) {
|
|
1189
1319
|
if (logDir) writeRequestLog(logDir, reqId, logSections);
|
|
1320
|
+
logFailedAttempt(account.name, 'request_shape_refused', 'subscription refused the request shape (not Claude Code)', 'moved');
|
|
1190
1321
|
return forwardRequest(
|
|
1191
1322
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1192
1323
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
1193
1324
|
);
|
|
1194
1325
|
}
|
|
1195
1326
|
if (logDir) writeRequestLog(logDir, reqId, logSections);
|
|
1327
|
+
logFailedAttempt(account.name, 'request_shape_refused', 'subscription refused the request shape; no route serves it', 'surfaced');
|
|
1196
1328
|
ctx.status = 429;
|
|
1197
1329
|
sendErrorResponse(res, requestInfo, 429, {
|
|
1198
1330
|
type: 'error',
|
|
@@ -1222,9 +1354,10 @@ async function forwardRequest(
|
|
|
1222
1354
|
if (
|
|
1223
1355
|
canRetryBufferedBody
|
|
1224
1356
|
&& retryCount + 1 < maxAttempts
|
|
1225
|
-
&& !res
|
|
1357
|
+
&& !clientHasRealBytes(res, requestInfo)
|
|
1226
1358
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1227
1359
|
) {
|
|
1360
|
+
logFailedAttempt(account.name, 'upstream_429', `server-side throttle, retry-after ${retryAfter}s`, 'moved');
|
|
1228
1361
|
return forwardRequest(
|
|
1229
1362
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1230
1363
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -1243,6 +1376,7 @@ async function forwardRequest(
|
|
|
1243
1376
|
'Anthropic upstream is temporarily limiting requests',
|
|
1244
1377
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1245
1378
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'upstream_throttle',
|
|
1379
|
+
{ attempt: [account.name, 'upstream_429', 'request-wide server-side throttle'] },
|
|
1246
1380
|
);
|
|
1247
1381
|
if (queued) return;
|
|
1248
1382
|
|
|
@@ -1261,6 +1395,7 @@ async function forwardRequest(
|
|
|
1261
1395
|
`all routes failed after server-side 429 from "${account.name}"`,
|
|
1262
1396
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1263
1397
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'capacity',
|
|
1398
|
+
{ attempt: [account.name, 'upstream_429', 'server-side throttle; no other route'] },
|
|
1264
1399
|
);
|
|
1265
1400
|
if (queued) return;
|
|
1266
1401
|
|
|
@@ -1288,6 +1423,7 @@ async function forwardRequest(
|
|
|
1288
1423
|
'Anthropic upstream is temporarily limiting requests',
|
|
1289
1424
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1290
1425
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'upstream_throttle',
|
|
1426
|
+
{ attempt: [account.name, 'upstream_429', 'matching ambiguous 429s — shared throttle opened'] },
|
|
1291
1427
|
);
|
|
1292
1428
|
if (queued) return;
|
|
1293
1429
|
|
|
@@ -1310,13 +1446,15 @@ async function forwardRequest(
|
|
|
1310
1446
|
}
|
|
1311
1447
|
console.log(`[Maxpool] 429 on "${account.name}" — failing over before first byte`);
|
|
1312
1448
|
excludedIndexes.add(account.index);
|
|
1449
|
+
const rlKind = rateLimit.modelScope ? 'model_rate_limited' : 'rate_limited';
|
|
1313
1450
|
|
|
1314
1451
|
if (
|
|
1315
1452
|
canRetryBufferedBody &&
|
|
1316
1453
|
retryCount + 1 < maxAttempts &&
|
|
1317
|
-
!res
|
|
1454
|
+
!clientHasRealBytes(res, requestInfo) &&
|
|
1318
1455
|
hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1319
1456
|
) {
|
|
1457
|
+
logFailedAttempt(account.name, rlKind, `HTTP 429, benched ${retryAfter}s`, 'moved');
|
|
1320
1458
|
return forwardRequest(
|
|
1321
1459
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1322
1460
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -1327,6 +1465,7 @@ async function forwardRequest(
|
|
|
1327
1465
|
`all routes failed after 429 from "${account.name}"`,
|
|
1328
1466
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1329
1467
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canRetryBufferedBody, 'quota',
|
|
1468
|
+
{ attempt: [account.name, rlKind, `HTTP 429, benched ${retryAfter}s; no other route`] },
|
|
1330
1469
|
);
|
|
1331
1470
|
if (queued) return;
|
|
1332
1471
|
|
|
@@ -1369,19 +1508,22 @@ async function forwardRequest(
|
|
|
1369
1508
|
if (errorBody) logSections.push(`=== ERROR BODY ===\n${errorBody}`);
|
|
1370
1509
|
}
|
|
1371
1510
|
console.log(`[Maxpool] ${upstreamRes.status} on provider "${account.name}" — ${forbidden ? `cooled down ${PROVIDER_FORBIDDEN_COOLDOWN_SEC}s (recoverable)` : 'disabled'} and failing over before first byte`);
|
|
1511
|
+
const authKind = forbidden ? 'provider_403' : 'auth_failed';
|
|
1372
1512
|
|
|
1373
1513
|
if (
|
|
1374
1514
|
canRetryBufferedBody &&
|
|
1375
1515
|
retryCount + 1 < maxAttempts &&
|
|
1376
|
-
!res
|
|
1516
|
+
!clientHasRealBytes(res, requestInfo) &&
|
|
1377
1517
|
hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1378
1518
|
) {
|
|
1519
|
+
logFailedAttempt(account.name, authKind, `HTTP ${upstreamRes.status}`, 'moved');
|
|
1379
1520
|
return forwardRequest(
|
|
1380
1521
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1381
1522
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
1382
1523
|
);
|
|
1383
1524
|
}
|
|
1384
1525
|
|
|
1526
|
+
logFailedAttempt(account.name, authKind, `HTTP ${upstreamRes.status}; no other route`, 'surfaced');
|
|
1385
1527
|
ctx.status = 502;
|
|
1386
1528
|
if (logDir) writeRequestLog(logDir, reqId, logSections);
|
|
1387
1529
|
sendErrorResponse(res, requestInfo, 502, {
|
|
@@ -1416,9 +1558,10 @@ async function forwardRequest(
|
|
|
1416
1558
|
if (
|
|
1417
1559
|
canRetryBufferedBody
|
|
1418
1560
|
&& retryCount + 1 < maxAttempts
|
|
1419
|
-
&& !res
|
|
1561
|
+
&& !clientHasRealBytes(res, requestInfo)
|
|
1420
1562
|
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)
|
|
1421
1563
|
) {
|
|
1564
|
+
logFailedAttempt(account.name, 'upstream_529', `overloaded, retry-after ${retryAfter}s`, 'moved');
|
|
1422
1565
|
return forwardRequest(
|
|
1423
1566
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1424
1567
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -1434,6 +1577,7 @@ async function forwardRequest(
|
|
|
1434
1577
|
'Anthropic upstream is overloaded',
|
|
1435
1578
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1436
1579
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'upstream_throttle',
|
|
1580
|
+
{ attempt: [account.name, 'upstream_529', 'request-wide overload — shared throttle opened'] },
|
|
1437
1581
|
);
|
|
1438
1582
|
if (queued) return;
|
|
1439
1583
|
|
|
@@ -1452,6 +1596,7 @@ async function forwardRequest(
|
|
|
1452
1596
|
`all routes failed after HTTP 529 from "${account.name}"`,
|
|
1453
1597
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1454
1598
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, 'capacity',
|
|
1599
|
+
{ attempt: [account.name, 'upstream_529', 'overloaded; no other route'] },
|
|
1455
1600
|
);
|
|
1456
1601
|
if (queued) return;
|
|
1457
1602
|
|
|
@@ -1471,6 +1616,7 @@ async function forwardRequest(
|
|
|
1471
1616
|
}
|
|
1472
1617
|
|
|
1473
1618
|
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
|
|
1619
|
+
logFailedAttempt(account.name, `upstream_${upstreamRes.status}`, 'retriable upstream status, cooling down', 'moved');
|
|
1474
1620
|
return forwardRequest(
|
|
1475
1621
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1476
1622
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -1481,6 +1627,7 @@ async function forwardRequest(
|
|
|
1481
1627
|
`all routes failed after ${upstreamRes.status} from "${account.name}"`,
|
|
1482
1628
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
1483
1629
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canRetryBufferedBody, 'capacity',
|
|
1630
|
+
{ attempt: [account.name, `upstream_${upstreamRes.status}`, 'retriable upstream status; no other route'] },
|
|
1484
1631
|
);
|
|
1485
1632
|
if (queued) return;
|
|
1486
1633
|
|
|
@@ -1631,6 +1778,7 @@ async function forwardRequest(
|
|
|
1631
1778
|
accountManager.revertSessionBinding(requestInfo.sessionKey, lease.migratedFromName);
|
|
1632
1779
|
excludedIndexes.add(account.index);
|
|
1633
1780
|
console.log(`[Maxpool] Thinking-signature fail-safe: reverting session to issuer "${lease.migratedFromName}" and retrying`);
|
|
1781
|
+
logFailedAttempt(account.name, 'invalid_thinking_signature', `migrated replay rejected; back to issuer "${lease.migratedFromName}"`, 'moved');
|
|
1634
1782
|
return forwardRequest(
|
|
1635
1783
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1636
1784
|
retryConfig, queueConfig, { ...requestInfo, pinnedAccountName: lease.migratedFromName },
|
|
@@ -1783,42 +1931,27 @@ async function forwardRequest(
|
|
|
1783
1931
|
// session) and retry PROVIDER-only, rather than surfacing the 400. Only worth
|
|
1784
1932
|
// it when a provider can actually serve it (profile=all with a GLM/Kimi token).
|
|
1785
1933
|
//
|
|
1786
|
-
//
|
|
1787
|
-
//
|
|
1788
|
-
//
|
|
1789
|
-
//
|
|
1790
|
-
//
|
|
1791
|
-
//
|
|
1792
|
-
//
|
|
1793
|
-
// converted body replays 200 (verified live, count_tokens, 2026-11-06).
|
|
1934
|
+
// NO post-rejection conversion retry (review r2, 2026-11-09): detectTranscriptOrigin
|
|
1935
|
+
// flags exactly the ids the converter can convert, so the up-front strip has
|
|
1936
|
+
// already converted every curable block before the first send (pinned by
|
|
1937
|
+
// glm-mcp-two-turn F2). A rejection that reaches here has nothing convertible —
|
|
1938
|
+
// re-sending it was a guaranteed identical 400, and its latch stripped valid Claude
|
|
1939
|
+
// thinking from every later turn. Move to GLM/Kimi when one is available; else
|
|
1940
|
+
// the original 400 surfaces (below).
|
|
1794
1941
|
const providerAvailable = accountManager.accounts?.some(a => a.type === 'provider' && a.enabled !== false);
|
|
1795
|
-
// Exact-shape gate (red-team M2)
|
|
1796
|
-
// fires on unrelated unknown-tag 400s (its
|
|
1797
|
-
// server_tool_use)
|
|
1798
|
-
// deterministic server-tool-id wording, never the fuzzy predicate.
|
|
1799
|
-
const
|
|
1942
|
+
// Exact-shape gate (red-team M2) for the surfaced attempt line: the loose
|
|
1943
|
+
// isAnthropicIncompatBody matcher also fires on unrelated unknown-tag 400s (its
|
|
1944
|
+
// expected-tags list mentions server_tool_use), so key on the two MCP wordings or
|
|
1945
|
+
// the deterministic server-tool-id wording, never the fuzzy predicate.
|
|
1946
|
+
const incompatTagRejection = /server_tool_use\.id: String should match pattern/i.test(errorBody)
|
|
1800
1947
|
|| /mcp_tool_(use|result)/.test(errorBody)
|
|
1801
1948
|
&& /(does not match any of the expected tags|without a corresponding)/i.test(errorBody);
|
|
1802
|
-
if (anthropicIncompat && incompatRepairable && !requestInfo.incompatRepairTried && canRepairBody) {
|
|
1803
|
-
const repaired = stripForeignThinkingBlocks(body);
|
|
1804
|
-
if (repaired.body) {
|
|
1805
|
-
// Latch the pre-strip sticky so the NEXT turn repairs up front instead of
|
|
1806
|
-
// re-paying this rejected round-trip every turn (the client resends the whole
|
|
1807
|
-
// poisoned history each turn — same cache-churn the audit is about).
|
|
1808
|
-
accountManager.markSessionThinkingContaminated?.(requestInfo.sessionKey);
|
|
1809
|
-
console.log(`[Maxpool] Recovering session on Claude: incompatibility repair removed ${repaired.removed} thinking block(s), converted ${repaired.converted} provider tool block(s) — retrying on Claude before pinning`);
|
|
1810
|
-
return forwardRequest(
|
|
1811
|
-
req, res, repaired.body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1812
|
-
retryConfig, queueConfig, { ...requestInfo, incompatRepairTried: true, anthropicIncompatible: false, repairCount: repairCount + 1 },
|
|
1813
|
-
repaired.body.length <= retryConfig.maxRetryBufferBytes, canQueueBufferedBody, excludedIndexes,
|
|
1814
|
-
);
|
|
1815
|
-
}
|
|
1816
|
-
}
|
|
1817
1949
|
if (anthropicIncompat && requestInfo.sessionKey && providerAvailable
|
|
1818
1950
|
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
|
|
1819
1951
|
accountManager.markSessionIncompatible?.(requestInfo.sessionKey, requestInfo.homeProvider);
|
|
1820
1952
|
excludedIndexes.add(account.index);
|
|
1821
1953
|
console.log(`[Maxpool] Anthropic rejected this transcript (server_tool_use/thinking); pinning session to GLM/Kimi and retrying`);
|
|
1954
|
+
logFailedAttempt(account.name, 'transcript_rejected', 'Anthropic rejected the transcript; pinning to GLM/Kimi', 'moved');
|
|
1822
1955
|
return forwardRequest(
|
|
1823
1956
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1824
1957
|
retryConfig, queueConfig, { ...requestInfo, anthropicIncompatible: true },
|
|
@@ -1846,6 +1979,7 @@ async function forwardRequest(
|
|
|
1846
1979
|
if (a.type === 'provider') excludedIndexes.add(a.index);
|
|
1847
1980
|
}
|
|
1848
1981
|
console.log(`[Maxpool] Provider "${account.name}" context too small for this request; pinning session to Claude and retrying`);
|
|
1982
|
+
logFailedAttempt(account.name, 'provider_context_too_small', 'request exceeds the provider context; pinning to Claude', 'moved');
|
|
1849
1983
|
return forwardRequest(
|
|
1850
1984
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1851
1985
|
retryConfig, queueConfig, { ...requestInfo, largeContext: true },
|
|
@@ -1863,6 +1997,7 @@ async function forwardRequest(
|
|
|
1863
1997
|
if (a.type === 'provider') excludedIndexes.add(a.index);
|
|
1864
1998
|
}
|
|
1865
1999
|
console.log(`[Maxpool] Provider "${account.name}" rejected this request's shape (${String(errorBody).slice(0, 120)}); retrying on Claude`);
|
|
2000
|
+
logFailedAttempt(account.name, 'provider_shape_rejected', 'provider will not take this body; retrying on Claude', 'moved');
|
|
1866
2001
|
return forwardRequest(
|
|
1867
2002
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
1868
2003
|
retryConfig, queueConfig, requestInfo,
|
|
@@ -1918,6 +2053,9 @@ async function forwardRequest(
|
|
|
1918
2053
|
return;
|
|
1919
2054
|
}
|
|
1920
2055
|
|
|
2056
|
+
if (anthropicIncompat && incompatTagRejection) {
|
|
2057
|
+
logFailedAttempt(account.name, 'transcript_rejected', 'Anthropic rejected the transcript; nothing convertible to repair', 'surfaced');
|
|
2058
|
+
}
|
|
1921
2059
|
ctx.status = upstreamRes.status;
|
|
1922
2060
|
sendErrorBody(res, requestInfo, upstreamRes.status, errorBody, upstreamRes.headers);
|
|
1923
2061
|
return;
|
|
@@ -2082,8 +2220,11 @@ async function forwardRequest(
|
|
|
2082
2220
|
// was still waiting" — and the elapsed time is what exposes a client watchdog firing
|
|
2083
2221
|
// at its floor while maxpool was still happily holding the request.
|
|
2084
2222
|
const heldMs = Date.now() - (requestInfo.startedAt || Date.now());
|
|
2223
|
+
// clientHasRealBytes, not headersSent: the keep-alive commits headers with only pings.
|
|
2224
|
+
const gotOutput = clientHasRealBytes(res, requestInfo);
|
|
2085
2225
|
console.log(`[Maxpool] Client left after ${(heldMs / 1000).toFixed(1)}s on "${account.name}" `
|
|
2086
|
-
+ `(${
|
|
2226
|
+
+ `(${gotOutput ? 'mid-response — output already sent' : 'still waiting, nothing sent'})`);
|
|
2227
|
+
logFailedAttempt(account.name, 'client_cancelled', `client left after ${(heldMs / 1000).toFixed(1)}s (${gotOutput ? 'mid-response' : 'still waiting'})`, 'released-cancel');
|
|
2087
2228
|
releaseOnClientGone();
|
|
2088
2229
|
return;
|
|
2089
2230
|
}
|
|
@@ -2103,7 +2244,9 @@ async function forwardRequest(
|
|
|
2103
2244
|
// provider. If no provider is available,
|
|
2104
2245
|
// fall through to the normal error path — the client sees the same message it
|
|
2105
2246
|
// would have seen before this feature existed.
|
|
2106
|
-
|
|
2247
|
+
// !clientHasRealBytes, not !headersSent: a slow turn whose keep-alive committed
|
|
2248
|
+
// (pings only) is just as reroutable as a fast one (review r2 B).
|
|
2249
|
+
if (err.code === 'REFUSAL_RETRY' && canRetryBufferedBody && !clientHasRealBytes(res, requestInfo)) {
|
|
2107
2250
|
const category = err.refusalCategory || 'unknown';
|
|
2108
2251
|
const anthropicIndexes = accountManager.accounts
|
|
2109
2252
|
.filter(a => a.type !== 'provider')
|
|
@@ -2115,6 +2258,7 @@ async function forwardRequest(
|
|
|
2115
2258
|
// 2026-09-14: removing a latch changed nothing on any constructible path.
|
|
2116
2259
|
if (providerAvailable && retryCount + 1 <= maxAttempts) {
|
|
2117
2260
|
console.log(`[Maxpool] Anthropic safeguard refusal (${category}) on "${account.name}" — rerouting turn to a provider account`);
|
|
2261
|
+
logFailedAttempt(account.name, 'safeguard_refusal', `${category}; rerouting to a provider`, 'moved');
|
|
2118
2262
|
accountManager.releaseAccount(lease, { success: true, status: 200, refusal: category });
|
|
2119
2263
|
return forwardRequest(
|
|
2120
2264
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
@@ -2146,10 +2290,41 @@ async function forwardRequest(
|
|
|
2146
2290
|
// Network-class (ECONNRESET / fetch failed / timeout / terminated): short fixed
|
|
2147
2291
|
// cooldown, no exponential escalation — the fleet auto-recovers seconds after
|
|
2148
2292
|
// connectivity returns instead of being benched for up to 15 min.
|
|
2149
|
-
|
|
2293
|
+
const transientKind = err.code || (err.message || 'network_error').slice(0, 40);
|
|
2294
|
+
// D3 retry-same: a first-byte timeout on a BIG prompt goes back to the SAME
|
|
2295
|
+
// account once — no cooldown (it would make the account unavailable to its own
|
|
2296
|
+
// retry), no exclusion, through the normal acquire path via a one-hop hint (P1).
|
|
2297
|
+
// The second timeout falls through to the move path below.
|
|
2298
|
+
if (err.code === 'UPSTREAM_TTFB' && canRetryBufferedBody && retrySameEligible(account, requestInfo, res, queueConfig)) {
|
|
2299
|
+
requestInfo.sameAccountRetried = true;
|
|
2300
|
+
requestInfo.retrySameAccountName = account.name;
|
|
2301
|
+
accountManager.releaseAccount(lease, { error: 'upstream_ttfb', scope: 'request' });
|
|
2302
|
+
requestInfo.pendingRetrySame = { account: account.name, kind: transientKind,
|
|
2303
|
+
detail: `no first byte after ${heldSeconds(lease)}s on a ${Math.round((requestInfo.bodyBytes || 0) / 1000)}KB prompt` };
|
|
2304
|
+
// retryCount NOT bumped: maxAttempts is sized by ACCOUNT COUNT (failover budget);
|
|
2305
|
+
// a same-account retry is not a failover and must not eat the move it may still
|
|
2306
|
+
// need (2 accounts: retry-same + move = 2 hops). sameAccountRetried bounds it.
|
|
2307
|
+
return forwardRequest(
|
|
2308
|
+
req, res, body, accountManager, upstream, retryCount, hooks, reqId, ctx, logDir,
|
|
2309
|
+
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
2310
|
+
);
|
|
2311
|
+
}
|
|
2312
|
+
// A first-byte / idle / stalled-body timeout is REQUEST-scoped (D4, review r2 A): it soft-cools the
|
|
2313
|
+
// account for NEW placements only — conversations held on it keep it — and only 3
|
|
2314
|
+
// in a row escalate to an account fault. Every other network class (reset, DNS,
|
|
2315
|
+
// fetch failed) is fleet-wide connectivity: the short fixed account cooldown. The
|
|
2316
|
+
// release carries no error/status, so it moves no counter itself.
|
|
2317
|
+
const requestScoped = isRequestScopedTimeout(err);
|
|
2318
|
+
accountManager.markTransientFailure(account.index, err.code || err.message || 'network_error',
|
|
2319
|
+
{ network: true, scope: requestScoped ? 'request' : 'account' });
|
|
2150
2320
|
accountManager.releaseAccount(lease);
|
|
2151
2321
|
excludedIndexes.add(account.index);
|
|
2152
|
-
|
|
2322
|
+
// hasEligibleRoute like every sibling gate: with no route left the network-cause
|
|
2323
|
+
// hold below is the honest path (a move would land in the no-account queue and
|
|
2324
|
+
// answer with a misleading quota message).
|
|
2325
|
+
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2326
|
+
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
2327
|
+
logFailedAttempt(account.name, transientKind, `transient after ${heldSeconds(lease)}s`, 'moved');
|
|
2153
2328
|
return forwardRequest(
|
|
2154
2329
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
2155
2330
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -2159,6 +2334,7 @@ async function forwardRequest(
|
|
|
2159
2334
|
`all routes failed after network error from "${account.name}"`,
|
|
2160
2335
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
2161
2336
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canRetryBufferedBody, 'network',
|
|
2337
|
+
{ attempt: [account.name, transientKind, 'transient; no other route'] },
|
|
2162
2338
|
);
|
|
2163
2339
|
if (queued) return;
|
|
2164
2340
|
ctx.status = 503;
|
|
@@ -2188,6 +2364,7 @@ async function forwardRequest(
|
|
|
2188
2364
|
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
|
|
2189
2365
|
account.status = 'error';
|
|
2190
2366
|
excludedIndexes.add(account.index);
|
|
2367
|
+
logFailedAttempt(account.name, (err.code || err.message || 'error').toString().slice(0, 40), err.message, 'moved');
|
|
2191
2368
|
return forwardRequest(
|
|
2192
2369
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
2193
2370
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
|
|
@@ -2197,6 +2374,7 @@ async function forwardRequest(
|
|
|
2197
2374
|
`all routes failed after proxy error from "${account.name}"`,
|
|
2198
2375
|
req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
2199
2376
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canRetryBufferedBody, 'proxy',
|
|
2377
|
+
{ attempt: [account.name, (err.code || err.message || 'error').toString().slice(0, 40), `no other route after ${err.message}`] },
|
|
2200
2378
|
);
|
|
2201
2379
|
if (queued) return;
|
|
2202
2380
|
ctx.status = 502;
|
|
@@ -2208,6 +2386,83 @@ async function forwardRequest(
|
|
|
2208
2386
|
}
|
|
2209
2387
|
}
|
|
2210
2388
|
|
|
2389
|
+
/**
|
|
2390
|
+
* The longest this request may sit waiting for a first byte before the CLIENT gives up.
|
|
2391
|
+
* Streaming: the client's own tolerance (x-maxpool-client-stream-idle-ms × 0.8, else the
|
|
2392
|
+
* conservative streamClientToleranceMs = 240s for a bare `claude`), never past the CLI's
|
|
2393
|
+
* absolute 30-min stream ceiling. The pre-headers keep-alive (streamForwardGraceMs) is
|
|
2394
|
+
* what keeps the client's idle watchdog fed during that wait, on both paths.
|
|
2395
|
+
* Non-streaming: nothing keeps the client alive pre-headers, so no scaling at all — the
|
|
2396
|
+
* ceiling IS the flat floor (null = "use the floor").
|
|
2397
|
+
*/
|
|
2398
|
+
function clientWaitCeilingMs(requestInfo = {}, queueConfig = {}) {
|
|
2399
|
+
if (!requestInfo.stream) return null;
|
|
2400
|
+
const tolerance = Number.isFinite(requestInfo.clientToleranceMs)
|
|
2401
|
+
? requestInfo.clientToleranceMs
|
|
2402
|
+
: Math.max(0, Number(queueConfig?.streamClientToleranceMs ?? DEFAULT_QUEUE.streamClientToleranceMs) || 0);
|
|
2403
|
+
return Math.min(tolerance, CLI_STREAM_CEILING_MS);
|
|
2404
|
+
}
|
|
2405
|
+
|
|
2406
|
+
/** D2: first-byte wait for one attempt. Never below `floorMs`; scaled part clamped at `ceilingMs`. */
|
|
2407
|
+
function scaledTtfbMs(floorMs, msPerByte, bodyBytes, ceilingMs) {
|
|
2408
|
+
if (ceilingMs == null) return floorMs; // 0 ms/byte needs no branch: max(floor, 0) = floor
|
|
2409
|
+
const scaled = Math.ceil((Number(bodyBytes) || 0) * msPerByte);
|
|
2410
|
+
return Math.max(floorMs, Math.min(scaled, ceilingMs));
|
|
2411
|
+
}
|
|
2412
|
+
|
|
2413
|
+
/** Log the chosen first-byte wait once per request, only when it exceeds the floor. */
|
|
2414
|
+
function noteScaledTtfb(requestInfo, accountName, path, waitMs, floorMs, ceilingMs) {
|
|
2415
|
+
if (waitMs <= floorMs || requestInfo._ttfbWaitLogged) return;
|
|
2416
|
+
requestInfo._ttfbWaitLogged = true;
|
|
2417
|
+
console.log(`[Maxpool] first-byte wait ${Math.round(waitMs / 1000)}s for ${Math.round((requestInfo.bodyBytes || 0) / 1000)}KB prompt on "${accountName}" (${path}; floor ${Math.round(floorMs / 1000)}s${waitMs >= ceilingMs ? ', clamped at client patience' : ''})`);
|
|
2418
|
+
}
|
|
2419
|
+
|
|
2420
|
+
/**
|
|
2421
|
+
* D3: may this first-byte timeout retry the SAME account instead of moving?
|
|
2422
|
+
* Only for a big prompt, once per request, on an account that is not throttled /
|
|
2423
|
+
* auth-dead / cooling, while nothing real has reached the client, and only when the
|
|
2424
|
+
* second wait can still end inside the client's patience. Direct path only.
|
|
2425
|
+
*/
|
|
2426
|
+
function retrySameEligible(account, requestInfo, res, queueConfig = {}) {
|
|
2427
|
+
if (requestInfo.sameAccountRetried) return false;
|
|
2428
|
+
if ((requestInfo.bodyBytes || 0) < BIG_PROMPT_BYTES) return false;
|
|
2429
|
+
if (!account || account.status === 'throttled' || account.status === 'error'
|
|
2430
|
+
|| account.subscriptionGone || account.refreshDead) return false;
|
|
2431
|
+
if (account.cooldownUntil && Date.now() < account.cooldownUntil) return false;
|
|
2432
|
+
// The second attempt must be able to FINISH inside the client's patience: a bare
|
|
2433
|
+
// `claude` (240s ceiling, hard 300s death) whose first wait already ran 240s cannot
|
|
2434
|
+
// get a first byte from a 240s retry — that is guaranteed-wasted load; move instead.
|
|
2435
|
+
// Non-streaming has no ceiling (null): its wait is the flat floor either way.
|
|
2436
|
+
const ceilingMs = clientWaitCeilingMs(requestInfo, queueConfig);
|
|
2437
|
+
if (ceilingMs != null) {
|
|
2438
|
+
const nextWaitMs = scaledTtfbMs(UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, requestInfo.bodyBytes, ceilingMs);
|
|
2439
|
+
const elapsedMs = Date.now() - (requestInfo.startedAt || Date.now());
|
|
2440
|
+
if (elapsedMs + nextWaitMs > ceilingMs) return false;
|
|
2441
|
+
}
|
|
2442
|
+
return !clientHasRealBytes(res, requestInfo);
|
|
2443
|
+
}
|
|
2444
|
+
|
|
2445
|
+
/**
|
|
2446
|
+
* D5: one line per FAILED attempt, naming the account, the error class and what the
|
|
2447
|
+
* pool did next — so "why did this move" is answerable from the log without a forensic
|
|
2448
|
+
* dig (the Mac pool logged failed attempts not at all before this). nextAction tokens:
|
|
2449
|
+
* released-cancel | retry-same | moved | queued | surfaced.
|
|
2450
|
+
*/
|
|
2451
|
+
function logFailedAttempt(account, kind, detail, nextAction) {
|
|
2452
|
+
console.log(`[Maxpool] attempt failed (${kind}) on "${account}" — ${detail} → ${nextAction}`);
|
|
2453
|
+
}
|
|
2454
|
+
|
|
2455
|
+
/** A first-byte, mid-stream idle or stalled non-streaming body timeout concerns ONE
|
|
2456
|
+
* request, not the account (review r2 A / r3 R3-1). Resets, DNS, fetch failures do not. */
|
|
2457
|
+
function isRequestScopedTimeout(err) {
|
|
2458
|
+
return err?.code === 'UPSTREAM_TTFB' || err?.code === 'UPSTREAM_IDLE' || err?.code === 'UPSTREAM_BODY';
|
|
2459
|
+
}
|
|
2460
|
+
|
|
2461
|
+
/** Seconds this lease has been held on its account, one decimal ("12.3"). */
|
|
2462
|
+
function heldSeconds(lease) {
|
|
2463
|
+
return ((Date.now() - (lease.startedAt || Date.now())) / 1000).toFixed(1);
|
|
2464
|
+
}
|
|
2465
|
+
|
|
2211
2466
|
function parseRetryAfter(value) {
|
|
2212
2467
|
if (value == null) return null;
|
|
2213
2468
|
const n = parseInt(value, 10);
|
|
@@ -2432,7 +2687,7 @@ function isContextLengthError(errorBody) {
|
|
|
2432
2687
|
return /exceeded model token limit|maximum context length|context length exceeded|context window (?:size )?(?:exceeded|too)|prompt is too long|input is too long|reduce the length of|too many (?:input )?tokens|request too large/i.test(errorBody);
|
|
2433
2688
|
}
|
|
2434
2689
|
|
|
2435
|
-
export const __serverTest = { directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
2690
|
+
export const __serverTest = { directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
2436
2691
|
|
|
2437
2692
|
async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
|
|
2438
2693
|
if (!upstreamRes.body) return '';
|
|
@@ -3142,10 +3397,40 @@ function clientHasRealBytes(res, requestInfo) {
|
|
|
3142
3397
|
return res.headersSent && !requestInfo.queueHeartbeatActive;
|
|
3143
3398
|
}
|
|
3144
3399
|
|
|
3145
|
-
async function queueAndRetry(
|
|
3400
|
+
async function queueAndRetry(...args) {
|
|
3401
|
+
// A call WITHOUT an attempt descriptor is exactly 16 args (through `cause`) — never
|
|
3402
|
+
// strip that. A 17th `{ attempt: [account, kind, detail] }` arg makes the queue print
|
|
3403
|
+
// the failed attempt's ONE line once its outcome is known (D5): `→ queued` when the
|
|
3404
|
+
// hold commits, `→ surfaced` when the queue declines without holding. A hold that
|
|
3405
|
+
// later gives up prints a separate `hold gave up` line — the attempt was queued.
|
|
3406
|
+
const desc = args.length > 16 ? args[args.length - 1] : null;
|
|
3407
|
+
if (!desc || !Array.isArray(desc.attempt)) return queueAndRetryInner(...args, null, null);
|
|
3408
|
+
const inner = args.slice(0, -1);
|
|
3409
|
+
const [acct, kind, detail] = desc.attempt;
|
|
3410
|
+
let held = false;
|
|
3411
|
+
let finalized = false;
|
|
3412
|
+
const onHold = () => {
|
|
3413
|
+
if (held) return;
|
|
3414
|
+
held = true;
|
|
3415
|
+
logFailedAttempt(acct, kind, detail, 'queued');
|
|
3416
|
+
};
|
|
3417
|
+
const onGiveUp = () => {
|
|
3418
|
+
if (finalized) return;
|
|
3419
|
+
finalized = true;
|
|
3420
|
+
if (held) console.log(`[Maxpool] hold gave up for the (${kind}) attempt on "${acct}" → surfaced`);
|
|
3421
|
+
else logFailedAttempt(acct, kind, `${detail}; not held`, 'surfaced');
|
|
3422
|
+
};
|
|
3423
|
+
const result = await queueAndRetryInner(...inner, onGiveUp, onHold);
|
|
3424
|
+
if (!result) onGiveUp();
|
|
3425
|
+
return result;
|
|
3426
|
+
}
|
|
3427
|
+
|
|
3428
|
+
async function queueAndRetryInner(
|
|
3146
3429
|
reason, req, res, body, accountManager, upstream, hooks, reqId, ctx, logDir,
|
|
3147
3430
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody = canRetryBufferedBody, cause = 'quota',
|
|
3431
|
+
onGiveUp = null, onHold = null,
|
|
3148
3432
|
) {
|
|
3433
|
+
const giveUp = (msg) => { onGiveUp?.(); return finishQueuedStreamIfNeeded(res, requestInfo, msg); };
|
|
3149
3434
|
if (!queueConfig.enabled || !canQueueBufferedBody || (res.headersSent && !requestInfo.queueHeartbeatActive) || res.destroyed) {
|
|
3150
3435
|
if (queueConfig.enabled && !canQueueBufferedBody && requestInfo.queueBlockedReason) {
|
|
3151
3436
|
console.log(`[Maxpool] Not queueing request: ${requestInfo.queueBlockedReason}`);
|
|
@@ -3231,7 +3516,7 @@ async function queueAndRetry(
|
|
|
3231
3516
|
// Weekly-capped but the reset time is unknown (cold start / probe failure):
|
|
3232
3517
|
// we can't estimate a wait, so don't pretend to — error honestly now.
|
|
3233
3518
|
if (retryPlan.cause === 'weekly_reset_unknown') {
|
|
3234
|
-
return
|
|
3519
|
+
return giveUp(honestMessage);
|
|
3235
3520
|
}
|
|
3236
3521
|
|
|
3237
3522
|
const streamHoldMaxMs = queueConfig.streamHoldMaxMs == null
|
|
@@ -3254,7 +3539,7 @@ async function queueAndRetry(
|
|
|
3254
3539
|
countTokensMaxWaitMs,
|
|
3255
3540
|
});
|
|
3256
3541
|
|
|
3257
|
-
if (queueWindowMs <= 0) return
|
|
3542
|
+
if (queueWindowMs <= 0) return giveUp(honestMessage);
|
|
3258
3543
|
|
|
3259
3544
|
// HOLD-vs-ERROR oracle (from nextRetryForRequest): HOLD only when a TEMPORARY
|
|
3260
3545
|
// cause has a finite real reset within the ceiling. ERROR FAST for permanent /
|
|
@@ -3294,16 +3579,16 @@ async function queueAndRetry(
|
|
|
3294
3579
|
if (ticket === null) {
|
|
3295
3580
|
// Backpressure: too many requests already waiting / too many bytes buffered.
|
|
3296
3581
|
// Reject honestly instead of growing the heap unbounded.
|
|
3297
|
-
return
|
|
3298
|
-
'Maxpool queue is full — too many requests are already waiting for capacity. Try again shortly.');
|
|
3582
|
+
return giveUp('Maxpool queue is full — too many requests are already waiting for capacity. Try again shortly.');
|
|
3299
3583
|
}
|
|
3300
3584
|
const elapsed = Date.now() - requestInfo.queueStartedAt;
|
|
3301
3585
|
const remaining = queueWindowMs - elapsed;
|
|
3302
3586
|
if (remaining <= 0) {
|
|
3303
3587
|
accountManager.removeQueuedRequest?.(requestInfo);
|
|
3304
|
-
return
|
|
3588
|
+
return giveUp(honestMessage);
|
|
3305
3589
|
}
|
|
3306
3590
|
|
|
3591
|
+
onHold?.(); // the hold is committed: the attempt that led here is `→ queued`
|
|
3307
3592
|
ctx.account = '(queued)';
|
|
3308
3593
|
hooks.onRequestRouted?.(reqId, { account: '(queued)' });
|
|
3309
3594
|
// Hold-chain note: when this hold expires still-full (queueHoldChain), the
|
|
@@ -3315,7 +3600,7 @@ async function queueAndRetry(
|
|
|
3315
3600
|
const available = await waitForAvailableRoute(req, res, accountManager, requestInfo, queueConfig, remaining);
|
|
3316
3601
|
if (!available) {
|
|
3317
3602
|
accountManager.removeQueuedRequest?.(requestInfo);
|
|
3318
|
-
return
|
|
3603
|
+
return giveUp(honestMessage);
|
|
3319
3604
|
}
|
|
3320
3605
|
|
|
3321
3606
|
// NOTE: the heartbeat is deliberately NOT cleared here. It must stay alive
|
|
@@ -3334,6 +3619,25 @@ async function queueAndRetry(
|
|
|
3334
3619
|
).then(() => true);
|
|
3335
3620
|
}
|
|
3336
3621
|
|
|
3622
|
+
// Arms the stream-forward grace timer. Shared by the direct and link paths. The
|
|
3623
|
+
// deadline is anchored ONCE per request (??=) so it bounds CUMULATIVE pre-headers time
|
|
3624
|
+
// across re-forwards; returns the timer (or null) for the caller to clear the moment
|
|
3625
|
+
// headers arrive.
|
|
3626
|
+
function armStreamForwardGrace(res, requestInfo, queueConfig, accountManager) {
|
|
3627
|
+
if (!requestInfo.stream || res.headersSent) return null;
|
|
3628
|
+
const streamForwardGraceMs = queueConfig?.streamForwardGraceMs == null
|
|
3629
|
+
? 10000
|
|
3630
|
+
: Math.max(0, Number(queueConfig.streamForwardGraceMs) || 0);
|
|
3631
|
+
requestInfo.streamGraceDeadline ??= Date.now() + streamForwardGraceMs;
|
|
3632
|
+
const graceDelay = Math.max(0, requestInfo.streamGraceDeadline - Date.now());
|
|
3633
|
+
const t = setTimeout(
|
|
3634
|
+
() => commitStreamGraceHeartbeat(res, requestInfo, queueConfig, accountManager),
|
|
3635
|
+
graceDelay,
|
|
3636
|
+
);
|
|
3637
|
+
t.unref?.();
|
|
3638
|
+
return t;
|
|
3639
|
+
}
|
|
3640
|
+
|
|
3337
3641
|
// The stream-forward grace-timer callback (extracted so its logic is unit-testable
|
|
3338
3642
|
// in isolation). Fires when a STREAMING request's upstream is slow to first byte:
|
|
3339
3643
|
// commits the SSE stream + heartbeat so the client never idle-times-out. Client-
|
|
@@ -3383,16 +3687,25 @@ function ensureQueueHeartbeat(res, requestInfo, queueConfig, accountManager) {
|
|
|
3383
3687
|
reapDead();
|
|
3384
3688
|
return;
|
|
3385
3689
|
}
|
|
3386
|
-
requestInfo
|
|
3387
|
-
|
|
3388
|
-
|
|
3690
|
+
ensureQueueHeartbeatInterval(res, requestInfo, heartbeatMs, () => reapDead());
|
|
3691
|
+
}
|
|
3692
|
+
|
|
3693
|
+
/** The keep-alive interval half of the queue heartbeat, for a response whose headers
|
|
3694
|
+
* are ALREADY committed (the link pass-through writes the link's own headers, r4):
|
|
3695
|
+
* pings every heartbeatMs until clearQueueHeartbeat, so the client stays alive across
|
|
3696
|
+
* the link body wait and any failover hop while clientHasRealBytes stays false. */
|
|
3697
|
+
function ensureQueueHeartbeatInterval(res, requestInfo, heartbeatMs, onDead = null) {
|
|
3698
|
+
const timer = setInterval(() => {
|
|
3699
|
+
if (res.destroyed || res.writableEnded) { clearQueueHeartbeat(requestInfo); onDead?.(); return; }
|
|
3389
3700
|
try {
|
|
3390
3701
|
res.write(QUEUE_KEEPALIVE);
|
|
3391
3702
|
} catch {
|
|
3392
|
-
|
|
3703
|
+
clearQueueHeartbeat(requestInfo); onDead?.();
|
|
3393
3704
|
}
|
|
3394
3705
|
}, heartbeatMs);
|
|
3395
|
-
|
|
3706
|
+
timer.unref?.();
|
|
3707
|
+
requestInfo.queueHeartbeatActive = true;
|
|
3708
|
+
requestInfo.queueHeartbeatTimer = timer;
|
|
3396
3709
|
}
|
|
3397
3710
|
|
|
3398
3711
|
function clearQueueHeartbeat(requestInfo) {
|
|
@@ -3967,13 +4280,15 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
3967
4280
|
let readFailed = false;
|
|
3968
4281
|
|
|
3969
4282
|
|
|
3970
|
-
//
|
|
3971
|
-
//
|
|
3972
|
-
//
|
|
3973
|
-
// the
|
|
3974
|
-
//
|
|
3975
|
-
|
|
3976
|
-
|
|
4283
|
+
// The queue/keep-alive heartbeat keeps running until the FIRST REAL BYTE goes to the
|
|
4284
|
+
// client (review r2 D): until then the client has only pings, so a failure between
|
|
4285
|
+
// upstream headers and the first body chunk can still fail over (clientHasRealBytes
|
|
4286
|
+
// stays false) and the client's idle watchdog stays fed. realWrite stops it right
|
|
4287
|
+
// before the first real write, so no ping can land between live SSE events (Bug A).
|
|
4288
|
+
const realWrite = (chunk) => {
|
|
4289
|
+
if (requestInfo.queueHeartbeatActive) clearQueueHeartbeat(requestInfo);
|
|
4290
|
+
return res.write(chunk);
|
|
4291
|
+
};
|
|
3977
4292
|
|
|
3978
4293
|
// Client left mid-stream: the caller drops its disconnect-abort once headers
|
|
3979
4294
|
// arrive, and the loop below only checks res.destroyed AFTER a read resolves — so
|
|
@@ -3986,6 +4301,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
3986
4301
|
// MODEL ECHO NORMALIZATION state (rationale at the write site).
|
|
3987
4302
|
let modelEchoPending = true;
|
|
3988
4303
|
let modelEchoBuffer = null;
|
|
4304
|
+
let modelEchoHoldStart = null; // r4: the echo cap counts from the FIRST held byte, not stream start
|
|
3989
4305
|
|
|
3990
4306
|
// SAFEGUARD-REFUSAL REROUTE (2026-09-14). Anthropic's server-side classifier can
|
|
3991
4307
|
// terminate a turn with HTTP 200 + `stop_reason:"refusal"` + stop_details.category
|
|
@@ -4004,7 +4320,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4004
4320
|
let refusalRerouting = false;
|
|
4005
4321
|
const refusalWatch = requestInfo?._refusalRerouteEligible === true;
|
|
4006
4322
|
let refusalHoldBuffer = refusalWatch ? [] : null;
|
|
4007
|
-
|
|
4323
|
+
let holdByteAt = null; // r4/r5: when the FIRST byte entered the (refusal or echo) hold — ONE 30s budget from there
|
|
4008
4324
|
|
|
4009
4325
|
try {
|
|
4010
4326
|
while (true) {
|
|
@@ -4041,6 +4357,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4041
4357
|
// rerouted turn leaves the client socket completely untouched — the caller can
|
|
4042
4358
|
// then re-dispatch onto a provider as if the Anthropic attempt never happened.
|
|
4043
4359
|
if (refusalHoldBuffer) {
|
|
4360
|
+
holdByteAt ??= Date.now();
|
|
4044
4361
|
refusalHoldBuffer.push(value);
|
|
4045
4362
|
const held = decoder.decode(concatUint8(refusalHoldBuffer));
|
|
4046
4363
|
const verdict = classifyHeldStreamPrefix(held);
|
|
@@ -4052,13 +4369,15 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4052
4369
|
}
|
|
4053
4370
|
if (verdict.decision === 'hold'
|
|
4054
4371
|
&& totalLen(refusalHoldBuffer) <= REFUSAL_HOLD_MAX_BYTES
|
|
4055
|
-
&& Date.now() -
|
|
4372
|
+
&& holdByteAt != null && Date.now() - holdByteAt <= REFUSAL_HOLD_MAX_MS) { // r5: ONE 30s budget from the first held byte
|
|
4056
4373
|
continue; // nothing decisive yet — keep holding, write nothing
|
|
4057
4374
|
}
|
|
4058
4375
|
// Decisive content, or either cap hit → fail open: release the held bytes as
|
|
4059
|
-
// this chunk and never hold again on this stream.
|
|
4376
|
+
// this chunk and never hold again on this stream. The echo hold (below) inherits
|
|
4377
|
+
// when the FIRST byte was held, so the two stacked holds share ONE 30s budget.
|
|
4060
4378
|
value = concatUint8(refusalHoldBuffer);
|
|
4061
4379
|
refusalHoldBuffer = null;
|
|
4380
|
+
modelEchoHoldStart = holdByteAt;
|
|
4062
4381
|
}
|
|
4063
4382
|
|
|
4064
4383
|
if (!committed) {
|
|
@@ -4079,6 +4398,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4079
4398
|
// concat with decoder-safe helpers.
|
|
4080
4399
|
let out = value;
|
|
4081
4400
|
if (requestInfo?.model && modelEchoPending) {
|
|
4401
|
+
modelEchoHoldStart ??= Date.now();
|
|
4082
4402
|
modelEchoBuffer = modelEchoBuffer ? [...modelEchoBuffer, value] : [value];
|
|
4083
4403
|
const s = decoder.decode(concatUint8(modelEchoBuffer));
|
|
4084
4404
|
if (s.includes('"model"') && s.includes('\n\n')) {
|
|
@@ -4100,6 +4420,16 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4100
4420
|
out = Buffer.from(concatUint8(modelEchoBuffer));
|
|
4101
4421
|
modelEchoBuffer = null;
|
|
4102
4422
|
modelEchoPending = false;
|
|
4423
|
+
} else if (modelEchoHoldStart != null && Date.now() - modelEchoHoldStart > REFUSAL_HOLD_MAX_MS) {
|
|
4424
|
+
// Same fail-open cap as the refusal hold (review r3 R3-4), anchored at the
|
|
4425
|
+
// first held byte (r4): a split model-bearing event arriving >30s after stream
|
|
4426
|
+
// START must still be rewritten, not released verbatim to leak the provider id.
|
|
4427
|
+
// trickles events without ever emitting a model-bearing one must not hold the
|
|
4428
|
+
// stream forever — release the held bytes verbatim (a missed rewrite warning
|
|
4429
|
+
// beats a dead session) so the idle guard and abort paths can act.
|
|
4430
|
+
out = Buffer.from(concatUint8(modelEchoBuffer));
|
|
4431
|
+
modelEchoBuffer = null;
|
|
4432
|
+
modelEchoPending = false;
|
|
4103
4433
|
} else {
|
|
4104
4434
|
continue; // hold until the model-bearing event is complete
|
|
4105
4435
|
}
|
|
@@ -4111,7 +4441,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4111
4441
|
// single decoded text feeds the scrubber (client lane) and the accounting lane.
|
|
4112
4442
|
const chunkText = decoder.decode(out, { stream: true });
|
|
4113
4443
|
const writeText = scrubber ? scrubber.push(chunkText) : null;
|
|
4114
|
-
const ok = scrubber ? (writeText ?
|
|
4444
|
+
const ok = scrubber ? (writeText ? realWrite(writeText) : true) : realWrite(out);
|
|
4115
4445
|
|
|
4116
4446
|
const text = chunkText;
|
|
4117
4447
|
|
|
@@ -4196,7 +4526,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4196
4526
|
if (!readFailed && !res.destroyed) {
|
|
4197
4527
|
try {
|
|
4198
4528
|
if (!committed && !res.headersSent) { res.writeHead(status, responseHeaders); committed = true; }
|
|
4199
|
-
|
|
4529
|
+
realWrite(held);
|
|
4200
4530
|
} catch { /* client already gone */ }
|
|
4201
4531
|
}
|
|
4202
4532
|
}
|
|
@@ -4210,7 +4540,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4210
4540
|
const held = Buffer.from(concatUint8(modelEchoBuffer));
|
|
4211
4541
|
modelEchoBuffer = null;
|
|
4212
4542
|
try {
|
|
4213
|
-
if (!readFailed && !res.destroyed)
|
|
4543
|
+
if (!readFailed && !res.destroyed) realWrite(held);
|
|
4214
4544
|
} catch { /* client already gone */ }
|
|
4215
4545
|
const heldText = held.toString('utf8');
|
|
4216
4546
|
if (streamLog) streamLog.push(heldText);
|
|
@@ -4224,6 +4554,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4224
4554
|
// Cancel upstream reader to stop consuming data nobody needs
|
|
4225
4555
|
reader.cancel().catch(() => {});
|
|
4226
4556
|
if (!readFailed) {
|
|
4557
|
+
clearQueueHeartbeat(requestInfo); // finishing: no pings after this point
|
|
4227
4558
|
if (!committed && !res.headersSent) res.writeHead(status, responseHeaders);
|
|
4228
4559
|
// Truncation sentinel FIRST (before end()) — a write after end() throws and
|
|
4229
4560
|
// would be swallowed, silently producing the truncated stream this guards.
|
|
@@ -4239,11 +4570,11 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4239
4570
|
// client a corrupt frame ahead of the error.
|
|
4240
4571
|
if (scrubber && !truncatedMidEvent && !res.destroyed && !res.writableEnded) {
|
|
4241
4572
|
const tail = scrubber.flush();
|
|
4242
|
-
if (tail) { try {
|
|
4573
|
+
if (tail) { try { realWrite(tail); } catch { /* client gone */ } }
|
|
4243
4574
|
}
|
|
4244
4575
|
if ((committed || res.headersSent) && truncatedMidEvent && !res.destroyed && !res.writableEnded) {
|
|
4245
4576
|
try {
|
|
4246
|
-
|
|
4577
|
+
realWrite(`event: error\ndata: ${JSON.stringify({
|
|
4247
4578
|
type: 'error',
|
|
4248
4579
|
error: { type: 'api_error', message: 'Upstream closed the stream before the response completed. Retry the message.' },
|
|
4249
4580
|
})}\n\n`);
|