maxpool 1.24.91 → 1.24.93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/event-log.js +79 -1
- package/src/index.js +332 -149
- package/src/reload-protocol.js +92 -2
- package/src/server.js +353 -27
- package/src/terminal-mode.js +124 -0
- package/src/tui.js +119 -9
package/src/server.js
CHANGED
|
@@ -66,6 +66,120 @@ const isNetworkCode = c => Boolean(c) && NETWORK_ERROR_CODES.has(c);
|
|
|
66
66
|
const NETWORK_SOAK_NONSTREAM_MS = Math.max(5_000,
|
|
67
67
|
Number(process.env.MAXPOOL_NETWORK_SOAK_NONSTREAM_MS) || 45_000);
|
|
68
68
|
const QUEUE_KEEPALIVE = 'event: ping\ndata: {}\n\n';
|
|
69
|
+
|
|
70
|
+
// PER-REQUEST DEADLINE, NON-STREAMING (release B, 2026-11-13). A non-streaming request has
|
|
71
|
+
// no keep-alive, so the CLIENT's own timer decides when it is abandoned. maxpool used to
|
|
72
|
+
// wait a flat 120s per hop and chain hops without limit, serving into sockets the client had
|
|
73
|
+
// already closed. When the client tells us how long it will wait (X-Stainless-Timeout, or the
|
|
74
|
+
// pool header from the Mac), every wait is bounded by ONE deadline anchored at arrival.
|
|
75
|
+
// A request with no usable patience header gets NO deadline: today's behaviour, never a new cut-off.
|
|
76
|
+
const NONSTREAM_HOP_CAP_MS = 290_000; // undici's own headers/body timeout is 300s: nothing waits past this
|
|
77
|
+
const NONSTREAM_FLOOR_PATIENCE_MS = 15_000; // usable patience is never below 15s once the margin is taken
|
|
78
|
+
const MAX_PATIENCE_MS = 3_600_000;
|
|
79
|
+
const posMsFromEnv = (name, dflt) => {
|
|
80
|
+
const n = Number(process.env[name]);
|
|
81
|
+
return Number.isFinite(n) && n > 0 ? n : dflt;
|
|
82
|
+
};
|
|
83
|
+
/** Live knob values (env read per call so a two-pool test can inject a small cap). */
|
|
84
|
+
function deadlineKnobs() {
|
|
85
|
+
return {
|
|
86
|
+
NONSTREAM_MARGIN_MS: posMsFromEnv('MAXPOOL_NONSTREAM_MARGIN_MS', 15_000), // client header: leave this much before its timer
|
|
87
|
+
POOL_TRANSIT_MARGIN_MS: 3_000, // pool header: the Mac already took its margin
|
|
88
|
+
NONSTREAM_MIN_MOVE_MS: posMsFromEnv('MAXPOOL_NONSTREAM_MIN_MOVE_MS', 60_000), // below this, never re-send to another account
|
|
89
|
+
LINK_RETURN_MARGIN_MS: posMsFromEnv('MAXPOOL_LINK_RETURN_MARGIN_MS', 10_000), // G: the cloud's answer must travel back inside the Mac's wait
|
|
90
|
+
LINK_HOP_CAP_MS: Math.min(posMsFromEnv('MAXPOOL_LINK_HOP_CAP_MS', NONSTREAM_HOP_CAP_MS), NONSTREAM_HOP_CAP_MS),
|
|
91
|
+
UPSTREAM_BODY_MS,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
// A plain positive decimal. `Number('')` is 0, `Number('abc')` is NaN, `Number('Infinity')`
|
|
95
|
+
// is Infinity and `setTimeout(NaN)` fires in 1ms: every one of them must mean "no header".
|
|
96
|
+
const PLAIN_POSITIVE_NUMBER = /^(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/;
|
|
97
|
+
function parsePositiveHeader(raw) {
|
|
98
|
+
if (raw == null) return null;
|
|
99
|
+
const s = String(raw).trim();
|
|
100
|
+
if (!PLAIN_POSITIVE_NUMBER.test(s)) return null;
|
|
101
|
+
const n = Number(s);
|
|
102
|
+
return Number.isFinite(n) && n > 0 ? n : null;
|
|
103
|
+
}
|
|
104
|
+
/** The client's patience: first VALID of the pool header (ms) → X-Stainless-Timeout (s) →
|
|
105
|
+
* x-stainless-read-timeout (s). null = unknown, i.e. no deadline. */
|
|
106
|
+
function parsePatience(headers = {}) {
|
|
107
|
+
const k = deadlineKnobs();
|
|
108
|
+
const pool = parsePositiveHeader(headerValue(headers, 'x-maxpool-deadline-ms'));
|
|
109
|
+
if (pool != null) return { ms: Math.min(pool, MAX_PATIENCE_MS), source: 'pool', poolSupplied: true };
|
|
110
|
+
for (const [name, source] of [['x-stainless-timeout', 'stainless-timeout'], ['x-stainless-read-timeout', 'stainless-read-timeout']]) {
|
|
111
|
+
const secs = parsePositiveHeader(headerValue(headers, name));
|
|
112
|
+
if (secs == null) continue;
|
|
113
|
+
const floor = k.NONSTREAM_MARGIN_MS + NONSTREAM_FLOOR_PATIENCE_MS;
|
|
114
|
+
return { ms: Math.min(Math.max(secs * 1000, floor), MAX_PATIENCE_MS), source, poolSupplied: false };
|
|
115
|
+
}
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
/** deadlineAt for a request that arrived at `arrivedAt`; null when the client gave no usable patience. */
|
|
119
|
+
function computeDeadline(headers, arrivedAt) {
|
|
120
|
+
const p = parsePatience(headers);
|
|
121
|
+
if (!p) return null;
|
|
122
|
+
// F1 (mixed version): a request FORWARDED BY ANOTHER POOL (hop header) that carries no pool deadline header
|
|
123
|
+
// comes from an OLD Mac (v1.24.91). It forwards the client's x-stainless-timeout, but an old Mac books any
|
|
124
|
+
// 5xx it passes through as an account failure, so this pool must behave exactly as before: no deadline.
|
|
125
|
+
// (A new Mac always sends the pool header on a non-streaming link hop.)
|
|
126
|
+
if (!p.poolSupplied && headerValue(headers, HOP_HEADER) !== '') return null;
|
|
127
|
+
const k = deadlineKnobs();
|
|
128
|
+
const usableMs = p.poolSupplied
|
|
129
|
+
? Math.max(p.ms - k.POOL_TRANSIT_MARGIN_MS, 1_000)
|
|
130
|
+
: p.ms - k.NONSTREAM_MARGIN_MS;
|
|
131
|
+
return { deadlineAt: arrivedAt + usableMs, patienceMs: p.ms, source: p.source, poolSupplied: p.poolSupplied };
|
|
132
|
+
}
|
|
133
|
+
/** Time left on a non-streaming request's deadline. Infinity = no deadline (streaming, or a header-less client). */
|
|
134
|
+
function nonStreamRemainingMs(requestInfo = {}) {
|
|
135
|
+
if (requestInfo.stream || requestInfo.deadlineAt == null) return Infinity;
|
|
136
|
+
return requestInfo.deadlineAt - Date.now();
|
|
137
|
+
}
|
|
138
|
+
/** First-byte wait for one DIRECT hop. WITH a deadline: min(remaining, hop cap) — the flat/scaled
|
|
139
|
+
* first-byte floor (120s) does NOT apply, because the client has said it will wait longer and a
|
|
140
|
+
* floor timeout would abandon an answer that still fits (F3). WITHOUT a deadline: `waitMs`, today's wait.
|
|
141
|
+
* The cap is the 290s undici bound (injectable lower via MAXPOOL_LINK_HOP_CAP_MS for scaled tests). */
|
|
142
|
+
function nonStreamFirstByteWait(requestInfo, waitMs) {
|
|
143
|
+
const remaining = nonStreamRemainingMs(requestInfo);
|
|
144
|
+
if (remaining === Infinity) return { ms: waitMs, deadlineBound: false };
|
|
145
|
+
const cap = deadlineKnobs().LINK_HOP_CAP_MS;
|
|
146
|
+
return remaining <= cap ? { ms: Math.max(1, Math.floor(remaining)), deadlineBound: true } : { ms: cap, deadlineBound: false };
|
|
147
|
+
}
|
|
148
|
+
/** Non-streaming body-read wait: min(UPSTREAM_BODY_MS, remaining). */
|
|
149
|
+
function nonStreamBodyWait(requestInfo, bodyMs) {
|
|
150
|
+
const remaining = nonStreamRemainingMs(requestInfo);
|
|
151
|
+
if (remaining === Infinity) return { ms: bodyMs, deadlineBound: false };
|
|
152
|
+
return remaining <= bodyMs ? { ms: Math.max(1, Math.floor(remaining)), deadlineBound: true } : { ms: bodyMs, deadlineBound: false };
|
|
153
|
+
}
|
|
154
|
+
/** ONE NUMBER for a link hop: linkWait = min(remaining − G, cap) (never below min(15s, remaining)),
|
|
155
|
+
* header = linkWait − G. So header + G ≤ linkWait always: the cloud's whole hop chain ends
|
|
156
|
+
* inside this ONE Mac link wait. null = no usable time left (never arm a 0ms timer). */
|
|
157
|
+
function computeLinkHop(remainingMs, { capMs, marginMs } = {}) {
|
|
158
|
+
const k = deadlineKnobs();
|
|
159
|
+
const G = marginMs ?? k.LINK_RETURN_MARGIN_MS;
|
|
160
|
+
const cap = capMs ?? k.LINK_HOP_CAP_MS;
|
|
161
|
+
if (!Number.isFinite(remainingMs) || remainingMs <= 0) return null;
|
|
162
|
+
const linkWaitMs = Math.floor(Math.min(cap, Math.max(remainingMs - G, Math.min(15_000, remainingMs))));
|
|
163
|
+
const headerMs = linkWaitMs - G;
|
|
164
|
+
if (headerMs < 1_000) return null;
|
|
165
|
+
return { linkWaitMs, headerMs, deadlineBound: remainingMs - G <= cap };
|
|
166
|
+
}
|
|
167
|
+
/** The answer for a request whose client patience ran out: retryable by every SDK (5xx). */
|
|
168
|
+
function deadlineErrorPayload() {
|
|
169
|
+
return {
|
|
170
|
+
type: 'error',
|
|
171
|
+
error: {
|
|
172
|
+
type: 'timeout_error',
|
|
173
|
+
message: 'This request ran out of time before Claude answered. Sending the message again normally works.',
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
/** 504 for a deadline expiry. The pool-only marker rides ONLY when the request itself carried the pool
|
|
178
|
+
* deadline header (an old Mac / header-less request keeps today's answer and never sees it). */
|
|
179
|
+
function sendDeadlineResponse(res, requestInfo) {
|
|
180
|
+
const headers = requestInfo.deadlinePoolSupplied ? { 'x-maxpool-request-scoped': '1' } : {};
|
|
181
|
+
sendErrorResponse(res, requestInfo, 504, deadlineErrorPayload(), headers);
|
|
182
|
+
}
|
|
69
183
|
// Account screen shell (served keyless at /maxpool/screen; the page itself prompts
|
|
70
184
|
// for a pool key and reads the key-gated /maxpool/status). Read once at boot.
|
|
71
185
|
const SCREEN_HTML = (() => {
|
|
@@ -98,12 +212,16 @@ const REFUSAL_HOLD_MAX_MS = 30_000;
|
|
|
98
212
|
// disabling it. Short (not reset-length) so a provider-pinned session's hold window
|
|
99
213
|
// isn't blown, and a still-exhausted provider just re-benches on the next single retry.
|
|
100
214
|
const PROVIDER_FORBIDDEN_COOLDOWN_SEC = Math.max(60, Number(process.env.MAXPOOL_PROVIDER_403_COOLDOWN_SEC) || 1800);
|
|
101
|
-
// Backstop reaper idle ceiling. Floored WELL above the longest a legit
|
|
102
|
-
// request can go silent (no heartbeat there): TTFB 120s + nonStreamMaxWaitMs
|
|
103
|
-
// UPSTREAM_BODY_MS 300s ≈ 720s — so a slow-but-alive non-streaming request is never
|
|
104
|
-
// reaped at completion.
|
|
215
|
+
// Backstop reaper idle ceiling. Floored WELL above the longest a legit header-less
|
|
216
|
+
// NON-streaming request can go silent (no heartbeat there): TTFB 120s + nonStreamMaxWaitMs
|
|
217
|
+
// 300s + UPSTREAM_BODY_MS 300s ≈ 720s — so a slow-but-alive non-streaming request is never
|
|
218
|
+
// reaped at completion. A request with a client-patience deadline can legitimately stay
|
|
219
|
+
// silent longer (290s hops, up to ~1470s inside a 1h patience); the reaper extends to
|
|
220
|
+
// deadlineAt + DEADLINE_REAP_GRACE_MS for those, since the deadline itself ends them.
|
|
221
|
+
// Streaming holds heartbeat, so they're safe at any ceiling.
|
|
105
222
|
// Exported so index.js sizes RELOAD_DRAIN_MS off the SAME value (no drift).
|
|
106
223
|
export const REQUEST_IDLE_MAX_MS = Math.max(900_000, Number(process.env.MAXPOOL_REQUEST_IDLE_MAX_MS) || 1_200_000);
|
|
224
|
+
const DEADLINE_REAP_GRACE_MS = 60_000;
|
|
107
225
|
|
|
108
226
|
const DEFAULT_QUEUE = {
|
|
109
227
|
enabled: true,
|
|
@@ -250,6 +368,8 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
250
368
|
requestTimeout: Math.max(1_000, Number(process.env.MAXPOOL_REQUEST_TIMEOUT_MS) || 300_000),
|
|
251
369
|
connectionsCheckingInterval: Math.max(100, Number(process.env.MAXPOOL_CONNECTIONS_CHECK_MS) || 30_000),
|
|
252
370
|
}, async (req, res) => {
|
|
371
|
+
// B1: the deadline is anchored HERE, at the request event, before auth and body buffering.
|
|
372
|
+
const arrivedAt = Date.now();
|
|
253
373
|
try {
|
|
254
374
|
// Disable Nagle on the client socket: flush each SSE event immediately instead
|
|
255
375
|
// of coalescing small writes into bursts. Smooths streaming cadence toward the
|
|
@@ -503,6 +623,15 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
503
623
|
// THIS account, from lease.startedAt.) Was never set, so every direct-path
|
|
504
624
|
// client-gone line read 0.0s.
|
|
505
625
|
requestInfo.startedAt = Date.now();
|
|
626
|
+
// B1: non-streaming only; a request with no usable patience header gets NO deadline.
|
|
627
|
+
if (!requestInfo.stream) {
|
|
628
|
+
const dl = computeDeadline(req.headers, arrivedAt);
|
|
629
|
+
if (dl) {
|
|
630
|
+
requestInfo.deadlineAt = dl.deadlineAt;
|
|
631
|
+
requestInfo.deadlineSource = dl.source;
|
|
632
|
+
requestInfo.deadlinePoolSupplied = dl.poolSupplied;
|
|
633
|
+
}
|
|
634
|
+
}
|
|
506
635
|
const maxQueuedBodyBytes = queueConfig.maxQueuedBodyBytes == null
|
|
507
636
|
? Infinity
|
|
508
637
|
: Math.max(0, Number(queueConfig.maxQueuedBodyBytes) || 0);
|
|
@@ -579,6 +708,10 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
579
708
|
// request's heartbeat commits writes LONG before upstream answers.
|
|
580
709
|
firstUpstreamByteAt: requestInfo?._firstUpstreamByteAt ?? null,
|
|
581
710
|
upstreamForwardStartAt: requestInfo?._upstreamForwardStartAt ?? null,
|
|
711
|
+
// B8: makes the non-streaming share measurable. null = no request shape was ever parsed.
|
|
712
|
+
// F9: only the message-creating call (POST /v1/messages) has a streaming shape — count_tokens and GETs
|
|
713
|
+
// must not read as "non-streaming" and swamp the share.
|
|
714
|
+
stream: requestInfo?.method === 'POST' && /^\/v1\/messages(?:\?|$)/.test(String(req.url || '')) ? Boolean(requestInfo.stream) : null,
|
|
582
715
|
});
|
|
583
716
|
}
|
|
584
717
|
} catch (err) {
|
|
@@ -808,6 +941,7 @@ async function forwardRequest(
|
|
|
808
941
|
// waiter instead of preferentially jumping ahead of the FIFO for the rest of
|
|
809
942
|
// this request's failover chain.
|
|
810
943
|
requestInfo.queueAdmitted = false;
|
|
944
|
+
requestInfo._clientRes = res; // B10: lets the shared chokepoints see the client socket
|
|
811
945
|
|
|
812
946
|
// Abort the upstream fetch and release the lease if the CLIENT disconnects during
|
|
813
947
|
// the pre-response window (token refresh + connect + waiting for the upstream's
|
|
@@ -935,6 +1069,7 @@ async function forwardRequest(
|
|
|
935
1069
|
// locally went live remotely). The remote pool owns its own accounts.
|
|
936
1070
|
if (/^x-maxpool-(zai|kimi)-/.test(lk)) continue;
|
|
937
1071
|
if (lk === 'x-maxpool-pin-account') continue; // M4: only this pool sets a pin
|
|
1072
|
+
if (lk === 'x-maxpool-deadline-ms') continue; // B4: ONE number — this hop computes its own below
|
|
938
1073
|
linkHeaders[k] = v;
|
|
939
1074
|
}
|
|
940
1075
|
linkHeaders[HOP_HEADER] = String(hop);
|
|
@@ -966,6 +1101,21 @@ async function forwardRequest(
|
|
|
966
1101
|
if (linkDone) return; linkDone = true;
|
|
967
1102
|
accountManager.releaseAccount(lease, outcome);
|
|
968
1103
|
};
|
|
1104
|
+
// B4 (non-streaming with a deadline): linkWait = min(remaining − G, 290s); the cloud is told
|
|
1105
|
+
// linkWait − G, so its whole hop chain ends inside this ONE wait. No time left → 504, no call.
|
|
1106
|
+
let linkHop = null;
|
|
1107
|
+
if (!requestInfo.stream && requestInfo.deadlineAt != null) {
|
|
1108
|
+
linkHop = computeLinkHop(nonStreamRemainingMs(requestInfo));
|
|
1109
|
+
if (!linkHop) {
|
|
1110
|
+
finishLink({ neutral: true, error: 'deadline', kind: 'deadline' });
|
|
1111
|
+
detachLinkClose();
|
|
1112
|
+
ctx.status = 504;
|
|
1113
|
+
logFailedAttempt(account.name, 'deadline', 'no time left before the client gives up', 'surfaced');
|
|
1114
|
+
sendDeadlineResponse(res, requestInfo);
|
|
1115
|
+
return;
|
|
1116
|
+
}
|
|
1117
|
+
linkHeaders['x-maxpool-deadline-ms'] = String(linkHop.headerMs);
|
|
1118
|
+
}
|
|
969
1119
|
try {
|
|
970
1120
|
// P4a: a blackholed link must not hang the request (and hold a lease) for
|
|
971
1121
|
// minutes — bound time-to-headers like the direct path's TTFB guard. GLM
|
|
@@ -977,7 +1127,7 @@ async function forwardRequest(
|
|
|
977
1127
|
// clears this timer for every HEALTHY link, so a scaled wait only ever ran against
|
|
978
1128
|
// a dead/blackholed tunnel — stretching its detection from 120s to minutes. The
|
|
979
1129
|
// big-prompt first-byte timeouts measured were all on the direct path.
|
|
980
|
-
const linkTtfbMs = Math.max(15_000, Number(process.env.MAXPOOL_LINK_TTFB_MS) || UPSTREAM_TTFB_MS);
|
|
1130
|
+
const linkTtfbMs = linkHop ? linkHop.linkWaitMs : Math.max(15_000, Number(process.env.MAXPOOL_LINK_TTFB_MS) || UPSTREAM_TTFB_MS);
|
|
981
1131
|
const linkCtl = new AbortController();
|
|
982
1132
|
const linkTtfb = setTimeout(() => linkCtl.abort(), linkTtfbMs);
|
|
983
1133
|
// P2: the link path had NO pre-headers keep-alive — a streaming client sat on a
|
|
@@ -1050,6 +1200,22 @@ async function forwardRequest(
|
|
|
1050
1200
|
sendErrorResponse(res, requestInfo, upRes.status, { type: 'error', error: { type: 'overloaded_error', message: `Team pool account "${account.name}" is at capacity` } });
|
|
1051
1201
|
return;
|
|
1052
1202
|
}
|
|
1203
|
+
// B7: the cloud ran out of the deadline THIS request gave it (504 + request-scoped). It says
|
|
1204
|
+
// nothing about the cloud row or the link: release NEUTRAL (no cooldownUntil, no failed count,
|
|
1205
|
+
// no streak) and treat it like any other request-scoped miss — try another route only if time remains.
|
|
1206
|
+
if (account.remote && upRes.status === 504 && upRes.headers.get('x-maxpool-request-scoped') === '1' && !clientHasRealBytes(res, requestInfo)) {
|
|
1207
|
+
const errorText = await readErrorBody(upRes);
|
|
1208
|
+
finishLink({ neutral: true, error: 'deadline', kind: 'deadline' });
|
|
1209
|
+
excludedIndexes.add(account.index);
|
|
1210
|
+
if (retryCount + 1 < maxAttempts && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
1211
|
+
logFailedAttempt(account.name, 'remote-504', 'linked pool ran out of the request deadline', 'moved');
|
|
1212
|
+
detachLinkClose();
|
|
1213
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
|
|
1214
|
+
}
|
|
1215
|
+
logFailedAttempt(account.name, 'remote-504', 'linked pool ran out of the request deadline; no time to move', 'surfaced');
|
|
1216
|
+
sendErrorBody(res, requestInfo, 504, errorText, upRes.headers);
|
|
1217
|
+
return;
|
|
1218
|
+
}
|
|
1053
1219
|
// A request released from the queue already holds a committed keep-alive stream.
|
|
1054
1220
|
// An error on it is framed as an SSE error; a 2xx keeps the heartbeat until the
|
|
1055
1221
|
// first real chunk (below).
|
|
@@ -1137,6 +1303,17 @@ async function forwardRequest(
|
|
|
1137
1303
|
logFailedAttempt(account.name, 'client_cancelled', `client left after ${heldS}s, nothing sent`, 'released-cancel');
|
|
1138
1304
|
return;
|
|
1139
1305
|
}
|
|
1306
|
+
// B12: the wait was cut by the CLIENT'S deadline (not the per-hop cap, not a dead link): neither an
|
|
1307
|
+
// account fault nor a link fault — release neutral, and answer a retryable 504 (nothing left to move on).
|
|
1308
|
+
// (A no-headers AbortError that is not a client leaving can only be this hop's timer.)
|
|
1309
|
+
if (linkHop?.deadlineBound && err?.name === 'AbortError' && !linkClientGone && linkStatus == null && !clientHasRealBytes(res, requestInfo)) {
|
|
1310
|
+
finishLink({ neutral: true, error: 'deadline', kind: 'deadline' });
|
|
1311
|
+
detachLinkClose();
|
|
1312
|
+
ctx.status = 504;
|
|
1313
|
+
logFailedAttempt(account.name, 'deadline', `no answer after ${heldS}s (client patience), non-streaming`, 'surfaced');
|
|
1314
|
+
sendDeadlineResponse(res, requestInfo);
|
|
1315
|
+
return;
|
|
1316
|
+
}
|
|
1140
1317
|
// Everything that reaches this catch is request-scoped (D4): a remote 429/503 is
|
|
1141
1318
|
// handled — account-scoped — by its own branch above before anything can throw, so
|
|
1142
1319
|
// only a TTFB abort, a mid-stream cancel or an unknown link error lands here. It
|
|
@@ -1239,10 +1416,27 @@ async function forwardRequest(
|
|
|
1239
1416
|
// this fixed clock, so a long healthy stream is never cut.
|
|
1240
1417
|
const ttfbController = new AbortController();
|
|
1241
1418
|
const ttfbCeilingMs = clientWaitCeilingMs(requestInfo, queueConfig);
|
|
1242
|
-
|
|
1419
|
+
let ttfbMs = scaledTtfbMs(UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, requestInfo.bodyBytes, ttfbCeilingMs);
|
|
1420
|
+
// B2/N-3: a non-streaming request with a deadline never waits past it (nor past 290s, undici's own
|
|
1421
|
+
// bound), and with no time left never reaches upstream at all (never arm a 0ms timer).
|
|
1422
|
+
let ttfbDeadlineBound = false;
|
|
1423
|
+
if (!requestInfo.stream && requestInfo.deadlineAt != null) {
|
|
1424
|
+
if (nonStreamRemainingMs(requestInfo) <= 0) {
|
|
1425
|
+
res.off('close', onClientClose);
|
|
1426
|
+
accountManager.relinquishUpstreamProbe?.(lease);
|
|
1427
|
+
accountManager.releaseAccount(lease, { neutral: true, error: 'deadline', kind: 'deadline' });
|
|
1428
|
+
ctx.status = 504;
|
|
1429
|
+
logFailedAttempt(account.name, 'deadline', 'no time left before the client gives up', 'surfaced');
|
|
1430
|
+
sendDeadlineResponse(res, requestInfo);
|
|
1431
|
+
return;
|
|
1432
|
+
}
|
|
1433
|
+
const w = nonStreamFirstByteWait(requestInfo, ttfbMs);
|
|
1434
|
+
ttfbMs = w.ms;
|
|
1435
|
+
ttfbDeadlineBound = w.deadlineBound;
|
|
1436
|
+
}
|
|
1243
1437
|
noteScaledTtfb(requestInfo, account.name, 'direct', ttfbMs, UPSTREAM_TTFB_MS, ttfbCeilingMs);
|
|
1244
1438
|
const ttfbTimer = setTimeout(
|
|
1245
|
-
() => ttfbController.abort(Object.assign(new Error('upstream TTFB timeout'), { code: 'UPSTREAM_TTFB' })),
|
|
1439
|
+
() => ttfbController.abort(Object.assign(new Error('upstream TTFB timeout'), { code: ttfbDeadlineBound ? 'UPSTREAM_DEADLINE' : 'UPSTREAM_TTFB' })),
|
|
1246
1440
|
ttfbMs,
|
|
1247
1441
|
);
|
|
1248
1442
|
ttfbTimer.unref?.();
|
|
@@ -1666,7 +1860,8 @@ async function forwardRequest(
|
|
|
1666
1860
|
logSections.push(`=== RESPONSE ${upstreamRes.status} — "${account.name}" cooling down, failing over ===\n${formatHeaders(upstreamRes.headers)}`);
|
|
1667
1861
|
}
|
|
1668
1862
|
|
|
1669
|
-
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
1863
|
+
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
1864
|
+
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
1670
1865
|
logFailedAttempt(account.name, `upstream_${upstreamRes.status}`, 'retriable upstream status, cooling down', 'moved');
|
|
1671
1866
|
return forwardRequest(
|
|
1672
1867
|
req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
|
|
@@ -1825,7 +2020,8 @@ async function forwardRequest(
|
|
|
1825
2020
|
: null;
|
|
1826
2021
|
const issuerIsClaude = issuer && issuer.type !== 'provider';
|
|
1827
2022
|
if (lease.migratedFromName && issuerIsClaude && requestInfo.sessionKey
|
|
1828
|
-
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2023
|
+
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2024
|
+
&& resendAllowed(requestInfo)) {
|
|
1829
2025
|
accountManager.revertSessionBinding(requestInfo.sessionKey, lease.migratedFromName);
|
|
1830
2026
|
excludedIndexes.add(account.index);
|
|
1831
2027
|
console.log(`[Maxpool] Thinking-signature fail-safe: reverting session to issuer "${lease.migratedFromName}" and retrying`);
|
|
@@ -1841,7 +2037,8 @@ async function forwardRequest(
|
|
|
1841
2037
|
// EFFORT REPAIR. A rejected output_config.effort is a hard failure of whatever the
|
|
1842
2038
|
// client was doing (a web search, a tool call) — worth healing rather than surfacing.
|
|
1843
2039
|
if (effortMode && !requestInfo.effortRepaired
|
|
1844
|
-
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2040
|
+
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2041
|
+
&& resendAllowed(requestInfo)) {
|
|
1845
2042
|
const fix = repairEffort(body, effortMode, errorBody);
|
|
1846
2043
|
if (fix.body) {
|
|
1847
2044
|
// Latch it so LATER turns apply the working level up front: the client resends the
|
|
@@ -1885,7 +2082,7 @@ async function forwardRequest(
|
|
|
1885
2082
|
const isDirectiveFormRejection = account.type !== 'provider'
|
|
1886
2083
|
&& upstreamRes.status === 400
|
|
1887
2084
|
&& /output_config[^:]*: *Extra inputs are not permitted/i.test(errorBody);
|
|
1888
|
-
if (isDirectiveFormRejection && requestInfo.orderingRepaired && !requestInfo.orderingFallback && canRepairBody) {
|
|
2085
|
+
if (isDirectiveFormRejection && requestInfo.orderingRepaired && !requestInfo.orderingFallback && canRepairBody && resendAllowed(requestInfo)) {
|
|
1889
2086
|
// Rewrite every directive-only system (the repair's own output) as a PLAIN
|
|
1890
2087
|
// assistant turn carrying the directives text — ordinary history, accepted by
|
|
1891
2088
|
// every validator, nothing dropped. A system ENDING the array stays legal
|
|
@@ -1909,7 +2106,7 @@ async function forwardRequest(
|
|
|
1909
2106
|
);
|
|
1910
2107
|
}
|
|
1911
2108
|
}
|
|
1912
|
-
if (isOrderingRejection && !requestInfo.orderingRepaired && canRepairBody) {
|
|
2109
|
+
if (isOrderingRejection && !requestInfo.orderingRepaired && canRepairBody && resendAllowed(requestInfo)) {
|
|
1913
2110
|
const coord = /messages\.(\d+)/.exec(errorBody);
|
|
1914
2111
|
const { messages: fixedMessages, converted } = directiveOnlySystemMessages(
|
|
1915
2112
|
JSON.parse(body.toString('utf8')).messages ?? [],
|
|
@@ -1926,7 +2123,7 @@ async function forwardRequest(
|
|
|
1926
2123
|
);
|
|
1927
2124
|
}
|
|
1928
2125
|
}
|
|
1929
|
-
if (isSignatureRejection && !requestInfo.thinkingStripped && canRepairBody) {
|
|
2126
|
+
if (isSignatureRejection && !requestInfo.thinkingStripped && canRepairBody && resendAllowed(requestInfo)) {
|
|
1930
2127
|
console.log(`[Maxpool] Anthropic rejected a block: ${describeRejectedBlock(body, errorBody)}`);
|
|
1931
2128
|
const { body: cleanBody, removed, converted } = stripForeignThinkingBlocks(body);
|
|
1932
2129
|
if (cleanBody) {
|
|
@@ -1951,7 +2148,7 @@ async function forwardRequest(
|
|
|
1951
2148
|
// model. Its own flag, so it still fires on a request whose broad strip already ran.
|
|
1952
2149
|
// Same reasoning as the broad strip above: a repair SHRINKS the body, so the
|
|
1953
2150
|
// re-send limit must not bar it.
|
|
1954
|
-
if (isSignatureRejection && !requestInfo.rejectedBlockStripped && canRepairBody) {
|
|
2151
|
+
if (isSignatureRejection && !requestInfo.rejectedBlockStripped && canRepairBody && resendAllowed(requestInfo)) {
|
|
1955
2152
|
const { body: fixedBody, removed, type } = stripRejectedBlockClass(body, errorBody);
|
|
1956
2153
|
if (fixedBody) {
|
|
1957
2154
|
// Latch the session ONLY for the class the pre-strip can actually repair up
|
|
@@ -1998,7 +2195,8 @@ async function forwardRequest(
|
|
|
1998
2195
|
|| /mcp_tool_(use|result)/.test(errorBody)
|
|
1999
2196
|
&& /(does not match any of the expected tags|without a corresponding)/i.test(errorBody);
|
|
2000
2197
|
if (anthropicIncompat && requestInfo.sessionKey && providerAvailable
|
|
2001
|
-
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2198
|
+
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2199
|
+
&& resendAllowed(requestInfo)) {
|
|
2002
2200
|
accountManager.markSessionIncompatible?.(requestInfo.sessionKey, requestInfo.homeProvider);
|
|
2003
2201
|
excludedIndexes.add(account.index);
|
|
2004
2202
|
console.log(`[Maxpool] Anthropic rejected this transcript (server_tool_use/thinking); pinning session to GLM/Kimi and retrying`);
|
|
@@ -2020,7 +2218,8 @@ async function forwardRequest(
|
|
|
2020
2218
|
// Claude account exists to serve it; with none, the 400 surfaces (nothing bigger).
|
|
2021
2219
|
const claudeAvailable = accountManager.accounts?.some(a => a.type !== 'provider' && a.enabled !== false);
|
|
2022
2220
|
if (providerTooSmall && requestInfo.sessionKey && claudeAvailable
|
|
2023
|
-
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2221
|
+
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2222
|
+
&& resendAllowed(requestInfo)) {
|
|
2024
2223
|
accountManager.markSessionLargeContext?.(requestInfo.sessionKey);
|
|
2025
2224
|
// Bench EVERY provider: today's GLM + Kimi coding legs both cap at ~256K, so once
|
|
2026
2225
|
// one 400s on size the others can't hold it either. If a genuine 1M-context
|
|
@@ -2043,7 +2242,8 @@ async function forwardRequest(
|
|
|
2043
2242
|
// latch the session — the fault is one request's shape, not a durable property of
|
|
2044
2243
|
// the conversation, and latching would evict a session from GLM on a single blip.
|
|
2045
2244
|
if (providerRejectedShape && claudeAvailable
|
|
2046
|
-
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2245
|
+
&& canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2246
|
+
&& resendAllowed(requestInfo)) {
|
|
2047
2247
|
for (const a of (accountManager.accounts || [])) {
|
|
2048
2248
|
if (a.type === 'provider') excludedIndexes.add(a.index);
|
|
2049
2249
|
}
|
|
@@ -2185,21 +2385,51 @@ async function forwardRequest(
|
|
|
2185
2385
|
} else { // Bound the non-streaming body read so a mid-body upstream stall can't hang
|
|
2186
2386
|
// the request forever and leak the lease (same class as the streaming idle
|
|
2187
2387
|
// guard). On timeout → UPSTREAM_BODY → the caller frees the lease.
|
|
2188
|
-
|
|
2388
|
+
// F4: read through a reader we OWN so an abandoned read can really be cancelled — arrayBuffer() holds
|
|
2389
|
+
// the stream's lock, which made `body.cancel()` a no-op and left the orphaned read (and its socket) running.
|
|
2390
|
+
const bodyReader = upstreamRes.body?.getReader?.() ?? null;
|
|
2391
|
+
const bodyP = bodyReader
|
|
2392
|
+
? (async () => {
|
|
2393
|
+
const chunks = [];
|
|
2394
|
+
for (;;) {
|
|
2395
|
+
const { done, value } = await bodyReader.read();
|
|
2396
|
+
if (done) return Buffer.concat(chunks);
|
|
2397
|
+
chunks.push(Buffer.from(value));
|
|
2398
|
+
}
|
|
2399
|
+
})()
|
|
2400
|
+
: upstreamRes.arrayBuffer();
|
|
2189
2401
|
bodyP.catch(() => {});
|
|
2190
2402
|
let bodyTimer;
|
|
2403
|
+
// B3: bounded by min(UPSTREAM_BODY_MS, remaining) when the request carries a deadline.
|
|
2404
|
+
const bodyWait = nonStreamBodyWait(requestInfo, UPSTREAM_BODY_MS);
|
|
2191
2405
|
const bodyTimeout = new Promise((_, reject) => {
|
|
2192
2406
|
bodyTimer = setTimeout(
|
|
2193
|
-
() => reject(Object.assign(new Error('upstream body timeout'), { code: 'UPSTREAM_BODY' })),
|
|
2194
|
-
|
|
2407
|
+
() => reject(Object.assign(new Error('upstream body timeout'), { code: bodyWait.deadlineBound ? 'UPSTREAM_DEADLINE' : 'UPSTREAM_BODY' })),
|
|
2408
|
+
bodyWait.ms,
|
|
2195
2409
|
);
|
|
2196
2410
|
bodyTimer.unref?.();
|
|
2197
2411
|
});
|
|
2412
|
+
// B10: the pre-response close listener was dropped when the headers arrived, so during this
|
|
2413
|
+
// body read nobody watches the client. Without this, a client that left mid-read still gets
|
|
2414
|
+
// a second upstream call when the read fails (the move path re-sends for a dead socket).
|
|
2415
|
+
let onBodyClientClose;
|
|
2416
|
+
const clientLeft = new Promise((_, reject) => {
|
|
2417
|
+
const gone = () => reject(Object.assign(new Error('client closed during body read'), { code: 'CLIENT_GONE_BODY' }));
|
|
2418
|
+
if (res.destroyed || res.closed) { gone(); return; }
|
|
2419
|
+
onBodyClientClose = gone;
|
|
2420
|
+
res.once('close', onBodyClientClose);
|
|
2421
|
+
});
|
|
2422
|
+
clientLeft.catch(() => {});
|
|
2198
2423
|
let arr;
|
|
2199
2424
|
try {
|
|
2200
|
-
arr = await Promise.race([bodyP, bodyTimeout]);
|
|
2425
|
+
arr = await Promise.race([bodyP, bodyTimeout, clientLeft]);
|
|
2426
|
+
} catch (e) {
|
|
2427
|
+
// Client gone / deadline / stalled body: stop the orphaned read and close the upstream socket now.
|
|
2428
|
+
try { await (bodyReader ? bodyReader.cancel() : upstreamRes.body?.cancel()); } catch { /* already closed */ }
|
|
2429
|
+
throw e;
|
|
2201
2430
|
} finally {
|
|
2202
2431
|
clearTimeout(bodyTimer);
|
|
2432
|
+
if (onBodyClientClose) res.off('close', onBodyClientClose);
|
|
2203
2433
|
}
|
|
2204
2434
|
const buf = Buffer.from(arr);
|
|
2205
2435
|
extractUsageFromBody(buf, account.index, accountManager, requestInfo);
|
|
@@ -2279,6 +2509,32 @@ async function forwardRequest(
|
|
|
2279
2509
|
releaseOnClientGone();
|
|
2280
2510
|
return;
|
|
2281
2511
|
}
|
|
2512
|
+
// B10: the client left during the non-streaming BODY read. Not an account fault, nothing to move,
|
|
2513
|
+
// nothing to write to: release neutral and stop (the move path below would fetch on a dead socket).
|
|
2514
|
+
if (err?.code === 'CLIENT_GONE_BODY') {
|
|
2515
|
+
const heldMs = Date.now() - (requestInfo.startedAt || Date.now());
|
|
2516
|
+
console.log(`[Maxpool] Client left after ${(heldMs / 1000).toFixed(1)}s on "${account.name}" (during the response body read) — released, not retried`);
|
|
2517
|
+
logFailedAttempt(account.name, 'client_cancelled', `client left after ${(heldMs / 1000).toFixed(1)}s (body read)`, 'released-cancel');
|
|
2518
|
+
accountManager.relinquishUpstreamProbe?.(lease);
|
|
2519
|
+
accountManager.releaseAccount(lease, { neutral: true, error: 'client_cancelled', kind: 'client_cancelled' });
|
|
2520
|
+
clearQueueHeartbeat(requestInfo);
|
|
2521
|
+
accountManager.removeQueuedRequest?.(requestInfo);
|
|
2522
|
+
return;
|
|
2523
|
+
}
|
|
2524
|
+
// B12: the wait was cut by the CLIENT'S deadline — neither an account fault (no markTransientFailure
|
|
2525
|
+
// streak) nor a link fault. Release neutral and answer a retryable 504; nothing is moved or re-sent.
|
|
2526
|
+
if (err.code === 'UPSTREAM_DEADLINE') {
|
|
2527
|
+
accountManager.relinquishUpstreamProbe?.(lease);
|
|
2528
|
+
accountManager.releaseAccount(lease, { neutral: true, error: 'deadline', kind: 'deadline' });
|
|
2529
|
+
ctx.status = 504;
|
|
2530
|
+
logFailedAttempt(account.name, 'deadline', `no answer after ${heldSeconds(lease)}s (client patience), non-streaming`, 'surfaced');
|
|
2531
|
+
if (logDir) {
|
|
2532
|
+
logSections.push(`=== DEADLINE ===\n${err.message}`);
|
|
2533
|
+
writeRequestLog(logDir, reqId, logSections);
|
|
2534
|
+
}
|
|
2535
|
+
sendDeadlineResponse(res, requestInfo);
|
|
2536
|
+
return;
|
|
2537
|
+
}
|
|
2282
2538
|
// undici reports every socket/DNS/TLS failure as the bare string "fetch failed" and
|
|
2283
2539
|
// hangs the REAL reason off err.cause. Logging err.message alone threw that away: 588
|
|
2284
2540
|
// "fetch failed" lines and ZERO ECONNRESET/ENOTFOUND/UND_ERR in the whole log, leaving
|
|
@@ -2297,7 +2553,7 @@ async function forwardRequest(
|
|
|
2297
2553
|
// would have seen before this feature existed.
|
|
2298
2554
|
// !clientHasRealBytes, not !headersSent: a slow turn whose keep-alive committed
|
|
2299
2555
|
// (pings only) is just as reroutable as a fast one (review r2 B).
|
|
2300
|
-
if (err.code === 'REFUSAL_RETRY' && canRetryBufferedBody && !clientHasRealBytes(res, requestInfo)) {
|
|
2556
|
+
if (err.code === 'REFUSAL_RETRY' && canRetryBufferedBody && !clientHasRealBytes(res, requestInfo) && resendAllowed(requestInfo)) {
|
|
2301
2557
|
const category = err.refusalCategory || 'unknown';
|
|
2302
2558
|
const anthropicIndexes = accountManager.accounts
|
|
2303
2559
|
.filter(a => a.type !== 'provider')
|
|
@@ -2388,6 +2644,15 @@ async function forwardRequest(
|
|
|
2388
2644
|
{ attempt: [account.name, transientKind, 'transient; no other route'] },
|
|
2389
2645
|
);
|
|
2390
2646
|
if (queued) return;
|
|
2647
|
+
// F3: a request with a deadline that ran out of routes on a first-byte timeout answers the request-scoped
|
|
2648
|
+
// 504 (the Mac books it neutral), never a bare 503 that cools its row and counts a failure. Direct
|
|
2649
|
+
// long-patience clients get the same answer (R2-2); an old Mac never has a deadline here (F1 gate).
|
|
2650
|
+
if (err.code === 'UPSTREAM_TTFB' && !requestInfo.stream && requestInfo.deadlineAt != null) {
|
|
2651
|
+
ctx.status = 504;
|
|
2652
|
+
logFailedAttempt(account.name, 'deadline', `no first byte after ${heldSeconds(lease)}s (hop cap), no route left`, 'surfaced');
|
|
2653
|
+
sendDeadlineResponse(res, requestInfo);
|
|
2654
|
+
return;
|
|
2655
|
+
}
|
|
2391
2656
|
ctx.status = 503;
|
|
2392
2657
|
// Record the user-facing failure explicitly so it's greppable in the event
|
|
2393
2658
|
// log (the in-memory TUI feed scrolls away). `err` is the last network error.
|
|
@@ -2412,7 +2677,8 @@ async function forwardRequest(
|
|
|
2412
2677
|
}
|
|
2413
2678
|
|
|
2414
2679
|
accountManager.releaseAccount(lease, { error: err.message });
|
|
2415
|
-
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2680
|
+
if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
|
|
2681
|
+
&& hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
2416
2682
|
account.status = 'error';
|
|
2417
2683
|
excludedIndexes.add(account.index);
|
|
2418
2684
|
logFailedAttempt(account.name, (err.code || err.message || 'error').toString().slice(0, 40), err.message, 'moved');
|
|
@@ -2475,6 +2741,9 @@ function noteScaledTtfb(requestInfo, accountName, path, waitMs, floorMs, ceiling
|
|
|
2475
2741
|
* second wait can still end inside the client's patience. Direct path only.
|
|
2476
2742
|
*/
|
|
2477
2743
|
function retrySameEligible(account, requestInfo, res, queueConfig = {}) {
|
|
2744
|
+
// B5: a non-streaming request is never retried on the same account — the second wait is a second
|
|
2745
|
+
// full hop of silence behind a client that cannot be told we are still working.
|
|
2746
|
+
if (!requestInfo.stream) return false;
|
|
2478
2747
|
if (requestInfo.sameAccountRetried) return false;
|
|
2479
2748
|
if ((requestInfo.bodyBytes || 0) < BIG_PROMPT_BYTES) return false;
|
|
2480
2749
|
if (!account || account.status === 'throttled' || account.status === 'error'
|
|
@@ -2525,10 +2794,33 @@ function isProviderAuthStatus(status) {
|
|
|
2525
2794
|
return status === 401 || status === 403;
|
|
2526
2795
|
}
|
|
2527
2796
|
|
|
2797
|
+
/** F5: the re-send guard shared by EVERY site that sends a request upstream a second time (account moves
|
|
2798
|
+
* AND body-repair / pin re-sends, which need no route check because they are not a fresh placement):
|
|
2799
|
+
* B10 — never re-send for a client that left; B6 — a non-streaming request with under the minimum
|
|
2800
|
+
* time left is not re-sent (a fresh hop cannot finish). */
|
|
2801
|
+
function resendAllowed(requestInfo = {}) {
|
|
2802
|
+
if (clientAlreadyGone(requestInfo)) {
|
|
2803
|
+
if (!requestInfo._clientGoneLogged) {
|
|
2804
|
+
requestInfo._clientGoneLogged = true;
|
|
2805
|
+
console.log('[Maxpool] Client already left — not re-sending to another account');
|
|
2806
|
+
logFailedAttempt('-', 'client_cancelled', 'client left before a re-send', 'released-cancel');
|
|
2807
|
+
}
|
|
2808
|
+
return false;
|
|
2809
|
+
}
|
|
2810
|
+
return nonStreamRemainingMs(requestInfo) >= deadlineKnobs().NONSTREAM_MIN_MOVE_MS;
|
|
2811
|
+
}
|
|
2812
|
+
|
|
2528
2813
|
function hasEligibleRoute(accountManager, requestInfo = {}, excludedIndexes = new Set()) {
|
|
2814
|
+
if (!resendAllowed(requestInfo)) return false;
|
|
2529
2815
|
return accountManager.hasAvailableRoute?.(requestInfo, excludedIndexes) || false;
|
|
2530
2816
|
}
|
|
2531
2817
|
|
|
2818
|
+
/** B10: has the client socket of this request closed? (requestInfo carries its res; set at forwardRequest entry.) */
|
|
2819
|
+
function clientAlreadyGone(requestInfo = {}) {
|
|
2820
|
+
const r = requestInfo._clientRes;
|
|
2821
|
+
return Boolean(r && (r.destroyed || r.closed));
|
|
2822
|
+
}
|
|
2823
|
+
|
|
2532
2824
|
function formatRetryDuration(seconds) {
|
|
2533
2825
|
const s = Math.max(0, Math.round(Number(seconds) || 0));
|
|
2534
2826
|
if (s >= 86400) return `${Math.round(s / 86400)}d`;
|
|
@@ -2738,7 +3030,7 @@ function isContextLengthError(errorBody) {
|
|
|
2738
3030
|
return /exceeded model token limit|maximum context length|context length exceeded|context window (?:size )?(?:exceeded|too)|prompt is too long|input is too long|reduce the length of|too many (?:input )?tokens|request too large/i.test(errorBody);
|
|
2739
3031
|
}
|
|
2740
3032
|
|
|
2741
|
-
export const __serverTest = { directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
3033
|
+
export const __serverTest = { hasEligibleRoute, resendAllowed, directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, parsePatience, computeDeadline, computeLinkHop, nonStreamFirstByteWait, nonStreamBodyWait, nonStreamRemainingMs, deadlineKnobs, deadlineErrorPayload, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
|
|
2742
3034
|
|
|
2743
3035
|
async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
|
|
2744
3036
|
if (!upstreamRes.body) return '';
|
|
@@ -3482,7 +3774,7 @@ async function queueAndRetryInner(
|
|
|
3482
3774
|
onGiveUp = null, onHold = null,
|
|
3483
3775
|
) {
|
|
3484
3776
|
const giveUp = (msg) => { onGiveUp?.(); return finishQueuedStreamIfNeeded(res, requestInfo, msg); };
|
|
3485
|
-
if (!queueConfig.enabled || !canQueueBufferedBody || (res.headersSent && !requestInfo.queueHeartbeatActive) || res.destroyed) {
|
|
3777
|
+
if (!queueConfig.enabled || !canQueueBufferedBody || (res.headersSent && !requestInfo.queueHeartbeatActive) || res.destroyed || res.closed) {
|
|
3486
3778
|
if (queueConfig.enabled && !canQueueBufferedBody && requestInfo.queueBlockedReason) {
|
|
3487
3779
|
console.log(`[Maxpool] Not queueing request: ${requestInfo.queueBlockedReason}`);
|
|
3488
3780
|
}
|
|
@@ -3577,7 +3869,7 @@ async function queueAndRetryInner(
|
|
|
3577
3869
|
const streamClientToleranceMs = Number.isFinite(requestInfo.clientToleranceMs)
|
|
3578
3870
|
? requestInfo.clientToleranceMs
|
|
3579
3871
|
: Math.max(0, Number(queueConfig.streamClientToleranceMs) || 0);
|
|
3580
|
-
|
|
3872
|
+
let queueWindowMs = computeQueueWindowMs({
|
|
3581
3873
|
cause,
|
|
3582
3874
|
stream: Boolean(requestInfo.stream),
|
|
3583
3875
|
retryPlanCause: retryPlan.cause,
|
|
@@ -3590,6 +3882,13 @@ async function queueAndRetryInner(
|
|
|
3590
3882
|
countTokensMaxWaitMs,
|
|
3591
3883
|
});
|
|
3592
3884
|
|
|
3885
|
+
// B6: a non-streaming hold is capped at the time the client still has; none left = give up now.
|
|
3886
|
+
const deadlineRemainingMs = nonStreamRemainingMs(requestInfo);
|
|
3887
|
+
if (deadlineRemainingMs !== Infinity) {
|
|
3888
|
+
if (deadlineRemainingMs <= 0) return giveUp(honestMessage);
|
|
3889
|
+
queueWindowMs = Math.min(queueWindowMs, deadlineRemainingMs);
|
|
3890
|
+
}
|
|
3891
|
+
|
|
3593
3892
|
if (queueWindowMs <= 0) return giveUp(honestMessage);
|
|
3594
3893
|
|
|
3595
3894
|
// HOLD-vs-ERROR oracle (from nextRetryForRequest): HOLD only when a TEMPORARY
|
|
@@ -4301,6 +4600,10 @@ function startIdleRequestReaper(res, reqId, idleMs, { now = Date.now, setInterva
|
|
|
4301
4600
|
if (getRequestInfo?.()?.queueHeartbeatActive) { lastProgressAt = now(); return; }
|
|
4302
4601
|
const bytes = res.socket?.bytesWritten ?? lastBytes;
|
|
4303
4602
|
if (bytes !== lastBytes) { lastBytes = bytes; lastProgressAt = now(); return; }
|
|
4603
|
+
// R2-1: a non-streaming request with a deadline is bounded by that deadline (504 at
|
|
4604
|
+
// deadlineAt); never cut it before deadline + grace. Never SHORTENS the ceiling.
|
|
4605
|
+
const info = getRequestInfo?.();
|
|
4606
|
+
if (info && !info.stream && info.deadlineAt != null && now() < info.deadlineAt + DEADLINE_REAP_GRACE_MS) return;
|
|
4304
4607
|
if (now() - lastProgressAt >= idleMs && !res.writableEnded && !res.destroyed) {
|
|
4305
4608
|
console.error(`[Maxpool] Request ${reqId} — no write progress for ${Math.round(idleMs / 1000)}s (backstop reaper); force-aborting a stuck request to free its account slot`);
|
|
4306
4609
|
res.destroy();
|
|
@@ -4358,9 +4661,29 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4358
4661
|
// upstream headers and the first body chunk can still fail over (clientHasRealBytes
|
|
4359
4662
|
// stays false) and the client's idle watchdog stays fed. realWrite stops it right
|
|
4360
4663
|
// before the first real write, so no ping can land between live SSE events (Bug A).
|
|
4664
|
+
// B9: a SCRUBBED provider stream drops every thinking event, so a long GLM think leaves the client
|
|
4665
|
+
// (and a linked Mac's idle timer) with nothing for tens of seconds — Claude Code reads that as a
|
|
4666
|
+
// dead stream. Once the first real byte is out, and only while a scrubber is active, keep a
|
|
4667
|
+
// keep-alive ping flowing. Safe mid-stream: the scrubber hands realWrite WHOLE events only
|
|
4668
|
+
// (provider-thinking-scrub.js push/event), so a ping can only land between events. Never armed
|
|
4669
|
+
// before the first real write (the model-echo/refusal holds `continue` ahead of realWrite, and a
|
|
4670
|
+
// ping before writeHead would commit default headers).
|
|
4671
|
+
let scrubPingTimer = null;
|
|
4672
|
+
let scrubPingStopped = false;
|
|
4673
|
+
const armScrubPing = () => {
|
|
4674
|
+
if (!scrubber || scrubPingTimer || scrubPingStopped || res.destroyed) return;
|
|
4675
|
+
const ms = Math.max(100, Number(process.env.MAXPOOL_SCRUB_PING_MS) || 3000);
|
|
4676
|
+
scrubPingTimer = setInterval(() => {
|
|
4677
|
+
if (res.destroyed || res.writableEnded) return;
|
|
4678
|
+
try { res.write('event: ping\ndata: {"type":"ping"}\n\n'); } catch { /* client gone */ }
|
|
4679
|
+
}, ms);
|
|
4680
|
+
scrubPingTimer.unref?.();
|
|
4681
|
+
};
|
|
4361
4682
|
const realWrite = (chunk) => {
|
|
4362
4683
|
if (requestInfo.queueHeartbeatActive) clearQueueHeartbeat(requestInfo);
|
|
4363
|
-
|
|
4684
|
+
const ok = res.write(chunk);
|
|
4685
|
+
armScrubPing();
|
|
4686
|
+
return ok;
|
|
4364
4687
|
};
|
|
4365
4688
|
|
|
4366
4689
|
// Client left mid-stream: the caller drops its disconnect-abort once headers
|
|
@@ -4573,6 +4896,9 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
4573
4896
|
if (err?.code === 'REFUSAL_RETRY') refusalRerouting = true;
|
|
4574
4897
|
throw err;
|
|
4575
4898
|
} finally {
|
|
4899
|
+
// B9: stop the keep-alive before anything else in the teardown, and never re-arm it.
|
|
4900
|
+
scrubPingStopped = true;
|
|
4901
|
+
if (scrubPingTimer) { clearInterval(scrubPingTimer); scrubPingTimer = null; }
|
|
4576
4902
|
// SAFEGUARD-REFUSAL HOLD last-resort flush. A stream can end while the hold is
|
|
4577
4903
|
// still undecided — a `usage`-only stream (message_delta with no content blocks
|
|
4578
4904
|
// and no message_stop) reaches `done` having never produced a decisive event, and
|