maxpool 1.24.91 → 1.24.92

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "maxpool",
3
- "version": "1.24.91",
3
+ "version": "1.24.92",
4
4
  "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
package/src/index.js CHANGED
@@ -42,7 +42,7 @@ net.setDefaultAutoSelectFamilyAttemptTimeout(
42
42
  import { Prober } from './prober.js';
43
43
  import { CapacityLedger } from './capacity-ledger.js';
44
44
  import { loginOAuth, fetchProfile, refreshAccessToken, isTokenExpiringSoon, tokenFingerprint, isLoginCancelled } from './oauth.js';
45
- import { TUI, writeTerminalSync, reclaimRawMode } from './tui.js';
45
+ import { TUI, writeTerminalSync, reclaimRawModeLogged } from './tui.js';
46
46
  import { RestartController } from './restart-controller.js';
47
47
  import { resolveAccounts } from './account-config.js';
48
48
  import { maybeCheckForUpdate, getCurrentVersion, markApplied, clearQuarantine, captureBootVersion } from './updater.js';
@@ -51,7 +51,7 @@ import {
51
51
  RELOAD_SWAPPED, RELOAD_ROLLED_BACK,
52
52
  MSG_LISTEN, MSG_RELEASE, MSG_TAKEOVER, MSG_PROBE_READY,
53
53
  MSG_RELOAD_REQUEST, MSG_READY, MSG_FAILED, MSG_RELEASED, MSG_PRIMARY, MSG_ROLLED_BACK,
54
- MSG_TTY_REASSERT, MSG_CONN,
54
+ MSG_TTY_REASSERT, MSG_CONN, describeOldWorkerExit,
55
55
  } from './reload-protocol.js';
56
56
 
57
57
  const args = process.argv.slice(2);
@@ -521,18 +521,25 @@ function reapOldWorker(worker, onExited = () => {}) {
521
521
  // SIGKILL only ever reaps a genuinely wedged worker, never cuts a live stream.
522
522
  const cap = RELOAD_DRAIN_MS + 30_000;
523
523
  let reaped = false;
524
+ let killed = false;
525
+ const swappedAt = Date.now();
524
526
  // `onExited` fires on BOTH paths (clean exit and SIGKILL): the terminal is clobbered
525
527
  // by the exit itself, however that exit came about.
526
528
  const finish = () => {
527
529
  if (reaped) return;
528
530
  reaped = true;
529
531
  clearTimeout(timer);
532
+ // B11: the update-window record — when the old worker actually exited (its exit resets the shared terminal).
533
+ // Event log ONLY (F6): the supervisor shares the terminal with the new worker's dashboard and is not
534
+ // stdout-muted, so a console line here would paint over the dashboard in the very window it measures.
535
+ try { appendEventLog(`[Maxpool] ${describeOldWorkerExit({ swappedAt, exitedAt: Date.now(), killed })}`); } catch { /* never break reaping */ }
530
536
  try { onExited(); } catch { /* never let the callback break reaping */ }
531
537
  };
532
538
  const timer = setTimeout(() => {
533
539
  if (reaped) return;
534
540
  console.error(`[Maxpool] Old worker outlived ${Math.ceil(cap / 1000)}s reload-drain cap; SIGKILL.`);
535
541
  try { worker.child.kill('SIGKILL'); } catch { /* ignore */ }
542
+ killed = true;
536
543
  finish();
537
544
  }, cap);
538
545
  timer.unref?.();
@@ -1648,7 +1655,7 @@ async function serverWorkerCommand() {
1648
1655
  // The old worker has now EXITED, and its exit reset the shared terminal out
1649
1656
  // of raw mode (libuv's uv_tty_reset_mode on process exit). Reclaim it —
1650
1657
  // without blocking on a terminal that is behind (reclaimRawMode).
1651
- await reclaimRawMode({ flush: tui?._nb ? o => tui.flushOutput(o) : null });
1658
+ await reclaimRawModeLogged({ flush: tui?._nb ? o => tui.flushOutput(o) : null });
1652
1659
  // Re-assert the screen state too: the dying worker's alt-screen exit and
1653
1660
  // cursor-show can land after ours.
1654
1661
  if (tui?.running && process.stdout.isTTY) {
@@ -135,3 +135,9 @@ export async function runReloadBaton({
135
135
  // in-process can detect it; the TUI keeps rendering while the kernel line-buffers, so
136
136
  // every keystroke (r / q / p) is swallowed. Measured on a real pty 2026-08-07.
137
137
  export const MSG_TTY_REASSERT = 'tty-reassert';
138
+
139
+ /** One log line for the update-window record (release B, B11): when the old worker exited and how. */
140
+ export function describeOldWorkerExit({ swappedAt, exitedAt, killed = false }) {
141
+ const after = ((exitedAt - swappedAt) / 1000).toFixed(1);
142
+ return `reload: old worker exited at ${new Date(exitedAt).toISOString()} (${after}s after swap, ${killed ? 'SIGKILL after the drain cap' : 'clean exit'})`;
143
+ }
package/src/server.js CHANGED
@@ -66,6 +66,120 @@ const isNetworkCode = c => Boolean(c) && NETWORK_ERROR_CODES.has(c);
66
66
  const NETWORK_SOAK_NONSTREAM_MS = Math.max(5_000,
67
67
  Number(process.env.MAXPOOL_NETWORK_SOAK_NONSTREAM_MS) || 45_000);
68
68
  const QUEUE_KEEPALIVE = 'event: ping\ndata: {}\n\n';
69
+
70
+ // PER-REQUEST DEADLINE, NON-STREAMING (release B, 2026-11-13). A non-streaming request has
71
+ // no keep-alive, so the CLIENT's own timer decides when it is abandoned. maxpool used to
72
+ // wait a flat 120s per hop and chain hops without limit, serving into sockets the client had
73
+ // already closed. When the client tells us how long it will wait (X-Stainless-Timeout, or the
74
+ // pool header from the Mac), every wait is bounded by ONE deadline anchored at arrival.
75
+ // A request with no usable patience header gets NO deadline: today's behaviour, never a new cut-off.
76
+ const NONSTREAM_HOP_CAP_MS = 290_000; // undici's own headers/body timeout is 300s: nothing waits past this
77
+ const NONSTREAM_FLOOR_PATIENCE_MS = 15_000; // usable patience is never below 15s once the margin is taken
78
+ const MAX_PATIENCE_MS = 3_600_000;
79
+ const posMsFromEnv = (name, dflt) => {
80
+ const n = Number(process.env[name]);
81
+ return Number.isFinite(n) && n > 0 ? n : dflt;
82
+ };
83
+ /** Live knob values (env read per call so a two-pool test can inject a small cap). */
84
+ function deadlineKnobs() {
85
+ return {
86
+ NONSTREAM_MARGIN_MS: posMsFromEnv('MAXPOOL_NONSTREAM_MARGIN_MS', 15_000), // client header: leave this much before its timer
87
+ POOL_TRANSIT_MARGIN_MS: 3_000, // pool header: the Mac already took its margin
88
+ NONSTREAM_MIN_MOVE_MS: posMsFromEnv('MAXPOOL_NONSTREAM_MIN_MOVE_MS', 60_000), // below this, never re-send to another account
89
+ LINK_RETURN_MARGIN_MS: posMsFromEnv('MAXPOOL_LINK_RETURN_MARGIN_MS', 10_000), // G: the cloud's answer must travel back inside the Mac's wait
90
+ LINK_HOP_CAP_MS: Math.min(posMsFromEnv('MAXPOOL_LINK_HOP_CAP_MS', NONSTREAM_HOP_CAP_MS), NONSTREAM_HOP_CAP_MS),
91
+ UPSTREAM_BODY_MS,
92
+ };
93
+ }
94
+ // A plain positive decimal. `Number('')` is 0, `Number('abc')` is NaN, `Number('Infinity')`
95
+ // is Infinity and `setTimeout(NaN)` fires in 1ms: every one of them must mean "no header".
96
+ const PLAIN_POSITIVE_NUMBER = /^(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/;
97
+ function parsePositiveHeader(raw) {
98
+ if (raw == null) return null;
99
+ const s = String(raw).trim();
100
+ if (!PLAIN_POSITIVE_NUMBER.test(s)) return null;
101
+ const n = Number(s);
102
+ return Number.isFinite(n) && n > 0 ? n : null;
103
+ }
104
+ /** The client's patience: first VALID of the pool header (ms) → X-Stainless-Timeout (s) →
105
+ * x-stainless-read-timeout (s). null = unknown, i.e. no deadline. */
106
+ function parsePatience(headers = {}) {
107
+ const k = deadlineKnobs();
108
+ const pool = parsePositiveHeader(headerValue(headers, 'x-maxpool-deadline-ms'));
109
+ if (pool != null) return { ms: Math.min(pool, MAX_PATIENCE_MS), source: 'pool', poolSupplied: true };
110
+ for (const [name, source] of [['x-stainless-timeout', 'stainless-timeout'], ['x-stainless-read-timeout', 'stainless-read-timeout']]) {
111
+ const secs = parsePositiveHeader(headerValue(headers, name));
112
+ if (secs == null) continue;
113
+ const floor = k.NONSTREAM_MARGIN_MS + NONSTREAM_FLOOR_PATIENCE_MS;
114
+ return { ms: Math.min(Math.max(secs * 1000, floor), MAX_PATIENCE_MS), source, poolSupplied: false };
115
+ }
116
+ return null;
117
+ }
118
+ /** deadlineAt for a request that arrived at `arrivedAt`; null when the client gave no usable patience. */
119
+ function computeDeadline(headers, arrivedAt) {
120
+ const p = parsePatience(headers);
121
+ if (!p) return null;
122
+ // F1 (mixed version): a request FORWARDED BY ANOTHER POOL (hop header) that carries no pool deadline header
123
+ // comes from an OLD Mac (v1.24.91). It forwards the client's x-stainless-timeout, but an old Mac books any
124
+ // 5xx it passes through as an account failure, so this pool must behave exactly as before: no deadline.
125
+ // (A new Mac always sends the pool header on a non-streaming link hop.)
126
+ if (!p.poolSupplied && headerValue(headers, HOP_HEADER) !== '') return null;
127
+ const k = deadlineKnobs();
128
+ const usableMs = p.poolSupplied
129
+ ? Math.max(p.ms - k.POOL_TRANSIT_MARGIN_MS, 1_000)
130
+ : p.ms - k.NONSTREAM_MARGIN_MS;
131
+ return { deadlineAt: arrivedAt + usableMs, patienceMs: p.ms, source: p.source, poolSupplied: p.poolSupplied };
132
+ }
133
+ /** Time left on a non-streaming request's deadline. Infinity = no deadline (streaming, or a header-less client). */
134
+ function nonStreamRemainingMs(requestInfo = {}) {
135
+ if (requestInfo.stream || requestInfo.deadlineAt == null) return Infinity;
136
+ return requestInfo.deadlineAt - Date.now();
137
+ }
138
+ /** First-byte wait for one DIRECT hop. WITH a deadline: min(remaining, hop cap) — the flat/scaled
139
+ * first-byte floor (120s) does NOT apply, because the client has said it will wait longer and a
140
+ * floor timeout would abandon an answer that still fits (F3). WITHOUT a deadline: `waitMs`, today's wait.
141
+ * The cap is the 290s undici bound (injectable lower via MAXPOOL_LINK_HOP_CAP_MS for scaled tests). */
142
+ function nonStreamFirstByteWait(requestInfo, waitMs) {
143
+ const remaining = nonStreamRemainingMs(requestInfo);
144
+ if (remaining === Infinity) return { ms: waitMs, deadlineBound: false };
145
+ const cap = deadlineKnobs().LINK_HOP_CAP_MS;
146
+ return remaining <= cap ? { ms: Math.max(1, Math.floor(remaining)), deadlineBound: true } : { ms: cap, deadlineBound: false };
147
+ }
148
+ /** Non-streaming body-read wait: min(UPSTREAM_BODY_MS, remaining). */
149
+ function nonStreamBodyWait(requestInfo, bodyMs) {
150
+ const remaining = nonStreamRemainingMs(requestInfo);
151
+ if (remaining === Infinity) return { ms: bodyMs, deadlineBound: false };
152
+ return remaining <= bodyMs ? { ms: Math.max(1, Math.floor(remaining)), deadlineBound: true } : { ms: bodyMs, deadlineBound: false };
153
+ }
154
+ /** ONE NUMBER for a link hop: linkWait = min(remaining − G, cap) (never below min(15s, remaining)),
155
+ * header = linkWait − G. So header + G ≤ linkWait always: the cloud's whole hop chain ends
156
+ * inside this ONE Mac link wait. null = no usable time left (never arm a 0ms timer). */
157
+ function computeLinkHop(remainingMs, { capMs, marginMs } = {}) {
158
+ const k = deadlineKnobs();
159
+ const G = marginMs ?? k.LINK_RETURN_MARGIN_MS;
160
+ const cap = capMs ?? k.LINK_HOP_CAP_MS;
161
+ if (!Number.isFinite(remainingMs) || remainingMs <= 0) return null;
162
+ const linkWaitMs = Math.floor(Math.min(cap, Math.max(remainingMs - G, Math.min(15_000, remainingMs))));
163
+ const headerMs = linkWaitMs - G;
164
+ if (headerMs < 1_000) return null;
165
+ return { linkWaitMs, headerMs, deadlineBound: remainingMs - G <= cap };
166
+ }
167
+ /** The answer for a request whose client patience ran out: retryable by every SDK (5xx). */
168
+ function deadlineErrorPayload() {
169
+ return {
170
+ type: 'error',
171
+ error: {
172
+ type: 'timeout_error',
173
+ message: 'This request ran out of time before Claude answered. Sending the message again normally works.',
174
+ },
175
+ };
176
+ }
177
+ /** 504 for a deadline expiry. The pool-only marker rides ONLY when the request itself carried the pool
178
+ * deadline header (an old Mac / header-less request keeps today's answer and never sees it). */
179
+ function sendDeadlineResponse(res, requestInfo) {
180
+ const headers = requestInfo.deadlinePoolSupplied ? { 'x-maxpool-request-scoped': '1' } : {};
181
+ sendErrorResponse(res, requestInfo, 504, deadlineErrorPayload(), headers);
182
+ }
69
183
  // Account screen shell (served keyless at /maxpool/screen; the page itself prompts
70
184
  // for a pool key and reads the key-gated /maxpool/status). Read once at boot.
71
185
  const SCREEN_HTML = (() => {
@@ -98,12 +212,16 @@ const REFUSAL_HOLD_MAX_MS = 30_000;
98
212
  // disabling it. Short (not reset-length) so a provider-pinned session's hold window
99
213
  // isn't blown, and a still-exhausted provider just re-benches on the next single retry.
100
214
  const PROVIDER_FORBIDDEN_COOLDOWN_SEC = Math.max(60, Number(process.env.MAXPOOL_PROVIDER_403_COOLDOWN_SEC) || 1800);
101
- // Backstop reaper idle ceiling. Floored WELL above the longest a legit NON-streaming
102
- // request can go silent (no heartbeat there): TTFB 120s + nonStreamMaxWaitMs 300s +
103
- // UPSTREAM_BODY_MS 300s ≈ 720s — so a slow-but-alive non-streaming request is never
104
- // reaped at completion. Streaming holds heartbeat, so they're safe at any ceiling.
215
+ // Backstop reaper idle ceiling. Floored WELL above the longest a legit header-less
216
+ // NON-streaming request can go silent (no heartbeat there): TTFB 120s + nonStreamMaxWaitMs
217
+ // 300s + UPSTREAM_BODY_MS 300s ≈ 720s — so a slow-but-alive non-streaming request is never
218
+ // reaped at completion. A request with a client-patience deadline can legitimately stay
219
+ // silent longer (290s hops, up to ~1470s inside a 1h patience); the reaper extends to
220
+ // deadlineAt + DEADLINE_REAP_GRACE_MS for those, since the deadline itself ends them.
221
+ // Streaming holds heartbeat, so they're safe at any ceiling.
105
222
  // Exported so index.js sizes RELOAD_DRAIN_MS off the SAME value (no drift).
106
223
  export const REQUEST_IDLE_MAX_MS = Math.max(900_000, Number(process.env.MAXPOOL_REQUEST_IDLE_MAX_MS) || 1_200_000);
224
+ const DEADLINE_REAP_GRACE_MS = 60_000;
107
225
 
108
226
  const DEFAULT_QUEUE = {
109
227
  enabled: true,
@@ -250,6 +368,8 @@ export function createProxyServer(accountManager, config, hooks = {}) {
250
368
  requestTimeout: Math.max(1_000, Number(process.env.MAXPOOL_REQUEST_TIMEOUT_MS) || 300_000),
251
369
  connectionsCheckingInterval: Math.max(100, Number(process.env.MAXPOOL_CONNECTIONS_CHECK_MS) || 30_000),
252
370
  }, async (req, res) => {
371
+ // B1: the deadline is anchored HERE, at the request event, before auth and body buffering.
372
+ const arrivedAt = Date.now();
253
373
  try {
254
374
  // Disable Nagle on the client socket: flush each SSE event immediately instead
255
375
  // of coalescing small writes into bursts. Smooths streaming cadence toward the
@@ -503,6 +623,15 @@ export function createProxyServer(accountManager, config, hooks = {}) {
503
623
  // THIS account, from lease.startedAt.) Was never set, so every direct-path
504
624
  // client-gone line read 0.0s.
505
625
  requestInfo.startedAt = Date.now();
626
+ // B1: non-streaming only; a request with no usable patience header gets NO deadline.
627
+ if (!requestInfo.stream) {
628
+ const dl = computeDeadline(req.headers, arrivedAt);
629
+ if (dl) {
630
+ requestInfo.deadlineAt = dl.deadlineAt;
631
+ requestInfo.deadlineSource = dl.source;
632
+ requestInfo.deadlinePoolSupplied = dl.poolSupplied;
633
+ }
634
+ }
506
635
  const maxQueuedBodyBytes = queueConfig.maxQueuedBodyBytes == null
507
636
  ? Infinity
508
637
  : Math.max(0, Number(queueConfig.maxQueuedBodyBytes) || 0);
@@ -579,6 +708,10 @@ export function createProxyServer(accountManager, config, hooks = {}) {
579
708
  // request's heartbeat commits writes LONG before upstream answers.
580
709
  firstUpstreamByteAt: requestInfo?._firstUpstreamByteAt ?? null,
581
710
  upstreamForwardStartAt: requestInfo?._upstreamForwardStartAt ?? null,
711
+ // B8: makes the non-streaming share measurable. null = no request shape was ever parsed.
712
+ // F9: only the message-creating call (POST /v1/messages) has a streaming shape — count_tokens and GETs
713
+ // must not read as "non-streaming" and swamp the share.
714
+ stream: requestInfo?.method === 'POST' && /^\/v1\/messages(?:\?|$)/.test(String(req.url || '')) ? Boolean(requestInfo.stream) : null,
582
715
  });
583
716
  }
584
717
  } catch (err) {
@@ -808,6 +941,7 @@ async function forwardRequest(
808
941
  // waiter instead of preferentially jumping ahead of the FIFO for the rest of
809
942
  // this request's failover chain.
810
943
  requestInfo.queueAdmitted = false;
944
+ requestInfo._clientRes = res; // B10: lets the shared chokepoints see the client socket
811
945
 
812
946
  // Abort the upstream fetch and release the lease if the CLIENT disconnects during
813
947
  // the pre-response window (token refresh + connect + waiting for the upstream's
@@ -935,6 +1069,7 @@ async function forwardRequest(
935
1069
  // locally went live remotely). The remote pool owns its own accounts.
936
1070
  if (/^x-maxpool-(zai|kimi)-/.test(lk)) continue;
937
1071
  if (lk === 'x-maxpool-pin-account') continue; // M4: only this pool sets a pin
1072
+ if (lk === 'x-maxpool-deadline-ms') continue; // B4: ONE number — this hop computes its own below
938
1073
  linkHeaders[k] = v;
939
1074
  }
940
1075
  linkHeaders[HOP_HEADER] = String(hop);
@@ -966,6 +1101,21 @@ async function forwardRequest(
966
1101
  if (linkDone) return; linkDone = true;
967
1102
  accountManager.releaseAccount(lease, outcome);
968
1103
  };
1104
+ // B4 (non-streaming with a deadline): linkWait = min(remaining − G, 290s); the cloud is told
1105
+ // linkWait − G, so its whole hop chain ends inside this ONE wait. No time left → 504, no call.
1106
+ let linkHop = null;
1107
+ if (!requestInfo.stream && requestInfo.deadlineAt != null) {
1108
+ linkHop = computeLinkHop(nonStreamRemainingMs(requestInfo));
1109
+ if (!linkHop) {
1110
+ finishLink({ neutral: true, error: 'deadline', kind: 'deadline' });
1111
+ detachLinkClose();
1112
+ ctx.status = 504;
1113
+ logFailedAttempt(account.name, 'deadline', 'no time left before the client gives up', 'surfaced');
1114
+ sendDeadlineResponse(res, requestInfo);
1115
+ return;
1116
+ }
1117
+ linkHeaders['x-maxpool-deadline-ms'] = String(linkHop.headerMs);
1118
+ }
969
1119
  try {
970
1120
  // P4a: a blackholed link must not hang the request (and hold a lease) for
971
1121
  // minutes — bound time-to-headers like the direct path's TTFB guard. GLM
@@ -977,7 +1127,7 @@ async function forwardRequest(
977
1127
  // clears this timer for every HEALTHY link, so a scaled wait only ever ran against
978
1128
  // a dead/blackholed tunnel — stretching its detection from 120s to minutes. The
979
1129
  // big-prompt first-byte timeouts measured were all on the direct path.
980
- const linkTtfbMs = Math.max(15_000, Number(process.env.MAXPOOL_LINK_TTFB_MS) || UPSTREAM_TTFB_MS);
1130
+ const linkTtfbMs = linkHop ? linkHop.linkWaitMs : Math.max(15_000, Number(process.env.MAXPOOL_LINK_TTFB_MS) || UPSTREAM_TTFB_MS);
981
1131
  const linkCtl = new AbortController();
982
1132
  const linkTtfb = setTimeout(() => linkCtl.abort(), linkTtfbMs);
983
1133
  // P2: the link path had NO pre-headers keep-alive — a streaming client sat on a
@@ -1050,6 +1200,22 @@ async function forwardRequest(
1050
1200
  sendErrorResponse(res, requestInfo, upRes.status, { type: 'error', error: { type: 'overloaded_error', message: `Team pool account "${account.name}" is at capacity` } });
1051
1201
  return;
1052
1202
  }
1203
+ // B7: the cloud ran out of the deadline THIS request gave it (504 + request-scoped). It says
1204
+ // nothing about the cloud row or the link: release NEUTRAL (no cooldownUntil, no failed count,
1205
+ // no streak) and treat it like any other request-scoped miss — try another route only if time remains.
1206
+ if (account.remote && upRes.status === 504 && upRes.headers.get('x-maxpool-request-scoped') === '1' && !clientHasRealBytes(res, requestInfo)) {
1207
+ const errorText = await readErrorBody(upRes);
1208
+ finishLink({ neutral: true, error: 'deadline', kind: 'deadline' });
1209
+ excludedIndexes.add(account.index);
1210
+ if (retryCount + 1 < maxAttempts && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
1211
+ logFailedAttempt(account.name, 'remote-504', 'linked pool ran out of the request deadline', 'moved');
1212
+ detachLinkClose();
1213
+ return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, excludedIndexes);
1214
+ }
1215
+ logFailedAttempt(account.name, 'remote-504', 'linked pool ran out of the request deadline; no time to move', 'surfaced');
1216
+ sendErrorBody(res, requestInfo, 504, errorText, upRes.headers);
1217
+ return;
1218
+ }
1053
1219
  // A request released from the queue already holds a committed keep-alive stream.
1054
1220
  // An error on it is framed as an SSE error; a 2xx keeps the heartbeat until the
1055
1221
  // first real chunk (below).
@@ -1137,6 +1303,17 @@ async function forwardRequest(
1137
1303
  logFailedAttempt(account.name, 'client_cancelled', `client left after ${heldS}s, nothing sent`, 'released-cancel');
1138
1304
  return;
1139
1305
  }
1306
+ // B12: the wait was cut by the CLIENT'S deadline (not the per-hop cap, not a dead link): neither an
1307
+ // account fault nor a link fault — release neutral, and answer a retryable 504 (nothing left to move on).
1308
+ // (A no-headers AbortError that is not a client leaving can only be this hop's timer.)
1309
+ if (linkHop?.deadlineBound && err?.name === 'AbortError' && !linkClientGone && linkStatus == null && !clientHasRealBytes(res, requestInfo)) {
1310
+ finishLink({ neutral: true, error: 'deadline', kind: 'deadline' });
1311
+ detachLinkClose();
1312
+ ctx.status = 504;
1313
+ logFailedAttempt(account.name, 'deadline', `no answer after ${heldS}s (client patience), non-streaming`, 'surfaced');
1314
+ sendDeadlineResponse(res, requestInfo);
1315
+ return;
1316
+ }
1140
1317
  // Everything that reaches this catch is request-scoped (D4): a remote 429/503 is
1141
1318
  // handled — account-scoped — by its own branch above before anything can throw, so
1142
1319
  // only a TTFB abort, a mid-stream cancel or an unknown link error lands here. It
@@ -1239,10 +1416,27 @@ async function forwardRequest(
1239
1416
  // this fixed clock, so a long healthy stream is never cut.
1240
1417
  const ttfbController = new AbortController();
1241
1418
  const ttfbCeilingMs = clientWaitCeilingMs(requestInfo, queueConfig);
1242
- const ttfbMs = scaledTtfbMs(UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, requestInfo.bodyBytes, ttfbCeilingMs);
1419
+ let ttfbMs = scaledTtfbMs(UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, requestInfo.bodyBytes, ttfbCeilingMs);
1420
+ // B2/N-3: a non-streaming request with a deadline never waits past it (nor past 290s, undici's own
1421
+ // bound), and with no time left never reaches upstream at all (never arm a 0ms timer).
1422
+ let ttfbDeadlineBound = false;
1423
+ if (!requestInfo.stream && requestInfo.deadlineAt != null) {
1424
+ if (nonStreamRemainingMs(requestInfo) <= 0) {
1425
+ res.off('close', onClientClose);
1426
+ accountManager.relinquishUpstreamProbe?.(lease);
1427
+ accountManager.releaseAccount(lease, { neutral: true, error: 'deadline', kind: 'deadline' });
1428
+ ctx.status = 504;
1429
+ logFailedAttempt(account.name, 'deadline', 'no time left before the client gives up', 'surfaced');
1430
+ sendDeadlineResponse(res, requestInfo);
1431
+ return;
1432
+ }
1433
+ const w = nonStreamFirstByteWait(requestInfo, ttfbMs);
1434
+ ttfbMs = w.ms;
1435
+ ttfbDeadlineBound = w.deadlineBound;
1436
+ }
1243
1437
  noteScaledTtfb(requestInfo, account.name, 'direct', ttfbMs, UPSTREAM_TTFB_MS, ttfbCeilingMs);
1244
1438
  const ttfbTimer = setTimeout(
1245
- () => ttfbController.abort(Object.assign(new Error('upstream TTFB timeout'), { code: 'UPSTREAM_TTFB' })),
1439
+ () => ttfbController.abort(Object.assign(new Error('upstream TTFB timeout'), { code: ttfbDeadlineBound ? 'UPSTREAM_DEADLINE' : 'UPSTREAM_TTFB' })),
1246
1440
  ttfbMs,
1247
1441
  );
1248
1442
  ttfbTimer.unref?.();
@@ -1666,7 +1860,8 @@ async function forwardRequest(
1666
1860
  logSections.push(`=== RESPONSE ${upstreamRes.status} — "${account.name}" cooling down, failing over ===\n${formatHeaders(upstreamRes.headers)}`);
1667
1861
  }
1668
1862
 
1669
- if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
1863
+ if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
1864
+ && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
1670
1865
  logFailedAttempt(account.name, `upstream_${upstreamRes.status}`, 'retriable upstream status, cooling down', 'moved');
1671
1866
  return forwardRequest(
1672
1867
  req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
@@ -1825,7 +2020,8 @@ async function forwardRequest(
1825
2020
  : null;
1826
2021
  const issuerIsClaude = issuer && issuer.type !== 'provider';
1827
2022
  if (lease.migratedFromName && issuerIsClaude && requestInfo.sessionKey
1828
- && canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
2023
+ && canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
2024
+ && resendAllowed(requestInfo)) {
1829
2025
  accountManager.revertSessionBinding(requestInfo.sessionKey, lease.migratedFromName);
1830
2026
  excludedIndexes.add(account.index);
1831
2027
  console.log(`[Maxpool] Thinking-signature fail-safe: reverting session to issuer "${lease.migratedFromName}" and retrying`);
@@ -1841,7 +2037,8 @@ async function forwardRequest(
1841
2037
  // EFFORT REPAIR. A rejected output_config.effort is a hard failure of whatever the
1842
2038
  // client was doing (a web search, a tool call) — worth healing rather than surfacing.
1843
2039
  if (effortMode && !requestInfo.effortRepaired
1844
- && canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
2040
+ && canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
2041
+ && resendAllowed(requestInfo)) {
1845
2042
  const fix = repairEffort(body, effortMode, errorBody);
1846
2043
  if (fix.body) {
1847
2044
  // Latch it so LATER turns apply the working level up front: the client resends the
@@ -1885,7 +2082,7 @@ async function forwardRequest(
1885
2082
  const isDirectiveFormRejection = account.type !== 'provider'
1886
2083
  && upstreamRes.status === 400
1887
2084
  && /output_config[^:]*: *Extra inputs are not permitted/i.test(errorBody);
1888
- if (isDirectiveFormRejection && requestInfo.orderingRepaired && !requestInfo.orderingFallback && canRepairBody) {
2085
+ if (isDirectiveFormRejection && requestInfo.orderingRepaired && !requestInfo.orderingFallback && canRepairBody && resendAllowed(requestInfo)) {
1889
2086
  // Rewrite every directive-only system (the repair's own output) as a PLAIN
1890
2087
  // assistant turn carrying the directives text — ordinary history, accepted by
1891
2088
  // every validator, nothing dropped. A system ENDING the array stays legal
@@ -1909,7 +2106,7 @@ async function forwardRequest(
1909
2106
  );
1910
2107
  }
1911
2108
  }
1912
- if (isOrderingRejection && !requestInfo.orderingRepaired && canRepairBody) {
2109
+ if (isOrderingRejection && !requestInfo.orderingRepaired && canRepairBody && resendAllowed(requestInfo)) {
1913
2110
  const coord = /messages\.(\d+)/.exec(errorBody);
1914
2111
  const { messages: fixedMessages, converted } = directiveOnlySystemMessages(
1915
2112
  JSON.parse(body.toString('utf8')).messages ?? [],
@@ -1926,7 +2123,7 @@ async function forwardRequest(
1926
2123
  );
1927
2124
  }
1928
2125
  }
1929
- if (isSignatureRejection && !requestInfo.thinkingStripped && canRepairBody) {
2126
+ if (isSignatureRejection && !requestInfo.thinkingStripped && canRepairBody && resendAllowed(requestInfo)) {
1930
2127
  console.log(`[Maxpool] Anthropic rejected a block: ${describeRejectedBlock(body, errorBody)}`);
1931
2128
  const { body: cleanBody, removed, converted } = stripForeignThinkingBlocks(body);
1932
2129
  if (cleanBody) {
@@ -1951,7 +2148,7 @@ async function forwardRequest(
1951
2148
  // model. Its own flag, so it still fires on a request whose broad strip already ran.
1952
2149
  // Same reasoning as the broad strip above: a repair SHRINKS the body, so the
1953
2150
  // re-send limit must not bar it.
1954
- if (isSignatureRejection && !requestInfo.rejectedBlockStripped && canRepairBody) {
2151
+ if (isSignatureRejection && !requestInfo.rejectedBlockStripped && canRepairBody && resendAllowed(requestInfo)) {
1955
2152
  const { body: fixedBody, removed, type } = stripRejectedBlockClass(body, errorBody);
1956
2153
  if (fixedBody) {
1957
2154
  // Latch the session ONLY for the class the pre-strip can actually repair up
@@ -1998,7 +2195,8 @@ async function forwardRequest(
1998
2195
  || /mcp_tool_(use|result)/.test(errorBody)
1999
2196
  && /(does not match any of the expected tags|without a corresponding)/i.test(errorBody);
2000
2197
  if (anthropicIncompat && requestInfo.sessionKey && providerAvailable
2001
- && canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
2198
+ && canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
2199
+ && resendAllowed(requestInfo)) {
2002
2200
  accountManager.markSessionIncompatible?.(requestInfo.sessionKey, requestInfo.homeProvider);
2003
2201
  excludedIndexes.add(account.index);
2004
2202
  console.log(`[Maxpool] Anthropic rejected this transcript (server_tool_use/thinking); pinning session to GLM/Kimi and retrying`);
@@ -2020,7 +2218,8 @@ async function forwardRequest(
2020
2218
  // Claude account exists to serve it; with none, the 400 surfaces (nothing bigger).
2021
2219
  const claudeAvailable = accountManager.accounts?.some(a => a.type !== 'provider' && a.enabled !== false);
2022
2220
  if (providerTooSmall && requestInfo.sessionKey && claudeAvailable
2023
- && canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
2221
+ && canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
2222
+ && resendAllowed(requestInfo)) {
2024
2223
  accountManager.markSessionLargeContext?.(requestInfo.sessionKey);
2025
2224
  // Bench EVERY provider: today's GLM + Kimi coding legs both cap at ~256K, so once
2026
2225
  // one 400s on size the others can't hold it either. If a genuine 1M-context
@@ -2043,7 +2242,8 @@ async function forwardRequest(
2043
2242
  // latch the session — the fault is one request's shape, not a durable property of
2044
2243
  // the conversation, and latching would evict a session from GLM on a single blip.
2045
2244
  if (providerRejectedShape && claudeAvailable
2046
- && canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
2245
+ && canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
2246
+ && resendAllowed(requestInfo)) {
2047
2247
  for (const a of (accountManager.accounts || [])) {
2048
2248
  if (a.type === 'provider') excludedIndexes.add(a.index);
2049
2249
  }
@@ -2185,21 +2385,51 @@ async function forwardRequest(
2185
2385
  } else { // Bound the non-streaming body read so a mid-body upstream stall can't hang
2186
2386
  // the request forever and leak the lease (same class as the streaming idle
2187
2387
  // guard). On timeout → UPSTREAM_BODY → the caller frees the lease.
2188
- const bodyP = upstreamRes.arrayBuffer();
2388
+ // F4: read through a reader we OWN so an abandoned read can really be cancelled — arrayBuffer() holds
2389
+ // the stream's lock, which made `body.cancel()` a no-op and left the orphaned read (and its socket) running.
2390
+ const bodyReader = upstreamRes.body?.getReader?.() ?? null;
2391
+ const bodyP = bodyReader
2392
+ ? (async () => {
2393
+ const chunks = [];
2394
+ for (;;) {
2395
+ const { done, value } = await bodyReader.read();
2396
+ if (done) return Buffer.concat(chunks);
2397
+ chunks.push(Buffer.from(value));
2398
+ }
2399
+ })()
2400
+ : upstreamRes.arrayBuffer();
2189
2401
  bodyP.catch(() => {});
2190
2402
  let bodyTimer;
2403
+ // B3: bounded by min(UPSTREAM_BODY_MS, remaining) when the request carries a deadline.
2404
+ const bodyWait = nonStreamBodyWait(requestInfo, UPSTREAM_BODY_MS);
2191
2405
  const bodyTimeout = new Promise((_, reject) => {
2192
2406
  bodyTimer = setTimeout(
2193
- () => reject(Object.assign(new Error('upstream body timeout'), { code: 'UPSTREAM_BODY' })),
2194
- UPSTREAM_BODY_MS,
2407
+ () => reject(Object.assign(new Error('upstream body timeout'), { code: bodyWait.deadlineBound ? 'UPSTREAM_DEADLINE' : 'UPSTREAM_BODY' })),
2408
+ bodyWait.ms,
2195
2409
  );
2196
2410
  bodyTimer.unref?.();
2197
2411
  });
2412
+ // B10: the pre-response close listener was dropped when the headers arrived, so during this
2413
+ // body read nobody watches the client. Without this, a client that left mid-read still gets
2414
+ // a second upstream call when the read fails (the move path re-sends for a dead socket).
2415
+ let onBodyClientClose;
2416
+ const clientLeft = new Promise((_, reject) => {
2417
+ const gone = () => reject(Object.assign(new Error('client closed during body read'), { code: 'CLIENT_GONE_BODY' }));
2418
+ if (res.destroyed || res.closed) { gone(); return; }
2419
+ onBodyClientClose = gone;
2420
+ res.once('close', onBodyClientClose);
2421
+ });
2422
+ clientLeft.catch(() => {});
2198
2423
  let arr;
2199
2424
  try {
2200
- arr = await Promise.race([bodyP, bodyTimeout]);
2425
+ arr = await Promise.race([bodyP, bodyTimeout, clientLeft]);
2426
+ } catch (e) {
2427
+ // Client gone / deadline / stalled body: stop the orphaned read and close the upstream socket now.
2428
+ try { await (bodyReader ? bodyReader.cancel() : upstreamRes.body?.cancel()); } catch { /* already closed */ }
2429
+ throw e;
2201
2430
  } finally {
2202
2431
  clearTimeout(bodyTimer);
2432
+ if (onBodyClientClose) res.off('close', onBodyClientClose);
2203
2433
  }
2204
2434
  const buf = Buffer.from(arr);
2205
2435
  extractUsageFromBody(buf, account.index, accountManager, requestInfo);
@@ -2279,6 +2509,32 @@ async function forwardRequest(
2279
2509
  releaseOnClientGone();
2280
2510
  return;
2281
2511
  }
2512
+ // B10: the client left during the non-streaming BODY read. Not an account fault, nothing to move,
2513
+ // nothing to write to: release neutral and stop (the move path below would fetch on a dead socket).
2514
+ if (err?.code === 'CLIENT_GONE_BODY') {
2515
+ const heldMs = Date.now() - (requestInfo.startedAt || Date.now());
2516
+ console.log(`[Maxpool] Client left after ${(heldMs / 1000).toFixed(1)}s on "${account.name}" (during the response body read) — released, not retried`);
2517
+ logFailedAttempt(account.name, 'client_cancelled', `client left after ${(heldMs / 1000).toFixed(1)}s (body read)`, 'released-cancel');
2518
+ accountManager.relinquishUpstreamProbe?.(lease);
2519
+ accountManager.releaseAccount(lease, { neutral: true, error: 'client_cancelled', kind: 'client_cancelled' });
2520
+ clearQueueHeartbeat(requestInfo);
2521
+ accountManager.removeQueuedRequest?.(requestInfo);
2522
+ return;
2523
+ }
2524
+ // B12: the wait was cut by the CLIENT'S deadline — neither an account fault (no markTransientFailure
2525
+ // streak) nor a link fault. Release neutral and answer a retryable 504; nothing is moved or re-sent.
2526
+ if (err.code === 'UPSTREAM_DEADLINE') {
2527
+ accountManager.relinquishUpstreamProbe?.(lease);
2528
+ accountManager.releaseAccount(lease, { neutral: true, error: 'deadline', kind: 'deadline' });
2529
+ ctx.status = 504;
2530
+ logFailedAttempt(account.name, 'deadline', `no answer after ${heldSeconds(lease)}s (client patience), non-streaming`, 'surfaced');
2531
+ if (logDir) {
2532
+ logSections.push(`=== DEADLINE ===\n${err.message}`);
2533
+ writeRequestLog(logDir, reqId, logSections);
2534
+ }
2535
+ sendDeadlineResponse(res, requestInfo);
2536
+ return;
2537
+ }
2282
2538
  // undici reports every socket/DNS/TLS failure as the bare string "fetch failed" and
2283
2539
  // hangs the REAL reason off err.cause. Logging err.message alone threw that away: 588
2284
2540
  // "fetch failed" lines and ZERO ECONNRESET/ENOTFOUND/UND_ERR in the whole log, leaving
@@ -2297,7 +2553,7 @@ async function forwardRequest(
2297
2553
  // would have seen before this feature existed.
2298
2554
  // !clientHasRealBytes, not !headersSent: a slow turn whose keep-alive committed
2299
2555
  // (pings only) is just as reroutable as a fast one (review r2 B).
2300
- if (err.code === 'REFUSAL_RETRY' && canRetryBufferedBody && !clientHasRealBytes(res, requestInfo)) {
2556
+ if (err.code === 'REFUSAL_RETRY' && canRetryBufferedBody && !clientHasRealBytes(res, requestInfo) && resendAllowed(requestInfo)) {
2301
2557
  const category = err.refusalCategory || 'unknown';
2302
2558
  const anthropicIndexes = accountManager.accounts
2303
2559
  .filter(a => a.type !== 'provider')
@@ -2388,6 +2644,15 @@ async function forwardRequest(
2388
2644
  { attempt: [account.name, transientKind, 'transient; no other route'] },
2389
2645
  );
2390
2646
  if (queued) return;
2647
+ // F3: a request with a deadline that ran out of routes on a first-byte timeout answers the request-scoped
2648
+ // 504 (the Mac books it neutral), never a bare 503 that cools its row and counts a failure. Direct
2649
+ // long-patience clients get the same answer (R2-2); an old Mac never has a deadline here (F1 gate).
2650
+ if (err.code === 'UPSTREAM_TTFB' && !requestInfo.stream && requestInfo.deadlineAt != null) {
2651
+ ctx.status = 504;
2652
+ logFailedAttempt(account.name, 'deadline', `no first byte after ${heldSeconds(lease)}s (hop cap), no route left`, 'surfaced');
2653
+ sendDeadlineResponse(res, requestInfo);
2654
+ return;
2655
+ }
2391
2656
  ctx.status = 503;
2392
2657
  // Record the user-facing failure explicitly so it's greppable in the event
2393
2658
  // log (the in-memory TUI feed scrolls away). `err` is the last network error.
@@ -2412,7 +2677,8 @@ async function forwardRequest(
2412
2677
  }
2413
2678
 
2414
2679
  accountManager.releaseAccount(lease, { error: err.message });
2415
- if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)) {
2680
+ if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !clientHasRealBytes(res, requestInfo)
2681
+ && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
2416
2682
  account.status = 'error';
2417
2683
  excludedIndexes.add(account.index);
2418
2684
  logFailedAttempt(account.name, (err.code || err.message || 'error').toString().slice(0, 40), err.message, 'moved');
@@ -2475,6 +2741,9 @@ function noteScaledTtfb(requestInfo, accountName, path, waitMs, floorMs, ceiling
2475
2741
  * second wait can still end inside the client's patience. Direct path only.
2476
2742
  */
2477
2743
  function retrySameEligible(account, requestInfo, res, queueConfig = {}) {
2744
+ // B5: a non-streaming request is never retried on the same account — the second wait is a second
2745
+ // full hop of silence behind a client that cannot be told we are still working.
2746
+ if (!requestInfo.stream) return false;
2478
2747
  if (requestInfo.sameAccountRetried) return false;
2479
2748
  if ((requestInfo.bodyBytes || 0) < BIG_PROMPT_BYTES) return false;
2480
2749
  if (!account || account.status === 'throttled' || account.status === 'error'
@@ -2525,10 +2794,33 @@ function isProviderAuthStatus(status) {
2525
2794
  return status === 401 || status === 403;
2526
2795
  }
2527
2796
 
2797
+ /** F5: the re-send guard shared by EVERY site that sends a request upstream a second time (account moves
2798
+ * AND body-repair / pin re-sends, which need no route check because they are not a fresh placement):
2799
+ * B10 — never re-send for a client that left; B6 — a non-streaming request with under the minimum
2800
+ * time left is not re-sent (a fresh hop cannot finish). */
2801
+ function resendAllowed(requestInfo = {}) {
2802
+ if (clientAlreadyGone(requestInfo)) {
2803
+ if (!requestInfo._clientGoneLogged) {
2804
+ requestInfo._clientGoneLogged = true;
2805
+ console.log('[Maxpool] Client already left — not re-sending to another account');
2806
+ logFailedAttempt('-', 'client_cancelled', 'client left before a re-send', 'released-cancel');
2807
+ }
2808
+ return false;
2809
+ }
2810
+ return nonStreamRemainingMs(requestInfo) >= deadlineKnobs().NONSTREAM_MIN_MOVE_MS;
2811
+ }
2812
+
2528
2813
  function hasEligibleRoute(accountManager, requestInfo = {}, excludedIndexes = new Set()) {
2814
+ if (!resendAllowed(requestInfo)) return false;
2529
2815
  return accountManager.hasAvailableRoute?.(requestInfo, excludedIndexes) || false;
2530
2816
  }
2531
2817
 
2818
+ /** B10: has the client socket of this request closed? (requestInfo carries its res; set at forwardRequest entry.) */
2819
+ function clientAlreadyGone(requestInfo = {}) {
2820
+ const r = requestInfo._clientRes;
2821
+ return Boolean(r && (r.destroyed || r.closed));
2822
+ }
2823
+
2532
2824
  function formatRetryDuration(seconds) {
2533
2825
  const s = Math.max(0, Math.round(Number(seconds) || 0));
2534
2826
  if (s >= 86400) return `${Math.round(s / 86400)}d`;
@@ -2738,7 +3030,7 @@ function isContextLengthError(errorBody) {
2738
3030
  return /exceeded model token limit|maximum context length|context length exceeded|context window (?:size )?(?:exceeded|too)|prompt is too long|input is too long|reduce the length of|too many (?:input )?tokens|request too large/i.test(errorBody);
2739
3031
  }
2740
3032
 
2741
- export const __serverTest = { directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
3033
+ export const __serverTest = { hasEligibleRoute, resendAllowed, directiveOnlySystemMessages, rewriteBodyForAccount, sanitizeBlocksForProvider, reanchorOrphanedSystemMessages, unavailableMessage, computeQueueWindowMs, waitForAvailableRoute, isRetriableUpstreamStatus, classifyEffortRejection, repairEffort, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, parseRejectedBlockPath, stripRejectedBlockClass, peekRejectedBlockType, describeRejectedBlock, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, ensureQueueHeartbeatInterval, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, isRequestShapeRefusal, looksLikeClaudeCode, parseProviderRetryAfter, secondsUntilParsedTime, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, isProviderParamRejection, describeBodyShape, logFailedAttempt, isRequestScopedTimeout, clientWaitCeilingMs, scaledTtfbMs, retrySameEligible, parsePatience, computeDeadline, computeLinkHop, nonStreamFirstByteWait, nonStreamBodyWait, nonStreamRemainingMs, deadlineKnobs, deadlineErrorPayload, ttfbKnobs: () => ({ UPSTREAM_TTFB_MS, TTFB_MS_PER_BYTE, BIG_PROMPT_BYTES, CLI_STREAM_CEILING_MS }), streamResponse, startIdleRequestReaper, normalizeModelEcho, classifyHeldStreamPrefix };
2742
3034
 
2743
3035
  async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
2744
3036
  if (!upstreamRes.body) return '';
@@ -3482,7 +3774,7 @@ async function queueAndRetryInner(
3482
3774
  onGiveUp = null, onHold = null,
3483
3775
  ) {
3484
3776
  const giveUp = (msg) => { onGiveUp?.(); return finishQueuedStreamIfNeeded(res, requestInfo, msg); };
3485
- if (!queueConfig.enabled || !canQueueBufferedBody || (res.headersSent && !requestInfo.queueHeartbeatActive) || res.destroyed) {
3777
+ if (!queueConfig.enabled || !canQueueBufferedBody || (res.headersSent && !requestInfo.queueHeartbeatActive) || res.destroyed || res.closed) {
3486
3778
  if (queueConfig.enabled && !canQueueBufferedBody && requestInfo.queueBlockedReason) {
3487
3779
  console.log(`[Maxpool] Not queueing request: ${requestInfo.queueBlockedReason}`);
3488
3780
  }
@@ -3577,7 +3869,7 @@ async function queueAndRetryInner(
3577
3869
  const streamClientToleranceMs = Number.isFinite(requestInfo.clientToleranceMs)
3578
3870
  ? requestInfo.clientToleranceMs
3579
3871
  : Math.max(0, Number(queueConfig.streamClientToleranceMs) || 0);
3580
- const queueWindowMs = computeQueueWindowMs({
3872
+ let queueWindowMs = computeQueueWindowMs({
3581
3873
  cause,
3582
3874
  stream: Boolean(requestInfo.stream),
3583
3875
  retryPlanCause: retryPlan.cause,
@@ -3590,6 +3882,13 @@ async function queueAndRetryInner(
3590
3882
  countTokensMaxWaitMs,
3591
3883
  });
3592
3884
 
3885
+ // B6: a non-streaming hold is capped at the time the client still has; none left = give up now.
3886
+ const deadlineRemainingMs = nonStreamRemainingMs(requestInfo);
3887
+ if (deadlineRemainingMs !== Infinity) {
3888
+ if (deadlineRemainingMs <= 0) return giveUp(honestMessage);
3889
+ queueWindowMs = Math.min(queueWindowMs, deadlineRemainingMs);
3890
+ }
3891
+
3593
3892
  if (queueWindowMs <= 0) return giveUp(honestMessage);
3594
3893
 
3595
3894
  // HOLD-vs-ERROR oracle (from nextRetryForRequest): HOLD only when a TEMPORARY
@@ -4301,6 +4600,10 @@ function startIdleRequestReaper(res, reqId, idleMs, { now = Date.now, setInterva
4301
4600
  if (getRequestInfo?.()?.queueHeartbeatActive) { lastProgressAt = now(); return; }
4302
4601
  const bytes = res.socket?.bytesWritten ?? lastBytes;
4303
4602
  if (bytes !== lastBytes) { lastBytes = bytes; lastProgressAt = now(); return; }
4603
+ // R2-1: a non-streaming request with a deadline is bounded by that deadline (504 at
4604
+ // deadlineAt); never cut it before deadline + grace. Never SHORTENS the ceiling.
4605
+ const info = getRequestInfo?.();
4606
+ if (info && !info.stream && info.deadlineAt != null && now() < info.deadlineAt + DEADLINE_REAP_GRACE_MS) return;
4304
4607
  if (now() - lastProgressAt >= idleMs && !res.writableEnded && !res.destroyed) {
4305
4608
  console.error(`[Maxpool] Request ${reqId} — no write progress for ${Math.round(idleMs / 1000)}s (backstop reaper); force-aborting a stuck request to free its account slot`);
4306
4609
  res.destroy();
@@ -4358,9 +4661,29 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
4358
4661
  // upstream headers and the first body chunk can still fail over (clientHasRealBytes
4359
4662
  // stays false) and the client's idle watchdog stays fed. realWrite stops it right
4360
4663
  // before the first real write, so no ping can land between live SSE events (Bug A).
4664
+ // B9: a SCRUBBED provider stream drops every thinking event, so a long GLM think leaves the client
4665
+ // (and a linked Mac's idle timer) with nothing for tens of seconds — Claude Code reads that as a
4666
+ // dead stream. Once the first real byte is out, and only while a scrubber is active, keep a
4667
+ // keep-alive ping flowing. Safe mid-stream: the scrubber hands realWrite WHOLE events only
4668
+ // (provider-thinking-scrub.js push/event), so a ping can only land between events. Never armed
4669
+ // before the first real write (the model-echo/refusal holds `continue` ahead of realWrite, and a
4670
+ // ping before writeHead would commit default headers).
4671
+ let scrubPingTimer = null;
4672
+ let scrubPingStopped = false;
4673
+ const armScrubPing = () => {
4674
+ if (!scrubber || scrubPingTimer || scrubPingStopped || res.destroyed) return;
4675
+ const ms = Math.max(100, Number(process.env.MAXPOOL_SCRUB_PING_MS) || 3000);
4676
+ scrubPingTimer = setInterval(() => {
4677
+ if (res.destroyed || res.writableEnded) return;
4678
+ try { res.write('event: ping\ndata: {"type":"ping"}\n\n'); } catch { /* client gone */ }
4679
+ }, ms);
4680
+ scrubPingTimer.unref?.();
4681
+ };
4361
4682
  const realWrite = (chunk) => {
4362
4683
  if (requestInfo.queueHeartbeatActive) clearQueueHeartbeat(requestInfo);
4363
- return res.write(chunk);
4684
+ const ok = res.write(chunk);
4685
+ armScrubPing();
4686
+ return ok;
4364
4687
  };
4365
4688
 
4366
4689
  // Client left mid-stream: the caller drops its disconnect-abort once headers
@@ -4573,6 +4896,9 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
4573
4896
  if (err?.code === 'REFUSAL_RETRY') refusalRerouting = true;
4574
4897
  throw err;
4575
4898
  } finally {
4899
+ // B9: stop the keep-alive before anything else in the teardown, and never re-arm it.
4900
+ scrubPingStopped = true;
4901
+ if (scrubPingTimer) { clearInterval(scrubPingTimer); scrubPingTimer = null; }
4576
4902
  // SAFEGUARD-REFUSAL HOLD last-resort flush. A stream can end while the hold is
4577
4903
  // still undecided — a `usage`-only stream (message_delta with no content blocks
4578
4904
  // and no message_stop) reaches `done` having never produced a decisive event, and
package/src/tui.js CHANGED
@@ -69,6 +69,23 @@ export async function reclaimRawMode({ stdin = process.stdin, stdout = process.s
69
69
  return false;
70
70
  }
71
71
 
72
+ /**
73
+ * reclaimRawMode + the update-window measurement (release B, B11): says how long the new
74
+ * worker took to take the terminal back, or that it was deferred to the terminal's drain
75
+ * (and how long that took when it finally happened). `now`/`log` are injectable for tests.
76
+ */
77
+ export async function reclaimRawModeLogged({ stdout = process.stdout, log = m => console.log(`[Maxpool] ${m}`), now = Date.now, ...rest } = {}) {
78
+ const t0 = now();
79
+ const done = await reclaimRawMode({ stdout, ...rest });
80
+ if (done) {
81
+ log(`reload: terminal reclaimed in ${now() - t0}ms`);
82
+ } else {
83
+ log('reload: terminal reclaim deferred to drain');
84
+ stdout.once('drain', () => log(`reload: terminal reclaimed after drain in ${now() - t0}ms`));
85
+ }
86
+ return done;
87
+ }
88
+
72
89
  const ANSI_RE = /\x1b\[[0-9;]*m/g;
73
90
  const strip = s => s.replace(ANSI_RE, '');
74
91
  const vw = s => strip(s).length;
@@ -735,7 +752,9 @@ export class TUI {
735
752
  ttfbMs = r.firstByteAt - r.started;
736
753
  }
737
754
  const ttfbTag = ttfbMs != null ? ` ttfb=${(Math.max(0, ttfbMs) / 1000).toFixed(1)}s` : '';
738
- this._addLog(`${who}${info.method} ${info.path} → ${acct} (${info.status}, ${dur}s)${provTag}${cvTag}${ttfbTag}`);
755
+ // B8: ' ns' = a non-streaming request (strict false: the oauth relay carries no shape, so no tag).
756
+ const nsTag = info.stream === false ? ' ns' : '';
757
+ this._addLog(`${who}${info.method} ${info.path} → ${acct} (${info.status}, ${dur}s)${nsTag}${provTag}${cvTag}${ttfbTag}`);
739
758
  }
740
759
 
741
760
  _addLog(msg) {