maxpool 1.5.50 → 1.5.52

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.5.50",
+  "version": "1.5.52",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/event-log.js

@@ -7,7 +7,11 @@
 
 import { appendFile, stat, rename, chmod } from 'node:fs/promises';
 
-const MAX_BYTES = 5 * 1024 * 1024;   // rotate at ~5 MB (one .1 backup kept)
+const MAX_BYTES = 5 * 1024 * 1024;   // rotate at ~5 MB
+// Keep 5 generations (~30 MB total), not 1. With a single backup the log rotated away the
+// evidence MID-INVESTIGATION on 2026-07-28 — the reported error's own line was already gone
+// before it could be read. Diagnosing a stall needs hours of history, not minutes.
+const MAX_GENERATIONS = Math.max(1, Number(process.env.MAXPOOL_LOG_GENERATIONS) || 5);
 const ROTATE_CHECK_MS = 2000;        // rotation-owner size-check cadence
 // Cap each line below the platform PIPE_BUF (512 B on macOS) so concurrent
 // O_APPEND writes from coexisting processes during a reload stay atomic (never
@@ -91,7 +95,13 @@ export async function rotateIfNeeded() {
   const path = logPath;
   if (!path) return;
   const st = await stat(path).catch(() => null);
-  if (st && st.size > MAX_BYTES) await rename(path, `${path}.1`).catch(() => {});
+  if (st && st.size > MAX_BYTES) {
+    // Cascade .N-1 -> .N (oldest first) so N generations survive instead of one.
+    for (let i = MAX_GENERATIONS - 1; i >= 1; i--) {
+      await rename(`${path}.${i}`, `${path}.${i + 1}`).catch(() => {});
+    }
+    await rename(path, `${path}.1`).catch(() => {});
+  }
 }
 
 /**

package/src/server.js

@@ -22,7 +22,25 @@ const DEFAULT_RETRY = {
 // serves EVERY provider (Anthropic/GLM/Kimi), so the idle gap is generous enough
 // that a legitimately-slow-but-alive stream is never cut (each chunk resets it).
 const UPSTREAM_TTFB_MS = Math.max(5_000, Number(process.env.MAXPOOL_TTFB_MS) || 120_000);        // headers must arrive within this
-const STREAM_IDLE_MS = Math.max(30_000, Number(process.env.MAXPOOL_STREAM_IDLE_MS) || 300_000);  // max gap BETWEEN streamed chunks (reset per chunk)
+// A real SSE EVENT, not a `:` comment. Claude Code's stall watchdog is reset only when its
+// SSE iterator YIELDS an event; per the spec a comment line is discarded by the parser and
+// never yields, so a comment keepalive resets nothing. That is why held requests died at
+// EXACTLY 300.0s — the client's floor — despite a 10s heartbeat (60 such deaths in 2.4
+// days). `ping` is an unknown event type the client ignores semantically while still
+// counting as traffic.
+const NETWORK_ERROR_CODES = new Set([
+  'ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'UND_ERR_CONNECT_TIMEOUT',
+  'ENOTFOUND', 'EAI_AGAIN', 'EPIPE', 'ECONNABORTED', 'UND_ERR_SOCKET', 'UND_ERR_HEADERS_TIMEOUT',
+]);
+const isNetworkCode = c => Boolean(c) && NETWORK_ERROR_CODES.has(c);
+const QUEUE_KEEPALIVE = 'event: ping\ndata: {}\n\n';
+// 240s, strictly BELOW Claude Code's hard 300s stall floor. At the old 300_000 the two
+// timers were a dead heat and the client always won — maxpool's clock starts when a chunk
+// is READ from upstream, strictly before the client parses it. Result: `upstream idle
+// timeout` fired 0 times in 2.4 days while 60 requests died silently client-side. Below the
+// floor maxpool wins and turns a silent stall into a labelled error. Anthropic pings during
+// extended thinking, so real gaps never approach 4 minutes.
+const STREAM_IDLE_MS = Math.max(30_000, Number(process.env.MAXPOOL_STREAM_IDLE_MS) || 240_000);  // max gap BETWEEN streamed chunks (reset per chunk)
 const UPSTREAM_BODY_MS = Math.max(30_000, Number(process.env.MAXPOOL_BODY_MS) || 300_000);       // non-streaming body read
 const CLIENT_DRAIN_MS = Math.max(5_000, Number(process.env.MAXPOOL_DRAIN_MS) || 60_000);         // max wait for a backpressured client to drain (half-open client → free the lease)
 // A provider 403 is (unlike a 401) almost always transient QUOTA/PLAN exhaustion — cool
@@ -42,6 +60,13 @@ const DEFAULT_QUEUE = {
   maxWaitMs: 24 * 60 * 60 * 1000,
   autoMaxWaitMs: null,
   capacityMaxWaitMs: 15 * 60 * 1000,
+  // NETWORK-cause hold ceiling. A quota/capacity hold is worth waiting out — a reset is
+  // genuinely coming. A NETWORK hold is not: the connection is dead, and holding it just
+  // parks the caller on a socket nothing is watching. On 2026-07-28 a request was held
+  // 10,445s (2h54m) having produced ZERO bytes; nothing aborted it until the user touched
+  // the keyboard. Give up quickly instead and return a retryable 429, so the client
+  // reconnects on a FRESH connection — which is what actually self-heals a dead route.
+  networkMaxWaitMs: 2 * 60 * 1000,
   weeklyMaxWaitMs: 24 * 60 * 60 * 1000, // legacy bound; streaming holds use streamHoldMaxMs
   // STREAMING hold ceiling: how long a streaming request may be HELD ALIVE on
   // the SSE heartbeat waiting for any account to free up. Defaults to 7d (the
@@ -62,7 +87,14 @@ const DEFAULT_QUEUE = {
   // (front-loaded work waiting for a free account ≈ a few hours) so beyond that the
   // request error-fasts with an honest retryable 429 instead of a silent multi-day park.
   // Pair with a raised client watchdog (the cc launch sets CLAUDE_STREAM_IDLE_TIMEOUT_MS).
-  streamClientToleranceMs: Math.max(60_000, Number(process.env.MAXPOOL_STREAM_CLIENT_TOLERANCE_MS) || 3 * 60 * 60 * 1000),
+  // Only trust a long hold when the client's watchdog was ACTUALLY raised. The `cc` alias
+  // exports CLAUDE_STREAM_IDLE_TIMEOUT_MS=3h, but a session started any other way keeps the
+  // 300s floor — holding its request for hours just parks a caller that left at 5 minutes.
+  // Derive from the env we can observe; otherwise stay under the real floor.
+  streamClientToleranceMs: Math.max(60_000, Number(process.env.MAXPOOL_STREAM_CLIENT_TOLERANCE_MS)
+    || (Number(process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS) > 300_000
+      ? Math.floor(Number(process.env.CLAUDE_STREAM_IDLE_TIMEOUT_MS) * 0.8)
+      : 240_000)),
   // Non-streaming requests have no SSE heartbeat to keep them alive, so a long
   // hold would die on the client timeout anyway. Cap their wait conservatively.
   nonStreamMaxWaitMs: 5 * 60 * 1000,
@@ -1226,10 +1258,24 @@ async function forwardRequest(
     // lease (free the scarce account) and STOP: no retry (the client is gone), no
     // write (the socket is dead).
     if (clientGone.signal.aborted) {
+      // LOG IT. This silent return hid every client-side give-up: 60 requests in 2.4 days
+      // died here leaving only an indistinguishable `(null, 300.0s)` line. `committed`
+      // separates "the user saw a partial answer" (real harm, unretryable) from "the user
+      // was still waiting" — and the elapsed time is what exposes a client watchdog firing
+      // at its floor while maxpool was still happily holding the request.
+      const heldMs = Date.now() - (requestInfo.startedAt || Date.now());
+      console.log(`[Maxpool] Client left after ${(heldMs / 1000).toFixed(1)}s on "${account.name}" `
+        + `(${res.headersSent ? 'mid-response — output already sent' : 'still waiting, nothing sent'})`);
       releaseOnClientGone();
       return;
     }
-    console.error(`[Maxpool] Upstream error (account "${account.name}"):`, err.message);
+    // undici reports every socket/DNS/TLS failure as the bare string "fetch failed" and
+    // hangs the REAL reason off err.cause. Logging err.message alone threw that away: 588
+    // "fetch failed" lines and ZERO ECONNRESET/ENOTFOUND/UND_ERR in the whole log, leaving
+    // every incident unattributable (DNS? TLS? socket exhaustion?).
+    const rootCause = err.cause?.code || err.cause?.message || err.code || '';
+    console.error(`[Maxpool] Upstream error (account "${account.name}"):`, err.message
+      + (rootCause && !String(err.message).includes(rootCause) ? ` (cause: ${rootCause})` : ''));
 
     if (logDir) {
       logSections.push(`=== ERROR ===\n${err.stack || err.message}`);
@@ -1238,8 +1284,10 @@ async function forwardRequest(
 
     const isTransient = err instanceof Error &&
       (err.message.includes('fetch failed') ||
-        err.code === 'ECONNRESET' || err.code === 'ECONNREFUSED' ||
-        err.code === 'ETIMEDOUT' || err.code === 'UND_ERR_CONNECT_TIMEOUT' ||
+        // Read BOTH: on an undici fetch rejection err.code is undefined and the code lives
+        // on err.cause, so the bare err.code branches were dead — only the 'fetch failed'
+        // string match kept this classification alive.
+        isNetworkCode(err.code) || isNetworkCode(err.cause?.code) ||
         // Upstream-hang guards: a stalled connection is network-class, not the
         // account's fault → 5s cooldown + release + (pre-headers) retry elsewhere;
         // once committed, isTransient falls through to sendErrorResponse which ends
@@ -1346,7 +1394,7 @@ function formatRetryDuration(seconds) {
 function computeQueueWindowMs({
   cause, stream, retryPlanCause,
   maxWaitMs, capacityMaxWaitMs, nonStreamMaxWaitMs, streamHoldMaxMs, streamClientToleranceMs,
-  isCountTokens, countTokensMaxWaitMs,
+  isCountTokens, countTokensMaxWaitMs, networkMaxWaitMs,
 }) {
   let windowMs;
   if (!stream) {
@@ -1367,6 +1415,11 @@ function computeQueueWindowMs({
   // the upstream processing once acquired) so a non-heartbeated metadata call fast-
   // fails with a retryable 429 instead of hanging past the client's idle window.
   if (isCountTokens && countTokensMaxWaitMs != null) windowMs = Math.min(windowMs, countTokensMaxWaitMs);
+  // A dead ROUTE is not a scheduled reset: holding it out is waiting for nothing. Cap it
+  // hard, independent of how patient the client is — a raised CLAUDE_STREAM_IDLE_TIMEOUT_MS
+  // (the `cc` alias sets 3h) otherwise licenses a multi-hour hold on a connection that is
+  // simply gone. Error-fast + client reconnect beats an unattended hold.
+  if (cause === 'network' && networkMaxWaitMs != null) windowMs = Math.min(windowMs, networkMaxWaitMs);
   return windowMs;
 }
 
@@ -1803,8 +1856,14 @@ function sendErrorResponse(res, requestInfo, status, payload, headers = {}) {
   if (requestInfo.queueHeartbeatActive || res.headersSent) {
     clearQueueHeartbeat(requestInfo);
     if (!res.destroyed && !res.writableEnded) {
-      res.write(`event: error\ndata: ${JSON.stringify(payload)}\n\n`);
-      res.end();
+      // Guarded: a write onto a half-dead socket throws, and there is no res.on('error')
+      // anywhere here — an uncaught one reaches the worker's uncaughtException handler,
+      // which process.exit()s and bounces EVERY other in-flight stream. ensureQueueHeartbeat
+      // already guards its identical write; this one did not.
+      try {
+        res.write(`event: error\ndata: ${JSON.stringify(payload)}\n\n`);
+      } catch { /* peer vanished mid-write — nothing to deliver, fall through to end() */ }
+      try { res.end(); } catch { /* already torn down */ }
     }
     return;
   }
@@ -1903,6 +1962,9 @@ async function queueAndRetry(
   const streamClientToleranceMs = queueConfig.streamClientToleranceMs == null
     ? 3 * 60 * 60 * 1000
     : Math.max(0, Number(queueConfig.streamClientToleranceMs) || 0);
+  const networkMaxWaitMs = queueConfig.networkMaxWaitMs == null
+    ? 2 * 60 * 1000
+    : Math.max(0, Number(queueConfig.networkMaxWaitMs) || 0);
   const queueWindowMs = computeQueueWindowMs({
     cause,
     stream: Boolean(requestInfo.stream),
@@ -1910,6 +1972,7 @@ async function queueAndRetry(
     maxWaitMs,
     capacityMaxWaitMs,
     nonStreamMaxWaitMs,
+    networkMaxWaitMs,
     streamHoldMaxMs,
     streamClientToleranceMs,
     isCountTokens: Boolean(requestInfo.isCountTokens),
@@ -1972,7 +2035,7 @@ async function queueAndRetry(
   // committed stream and DROP the held session. Keeping the heartbeat active lets
   // it re-hold. The heartbeat is instead stopped the instant real upstream bytes
   // start flowing, inside streamResponse — that prevents the Bug A interleave
-  // (': maxpool queued' comments injected between real SSE events) without losing
+  // (queue keepalive pings injected between real SSE events) without losing
   // re-holdability on a post-resume failover.
   return forwardRequest(
     req, res, body, accountManager, upstream, 0, hooks, reqId, ctx, logDir,
@@ -2024,7 +2087,7 @@ function ensureQueueHeartbeat(res, requestInfo, queueConfig, accountManager) {
       'X-Accel-Buffering': 'no',
     });
     res.flushHeaders?.();
-    res.write(': maxpool queued\n\n');
+    res.write(QUEUE_KEEPALIVE);
   } catch {
     reapDead();
     return;
@@ -2033,7 +2096,7 @@ function ensureQueueHeartbeat(res, requestInfo, queueConfig, accountManager) {
   requestInfo.queueHeartbeatTimer = setInterval(() => {
     if (res.destroyed || res.writableEnded) { reapDead(); return; }
     try {
-      res.write(': maxpool queued\n\n');
+      res.write(QUEUE_KEEPALIVE);
     } catch {
       reapDead();
     }
@@ -2369,7 +2432,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
   // We're now committed to streaming a real upstream response body onto this
   // response — there is no more failover for this forward. Stop the queue
   // heartbeat (if this was a resumed held stream) BEFORE the first real byte, so
-  // the setInterval can't inject ': maxpool queued' comments between live SSE
+  // the setInterval can't inject queue keepalive pings between live SSE
   // events (Bug A). It is deliberately NOT cleared earlier (on resume), so a
   // pre-byte failover can still re-hold the session via queueAndRetry.
   clearQueueHeartbeat(requestInfo);

package/src/sleep-guard.js

@@ -11,7 +11,12 @@
 import { spawn as realSpawn } from 'node:child_process';
 
 const POLL_MS = 5_000;      // re-check the in-flight signal (idle-sleep timers are minutes)
-const GRACE_MS = 60_000;    // stay awake this long after the last request, to bridge gaps
+// Stay awake this long after the last request. 60s lost a measured race on 2026-07-28: a
+// network outage drained all in-flight work, the guard released exactly 74s later, and
+// macOS (idle-sleep timer) slept in the SAME SECOND — so the machine went down while the
+// outage was still in progress and recovery had to wait for a keystroke. 5 minutes bridges
+// an outage-shaped gap without meaningfully delaying a genuinely idle sleep.
+const GRACE_MS = 300_000;
 
 export class SleepGuard {
   constructor({ getWorkPending, log = () => {}, spawn = realSpawn, enabled, pollMs = POLL_MS, graceMs = GRACE_MS } = {}) {