maxpool 1.5.35 → 1.5.37

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.5.35",
+  "version": "1.5.37",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/server.js

@@ -25,6 +25,11 @@ const UPSTREAM_TTFB_MS = Math.max(5_000, Number(process.env.MAXPOOL_TTFB_MS) ||
 const STREAM_IDLE_MS = Math.max(30_000, Number(process.env.MAXPOOL_STREAM_IDLE_MS) || 300_000);  // max gap BETWEEN streamed chunks (reset per chunk)
 const UPSTREAM_BODY_MS = Math.max(30_000, Number(process.env.MAXPOOL_BODY_MS) || 300_000);       // non-streaming body read
 const CLIENT_DRAIN_MS = Math.max(5_000, Number(process.env.MAXPOOL_DRAIN_MS) || 60_000);         // max wait for a backpressured client to drain (half-open client → free the lease)
+// A provider 403 is (unlike a 401) almost always transient QUOTA/PLAN exhaustion — cool
+// the provider down RECOVERABLY for this window, then re-probe, instead of permanently
+// disabling it. Short (not reset-length) so a provider-pinned session's hold window
+// isn't blown, and a still-exhausted provider just re-benches on the next single retry.
+const PROVIDER_FORBIDDEN_COOLDOWN_SEC = Math.max(60, Number(process.env.MAXPOOL_PROVIDER_403_COOLDOWN_SEC) || 1800);
 // Backstop reaper idle ceiling. Floored WELL above the longest a legit NON-streaming
 // request can go silent (no heartbeat there): TTFB 120s + nonStreamMaxWaitMs 300s +
 // UPSTREAM_BODY_MS 300s ≈ 720s — so a slow-but-alive non-streaming request is never
@@ -51,6 +56,11 @@ const DEFAULT_QUEUE = {
   // Non-streaming requests have no SSE heartbeat to keep them alive, so a long
   // hold would die on the client timeout anyway. Cap their wait conservatively.
   nonStreamMaxWaitMs: 5 * 60 * 1000,
+  // count_tokens is cheap non-streaming metadata — cap its queue wait VERY low so it
+  // fast-fails with a retryable 429 instead of hanging silently past the client's idle
+  // window. Kept well under any plausible client idle timeout (observed errors as low
+  // as ~23s). Env-overridable for tuning as the "Stream idle timeout" reports resolve.
+  countTokensMaxWaitMs: Math.max(1000, Number(process.env.MAXPOOL_COUNT_TOKENS_MAX_WAIT_MS) || 8000),
   // Backpressure: holds used to be 0ms, now they can be hours. Bound the queue
   // so 22 retrying agents can't grow the heap without limit.
   maxConcurrentQueued: 64,
@@ -84,6 +94,11 @@ export function createProxyServer(accountManager, config, hooks = {}) {
 
   const server = http.createServer(async (req, res) => {
     try {
+      // Disable Nagle on the client socket: flush each SSE event immediately instead
+      // of coalescing small writes into bursts. Smooths streaming cadence toward the
+      // HTTP/2-direct shape, reducing how often a client terminal's re-render churns
+      // on long streams. Same underlying socket as res, so this covers the write path.
+      try { req.socket?.setNoDelay(true); } catch { /* socket may already be gone */ }
       try { res.setHeader('x-maxpool-worker', workerStamp); } catch { /* headers sent */ }
       if (draining) {
         try { res.setHeader('Connection', 'close'); } catch { /* headers may be sent */ }
@@ -654,16 +669,27 @@ async function forwardRequest(
 
     if (account.type === 'provider' && isProviderAuthStatus(upstreamRes.status)) {
       const errorBody = await readErrorBody(upstreamRes);
-      const reason = upstreamRes.status === 401 ? 'auth_failed' : 'forbidden';
-      accountManager.markAuthFailed(account.index, upstreamRes.status, reason);
-      accountManager.releaseAccount(lease, { status: upstreamRes.status, error: reason });
+      const forbidden = upstreamRes.status === 403;
+      if (forbidden) {
+        // 403 = almost always QUOTA/PLAN exhaustion (e.g. Kimi Coding-Plan weekly maxed),
+        // NOT bad credentials — the usage probe still succeeds with the same key. Cool it
+        // down RECOVERABLY (rides the rate-limit recovery in _isAvailable → auto-un-benches
+        // at expiry) instead of the permanent auth-disable that stranded it until restart.
+        // `neutral` release so a transient quota-403 doesn't bump failure/scoring counters.
+        accountManager.markRateLimited(account.index, PROVIDER_FORBIDDEN_COOLDOWN_SEC, { status: 403, recordFailure: false });
+        accountManager.releaseAccount(lease, { neutral: true });
+      } else {
+        // 401 = genuinely bad/missing credentials → permanent disable (needs re-auth).
+        accountManager.markAuthFailed(account.index, upstreamRes.status, 'auth_failed');
+        accountManager.releaseAccount(lease, { status: upstreamRes.status, error: 'auth_failed' });
+      }
       excludedIndexes.add(account.index);
 
       if (logDir) {
-        logSections.push(`=== RESPONSE ${upstreamRes.status} — "${account.name}" disabled (${reason}), failing over ===\n${formatHeaders(upstreamRes.headers)}`);
+        logSections.push(`=== RESPONSE ${upstreamRes.status} — "${account.name}" ${forbidden ? `cooled down ${PROVIDER_FORBIDDEN_COOLDOWN_SEC}s (quota/forbidden)` : 'disabled (auth)'}, failing over ===\n${formatHeaders(upstreamRes.headers)}`);
         if (errorBody) logSections.push(`=== ERROR BODY ===\n${errorBody}`);
       }
-      console.log(`[Maxpool] ${upstreamRes.status} on provider "${account.name}" — disabled and failing over before first byte`);
+      console.log(`[Maxpool] ${upstreamRes.status} on provider "${account.name}" — ${forbidden ? `cooled down ${PROVIDER_FORBIDDEN_COOLDOWN_SEC}s (recoverable)` : 'disabled'} and failing over before first byte`);
 
       if (
         canRetryBufferedBody &&
@@ -1074,6 +1100,7 @@ function formatRetryDuration(seconds) {
 function computeQueueWindowMs({
   cause, stream, retryPlanCause,
   maxWaitMs, capacityMaxWaitMs, nonStreamMaxWaitMs, streamHoldMaxMs,
+  isCountTokens, countTokensMaxWaitMs,
 }) {
   let windowMs;
   if (!stream) {
@@ -1084,6 +1111,10 @@ function computeQueueWindowMs({
     windowMs = streamHoldMaxMs;
   }
   if (retryPlanCause === 'concurrency_cap') windowMs = Math.min(windowMs, capacityMaxWaitMs);
+  // count_tokens: cap the QUEUE wait low (bounds only the wait-for-an-account, never
+  // the upstream processing once acquired) so a non-heartbeated metadata call fast-
+  // fails with a retryable 429 instead of hanging past the client's idle window.
+  if (isCountTokens && countTokensMaxWaitMs != null) windowMs = Math.min(windowMs, countTokensMaxWaitMs);
   return windowMs;
 }
 
@@ -1371,6 +1402,9 @@ async function queueAndRetry(
   const nonStreamMaxWaitMs = queueConfig.nonStreamMaxWaitMs == null
     ? 5 * 60_000
     : Math.max(0, Number(queueConfig.nonStreamMaxWaitMs) || 0);
+  const countTokensMaxWaitMs = queueConfig.countTokensMaxWaitMs == null
+    ? 8000
+    : Math.max(0, Number(queueConfig.countTokensMaxWaitMs) || 0);
   const retryPlan = accountManager.nextRetryForRequest?.(requestInfo, new Set()) || {
     retryAfterMs: Infinity,
     cause: 'unavailable',
@@ -1405,6 +1439,8 @@ async function queueAndRetry(
     capacityMaxWaitMs,
     nonStreamMaxWaitMs,
     streamHoldMaxMs,
+    isCountTokens: Boolean(requestInfo.isCountTokens),
+    countTokensMaxWaitMs,
   });
 
   if (queueWindowMs <= 0) return finishQueuedStreamIfNeeded(res, requestInfo, honestMessage);
@@ -1575,6 +1611,11 @@ function describeRequest(req, body) {
     path: req.url,
     bodyBytes: body.length,
     weight,
+    // count_tokens is cheap non-streaming metadata with NO SSE heartbeat, so a long
+    // queue-hold just dies on the client's idle window ("Stream idle timeout - no
+    // chunks received"). Flag it (URL-based, so it's set regardless of body parse) to
+    // cap its queue wait short and fast-fail with a retryable 429 instead of hanging.
+    isCountTokens: /\/v1\/messages\/count_tokens\b/.test(req.url),
   };
   try {
     const json = JSON.parse(body.toString());