maxpool 1.5.44 → 1.5.46

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.5.44",
+  "version": "1.5.46",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/account-manager.js

@@ -159,6 +159,10 @@ const DEFAULT_SCHEDULER = {
   providerCrossFallback: true,
 };
 const LOAD_EVENT_MAX_AGE_MS = 60 * 60 * 1000;
+// Consecutive failed recovery probes before we stop trusting the SHARED breaker and
+// fall back to per-account handling. Bounds any future "poisoned probe" from becoming
+// an indefinite fleet-wide outage (2026-07-25 incident).
+const MAX_FAILED_PROBES = 4;
 const WEEK_MS = 7 * 24 * 60 * 60 * 1000;
 const FIVE_HOUR_MS = 5 * 60 * 60 * 1000;
 
@@ -734,6 +738,7 @@ export class AccountManager {
     this.upstreamThrottle.until = Math.max(this.upstreamThrottle.until || 0, until);
     this.upstreamThrottle.reason = reason;
     this.upstreamThrottle.probeInFlight = false;
+    this.upstreamThrottle.failedProbes = 0;   // fresh breaker → fresh probe budget
     this.upstreamThrottle.count++;
     this.upstreamThrottle.lastAt = Date.now();
     console.log(`[Maxpool] Anthropic upstream temporarily limiting requests for ${retryAfter}s; pausing Claude routes`);
@@ -744,6 +749,7 @@ export class AccountManager {
     this.upstreamThrottle.until = null;
     this.upstreamThrottle.reason = null;
     this.upstreamThrottle.probeInFlight = false;
+    this.upstreamThrottle.failedProbes = 0;
     this.queueState.rampUntil = Date.now() + 5000;
     this.queueState.lastAdmissionAt = Date.now();
     console.log(`[Maxpool] Anthropic upstream throttle cleared (${reason})`);
@@ -755,14 +761,36 @@ export class AccountManager {
     lease.upstreamThrottleProbe = false;
   }
 
+  /** Hand the recovery probe back UNUSED — the request never got an upstream answer
+   *  (client disconnected, token refresh failed), so it is no evidence either way.
+   *  Leaves the window open so the very next request can claim the probe, instead of
+   *  scoring a failure that would re-arm the fleet-wide throttle. */
+  relinquishUpstreamProbe(lease) {
+    if (!lease?.upstreamThrottleProbe) return;
+    lease.upstreamThrottleProbe = false;
+    this.upstreamThrottle.probeInFlight = false;
+  }
+
   deferUpstreamThrottleProbe(retryAfterSeconds = 5, reason = 'probe_failed') {
     if (!this.upstreamThrottle.until && !this.upstreamThrottle.probeInFlight) return;
-    const retryAfter = clampRetryAfterSeconds(retryAfterSeconds);
+    // PROBE BUDGET. A flat 5s retry with unbounded repetition is what let a single
+    // poisoned request hold the whole fleet down forever. Escalate the backoff, and
+    // after MAX_FAILED_PROBES consecutive failures give up on the shared breaker and
+    // fall back to per-account handling — the fleet then retries for real, and if
+    // Anthropic genuinely is throttling, shouldPromoteUpstreamFailure re-arms it.
+    // (This is a circuit breaker's half-open state, paced by the backoff.)
+    this.upstreamThrottle.failedProbes = (this.upstreamThrottle.failedProbes || 0) + 1;
+    if (this.upstreamThrottle.failedProbes >= MAX_FAILED_PROBES) {
+      this.clearUpstreamThrottle(`probe budget exhausted after ${this.upstreamThrottle.failedProbes} failures (${reason}) — deferring to per-account handling`);
+      return;
+    }
+    const backoff = Math.min(60, retryAfterSeconds * 2 ** (this.upstreamThrottle.failedProbes - 1));
+    const retryAfter = clampRetryAfterSeconds(backoff);
     this.upstreamThrottle.until = Date.now() + retryAfter * 1000;
     this.upstreamThrottle.reason = reason;
     this.upstreamThrottle.probeInFlight = false;
     this.upstreamThrottle.lastAt = Date.now();
-    console.log(`[Maxpool] Anthropic recovery probe failed; retrying in ${retryAfter}s (${reason})`);
+    console.log(`[Maxpool] Anthropic recovery probe failed (${this.upstreamThrottle.failedProbes}/${MAX_FAILED_PROBES}); retrying in ${retryAfter}s (${reason})`);
   }
 
   noteAmbiguousRateLimit(accountIndex, fingerprint, _retryAfterSeconds) {

package/src/server.js

@@ -402,6 +402,11 @@ async function forwardRequest(
   res.once('close', onClientClose);
   const releaseOnClientGone = () => {
     res.off('close', onClientClose);
+    // The client vanished — we never got an upstream answer, so this carries ZERO
+    // evidence about Anthropic's health. Hand the recovery probe back instead of
+    // letting releaseAccount score it as a FAILED probe (which would re-arm the
+    // fleet-wide throttle for 5s on every disconnect — a second deadlock amplifier).
+    accountManager.relinquishUpstreamProbe?.(lease);
     accountManager.releaseAccount(lease);
     clearQueueHeartbeat(requestInfo);
     accountManager.removeQueuedRequest?.(requestInfo);
@@ -421,6 +426,9 @@ async function forwardRequest(
     // (MaxListenersExceededWarning + leak); the recursive/resumed frame registers
     // its own.
     res.off('close', onClientClose);
+    // Token refresh failed — the request never reached Anthropic, so this is no
+    // evidence about upstream health. Relinquish rather than fail the probe.
+    accountManager.relinquishUpstreamProbe?.(lease);
     accountManager.releaseAccount(lease);
     excludedIndexes.add(account.index);
     if (
@@ -563,6 +571,18 @@ async function forwardRequest(
     }
     accountManager.updateQuota(account.index, rateLimitHeaders);
 
+    // SETTLE THE SHARED BREAKER AT THE HEADER BOUNDARY. Anthropic ANSWERED, so unless
+    // the status is itself a capacity signal, the upstream is provably reachable and
+    // serving — a per-request verdict (400 bad transcript, 401, 404, 413, 422) says
+    // NOTHING about capacity and must never keep the fleet-wide throttle armed.
+    // Without this, a poisoned request (e.g. a provider-authored thinking block that
+    // Anthropic 400s) claimed the recovery probe, "failed" it, re-armed the shared
+    // throttle every 5s, and benched EVERY healthy account indefinitely — a hard
+    // production-down deadlock (2026-07-25: 6 of 13 re-arms were client-side 400s
+    // while accounts sat at 2%/11%/25% weekly). confirmUpstreamProbe also clears
+    // lease.upstreamThrottleProbe, so the releaseAccount probe branch below no-ops.
+    if (!isCapacitySignalStatus(upstreamRes.status)) accountManager.confirmUpstreamProbe?.(lease);
+
     // Retry/failover can only happen before response bytes are sent. Once a
     // streaming response starts, rerouting would corrupt Claude Code's stream.
     if (upstreamRes.status === 429) {
@@ -872,6 +892,12 @@ async function forwardRequest(
       // 262144". Detect it ONLY on a provider (a Claude account's context-length 400 is
       // terminal — nothing bigger to fall to) so we can pin the session to Claude.
       const providerTooSmall = account.type === 'provider' && isContextLengthError(errorBody);
+      // DETERMINISTIC signature rejection (exact Anthropic wording) — the only trigger
+      // for the strip-and-recover retry below. Deliberately NOT the fuzzy
+      // isAnthropicIncompatBody heuristic, so a merely malformed request can never cause
+      // us to rewrite a user's transcript.
+      const isSignatureRejection = account.type !== 'provider'
+        && /invalid `signature` in `thinking`/i.test(errorBody);
       const errorType = errorBody.includes('Invalid `signature` in `thinking` block')
         ? 'invalid_thinking_signature'
         : anthropicIncompat ? 'anthropic_incompatible_transcript'
@@ -906,6 +932,26 @@ async function forwardRequest(
         }
       }
 
+      // RECOVER-ON-CLAUDE (preferred over the provider pin below): the transcript
+      // carries provider-authored thinking blocks whose signature Anthropic rejects.
+      // Strip exactly those blocks and retry on Claude, so a session that took even one
+      // GLM/Kimi fallback turn is NOT bricked (or exiled to a provider) for the rest of
+      // its life. Verified against the real API — the stripped history returns 200, with
+      // text + tool_use preserved. Tried once per request (thinkingStripped guard); if it
+      // still fails, the provider pin below is the fallback.
+      if (isSignatureRejection && !requestInfo.thinkingStripped
+        && canRetryBufferedBody && retryCount + 1 < maxAttempts && !res.headersSent) {
+        const { body: cleanBody, removed } = stripForeignThinkingBlocks(body);
+        if (cleanBody) {
+          console.log(`[Maxpool] Recovering session on Claude: stripped ${removed} provider-authored thinking block(s) Anthropic rejected`);
+          return forwardRequest(
+            req, res, cleanBody, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
+            retryConfig, queueConfig, { ...requestInfo, thinkingStripped: true },
+            canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
+          );
+        }
+      }
+
       // React-and-heal: this transcript can't run on Claude (foreign server_tool_use
       // id / thinking Anthropic can't validate). The 400 is pre-stream and the body
       // is buffered, so latch the session Anthropic-incompatible (sticky → once per
@@ -951,13 +997,44 @@ async function forwardRequest(
         );
       }
 
+      // Nothing could heal it — replace the cryptic upstream 400 with the real cause and
+      // the actual way out. This is what the user saw for hours as a bare
+      // "400 messages.51.content.8: Invalid `signature` in `thinking` block".
+      // Trigger on the DETERMINISTIC signature rejection only — never the fuzzy
+      // isAnthropicIncompatBody heuristic, so an unrelated malformed request keeps its
+      // own upstream error instead of being mislabelled a provider-contamination.
+      if (isSignatureRejection && !res.headersSent) {
+        const provs = (accountManager.accounts || []).filter(a => a.type === 'provider');
+        // Say only what ACTUALLY happened. `thinkingStripped` is set only when the strip
+        // ran; without it we found nothing provider-shaped, so claiming we stripped —
+        // or blaming GLM/Kimi — would misdirect the user.
+        const what = requestInfo.thinkingStripped
+          ? 'This session ran on GLM/Kimi earlier and Anthropic rejects the thinking blocks they wrote. Maxpool removed them and retried on Claude, but Anthropic still rejected the history.'
+          : "Anthropic rejected a thinking block in this session's history, and maxpool could not identify it as provider-authored, so it could not be repaired automatically.";
+        const hint = provs.length === 0
+          ? ''
+          : provs.every(a => a.enabled === false)
+            ? ' If this session previously ran on GLM/Kimi, re-enabling that provider in the maxpool TUI (a → t) lets it continue there.'
+            : '';
+        ctx.status = 400;
+        sendErrorResponse(res, requestInfo, 400, {
+          type: 'error',
+          error: {
+            type: 'invalid_request_error',
+            message: `Start a new session to keep working — this one cannot continue. ${what}${hint}`,
+          },
+        });
+        return;
+      }
+
       ctx.status = upstreamRes.status;
       sendErrorBody(res, requestInfo, upstreamRes.status, errorBody, upstreamRes.headers);
       return;
     }
 
     if (upstreamRes.status < 400) {
-      accountManager.confirmUpstreamProbe?.(lease);
+      // (the probe was already settled at the header boundary above — this only
+      // records the account-level "upstream accepted us" signal)
       accountManager.markUpstreamAccepted?.(account.index);
     }
 
@@ -1257,7 +1334,7 @@ function isContextLengthError(errorBody) {
   return /exceeded model token limit|maximum context length|context length exceeded|context window (?:size )?(?:exceeded|too)|prompt is too long|input is too long|reduce the length of|too many (?:input )?tokens|request too large/i.test(errorBody);
 }
 
-export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, streamResponse, startIdleRequestReaper };
+export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, isCapacitySignalStatus, isStrippableThinkingBlock, stripForeignThinkingBlocks, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, commitStreamGraceHeartbeat, describeRequest, classifyRateLimit, detectTranscriptOrigin, isAnthropicIncompatBody, isContextLengthError, streamResponse, startIdleRequestReaper };
 
 async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
   if (!upstreamRes.body) return '';
@@ -1433,6 +1510,74 @@ function secondsUntilParsedTime(value) {
   return null;
 }
 
+// Provider thinking signatures CANNOT be told apart from Anthropic's by shape — measured
+// against the live APIs 2026-07-25: Anthropic ~200-400 char base64, GLM a 24-char hex
+// digest, Kimi(K3) a 12,946-char base64 blob. A shape heuristic tuned to GLM silently
+// missed Kimi entirely. So the repair below does NOT guess: it strips EVERY `thinking`
+// block. That is safe precisely because it only ever runs in response to Anthropic's own
+// signature-rejection 400 — a healthy Claude-only session never reaches it — and a history
+// with thinking removed is accepted (200 OK, verified, incl. tool_use → tool_result).
+// `redacted_thinking` is never touched: it legitimately carries `data` and no signature.
+function isStrippableThinkingBlock(block) {
+  return block?.type === 'thinking';
+}
+
+/**
+ * Recovery for a provider-contaminated transcript: drop the assistant `thinking` /
+ * `redacted_thinking` blocks whose signature Anthropic can't validate, so the session
+ * can CONTINUE ON CLAUDE instead of being pinned to a provider forever.
+ *
+ * Empirically verified 2026-07-25 against the real API: replaying a GLM-authored
+ * thinking block to Anthropic returns 400; the SAME history with those blocks removed
+ * returns 200 — including an assistant turn carrying `tool_use` followed by a
+ * `tool_result` (thinking blocks are not required on replay for a new turn). Text and
+ * tool_use blocks are preserved, so no conversation content or tool wiring is lost.
+ *
+ * Returns { body, removed } — `body` is null when nothing needed stripping.
+ */
+function stripForeignThinkingBlocks(body) {
+  try {
+    const json = JSON.parse(Buffer.isBuffer(body) ? body.toString('utf8') : String(body));
+    if (!Array.isArray(json?.messages)) return { body: null, removed: 0 };
+    let removed = 0;
+    const messages = [];
+    for (const msg of json.messages) {
+      if (msg?.role !== 'assistant' || !Array.isArray(msg.content)) { messages.push(msg); continue; }
+      let localRemoved = 0;
+      const kept = msg.content.filter(block => {
+        if (!isStrippableThinkingBlock(block)) return true;
+        localRemoved++;
+        return false;
+      });
+      removed += localRemoved;
+      if (!localRemoved) { messages.push(msg); continue; }
+      // A turn that stripping empties is DROPPED, not left with its poisoned block:
+      // leaving it would resend the exact body that just 400'd while reporting success
+      // and burning the single recovery attempt. Verified against the live API — the
+      // resulting consecutive user messages are accepted (200 OK).
+      if (kept.length === 0) continue;
+      messages.push({ ...msg, content: kept });
+    }
+    if (!removed) return { body: null, removed: 0 };
+    json.messages = messages;
+    return { body: Buffer.from(JSON.stringify(json)), removed };
+  } catch {
+    return { body: null, removed: 0 };   // non-JSON / unparseable → no rewrite
+  }
+}
+
+/**
+ * Is this status a CAPACITY signal (i.e. evidence the upstream can't serve us right
+ * now), as opposed to a per-request verdict? Only these may keep the shared Anthropic
+ * throttle armed. 403 is included deliberately: on these plans a 403 is almost always
+ * quota/plan exhaustion, NOT bad credentials (see the provider-auth handler). 408 is a
+ * latency/capacity signal. Everything else in 4xx (400/401/404/413/422…) means Anthropic
+ * answered a specific request — the upstream is alive.
+ */
+function isCapacitySignalStatus(status) {
+  return status === 429 || status === 403 || status === 408 || status >= 500;
+}
+
 function isRetriableUpstreamStatus(status) {
   // 500 included: Anthropic 500s are transient server errors (same class as
   // 502/503/504). Without this they were passed straight through to the client

package/src/tui.js

@@ -253,6 +253,7 @@ export class TUI {
     // Set by index.js after construction (a deferred closure, not a constructor literal —
     // avoids the const TDZ on applyUpdateIfReady, which is defined after `new TUI`).
     this.checkNow = null;
+    this.updateBusy = null;   // live progress text while an update check/apply runs
 
     this.log = [];           // completed activity entries
     this.active = new Map(); // in-flight requests
@@ -456,10 +457,20 @@ export class TUI {
   _keyUpdates(k) {
     if (k === 'c') {
       // Check & apply now — the dance-killer: pull the latest + seamless-reload in place,
-      // no quit/relaunch. index.js wires this.checkNow (applies regardless of autoApply).
-      this.mode = 'normal';
-      if (this.checkNow) this.checkNow();
-      else this._addLog('Update check unavailable on this worker');
+      // no quit/relaunch. STAY on this screen and show live progress: bouncing straight
+      // back to the dashboard was indistinguishable from "nothing happened".
+      if (!this.checkNow) { this._addLog('Update check unavailable on this worker'); return; }
+      if (this.updateBusy) return;                       // already running
+      this.updateBusy = 'Checking npm…';
+      this.render();
+      Promise.resolve(this.checkNow())
+        .catch(() => {})
+        .finally(() => {
+          // If an update was found+applied, the seamless reload replaces this worker and
+          // this never renders. Otherwise report the outcome in place.
+          this.updateBusy = null;
+          if (this.mode === 'updates') this.render();
+        });
     } else if (k === 't') {
       this._toggleAutoUpdate();
     } else if (k === 'esc' || k === 'q') {
@@ -467,6 +478,29 @@ export class TUI {
     }
   }
 
+  /** The visible Updates panel. Answers, at a glance: what am I running, is anything
+   *  newer, is it automatic, and what is happening right now. */
+  _renderUpdatesDetail() {
+    const v = this.am.versionInfo;
+    const running = v?.current ? `v${v.current}` : 'unknown';
+    const auto = this._autoUpdateOn();
+    const out = [];
+    out.push(` ${bold('Updates')}`);
+    if (!v) {
+      out.push(` ${dim('Running')} ${running}   ${dim('· checking npm for a newer version…')}`);
+    } else if (v.hasUpdate && v.latest) {
+      out.push(` ${dim('Running')} ${running}   ${yellow(`v${v.latest} is available`)}`);
+      out.push(auto
+        ? ` ${dim('It installs itself automatically. Press')} ${bold('c')} ${dim('to get it right now.')}`
+        : ` ${dim('Automatic updates are off. Press')} ${bold('c')} ${dim('to install it now, or')} ${bold('t')} ${dim('to turn automatic on.')}`);
+    } else {
+      out.push(` ${dim('Running')} ${running}   ${green('up to date')}`);
+      out.push(` ${dim(auto ? 'New versions install themselves.' : 'Automatic updates are off — press t to turn them on.')}`);
+    }
+    if (this.updateBusy) out.push(` ${cyan(SPINNER[this.frame])} ${dim(this.updateBusy)}`);
+    return out;
+  }
+
   async _toggleAutoUpdate() {
     const turnOn = !this._autoUpdateOn();
     // ON = the full hands-free chain; OFF = keep checking (banner still shows) but never
@@ -1158,7 +1192,10 @@ export class TUI {
     }
 
     // Completed log
-    const footerH = this.mode === 'confirm' ? 3 : 2;
+    // 2 = separator + footer; confirm adds its detail line; updates adds its detail block.
+    const footerH = this.mode === 'confirm' ? 3
+      : this.mode === 'updates' ? 2 + this._renderUpdatesDetail().length
+      : 2;
     const space = Math.max(0, H - lines.length - footerH);
     for (let i = 0; i < space && i < this.log.length; i++) {
       lines.push(`   ${gray(this.log[i].t)}  ${this.log[i].msg}`);
@@ -1170,6 +1207,10 @@ export class TUI {
     // ── Footer
     lines.push(' ' + dim('─'.repeat(W - 2)));
     if (this.mode === 'confirm') lines.push(` ${this.confirmDetail}`);
+    // Updates DETAIL — without this, pressing 'u' changed only the one-line footer at the
+    // very bottom of a busy screen, which reads as "nothing happened" (reported twice).
+    // Every other mode draws something visible; this one must too.
+    if (this.mode === 'updates') lines.push(...this._renderUpdatesDetail());
     lines.push(this._renderFooter());
 
     // Write buffer