maxpool 1.5.9 → 1.5.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.5.9",
+  "version": "1.5.11",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/account-manager.js

@@ -75,10 +75,12 @@ const DEFAULT_SCHEDULER = {
   // (never a provider — GLM/Kimi can't validate an Anthropic signature). Anthropic
   // thinking-block signatures are content/model integrity, NOT account-bound —
   // verified empirically 2026-07-02 (a `partnerships`-signed block replayed under
-  // `personal` returned 200). DEFAULT OFF until the revert-to-issuer fail-safe lands
-  // (issue #16): with it off, a heavy thinking session still can't strand an account,
-  // but it also can't poison a session if Anthropic ever changes the contract.
-  crossAccountThinkingMigration: false,
+  // `personal` returned 200). ON by default: a heavy thinking session can now spread
+  // its later load onto fresh accounts instead of stranding one. The revert-to-issuer
+  // fail-safe (server.js, on `invalid_thinking_signature` for a migrated request)
+  // makes this safe even if Anthropic ever account-binds signatures — a rejected
+  // replay self-heals to the issuer instead of poisoning the session.
+  crossAccountThinkingMigration: true,
 };
 const LOAD_EVENT_MAX_AGE_MS = 60 * 60 * 1000;
 const WEEK_MS = 7 * 24 * 60 * 60 * 1000;
@@ -438,6 +440,12 @@ export class AccountManager {
 
   acquireAccount(requestInfo = {}, excludedIndexes = new Set()) {
     this._noteRequestPolicy(requestInfo);
+    // Capture the session's prior account BEFORE selection, so the caller can tell
+    // whether this acquire MOVED the session (the thinking-signature fail-safe needs
+    // the pre-migration issuing account to revert to).
+    const prevCurrentName = requestInfo.sessionKey
+      ? this._sessionBinding(requestInfo.sessionKey)?.currentName
+      : null;
     const account = this.getActiveAccount(requestInfo, excludedIndexes);
     if (!account) return null;
 
@@ -449,7 +457,22 @@ export class AccountManager {
     account.inFlight++;
     account.activeWeight += weight;
     account.lastUsedAt = Date.now();
-    return { account, weight, startedAt: Date.now(), upstreamThrottleProbe };
+    // Non-null only when this acquire moved the session off its prior account.
+    const migratedFromName = (prevCurrentName && account.name !== prevCurrentName) ? prevCurrentName : null;
+    return { account, weight, startedAt: Date.now(), upstreamThrottleProbe, migratedFromName };
+  }
+
+  /** Fail-safe: snap a session's binding back to the pre-migration issuing account
+   *  after a rejected cross-account thinking replay, so the retry (and future
+   *  requests) route to the account that actually generated the thinking blocks.
+   *  Defensive — signatures are portable in practice (verified 2026-07-02); this
+   *  only fires if Anthropic ever account-binds them. */
+  revertSessionBinding(sessionKey, name) {
+    if (!sessionKey || !name) return;
+    const binding = this._sessionBinding(sessionKey);
+    if (!binding) return;
+    binding.currentName = name;
+    this.sessionBindings.set(sessionKey, binding);
   }
 
   releaseAccount(lease, outcome = {}) {
@@ -1136,6 +1159,22 @@ export class AccountManager {
     const profile = requestInfo.profile || 'claude';
     const scoringCtx = this._scoringContext();
 
+    // Fail-safe retry pin: steer this request's remaining retry/queue chain onto a
+    // specific account (the pre-migration issuer, after a cross-account thinking
+    // replay was rejected). Honored ahead of everything else, but FALLS THROUGH to
+    // normal selection whenever that account is excluded/unavailable — a down issuer
+    // never strands the request (the retry then re-migrates and terminates via
+    // excludedIndexes + maxAttempts). See the thinking-signature fail-safe in server.js.
+    if (requestInfo.pinnedAccountName) {
+      const pinned = this.accounts.find(a => a.name === requestInfo.pinnedAccountName);
+      if (pinned && !excludedIndexes.has(pinned.index)
+        && this._matchesRequest(pinned, profile, requestInfo)
+        && this._isAvailable(pinned, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
+        this.currentIndex = pinned.index;
+        return pinned;
+      }
+    }
+
     const hasBinding = Boolean(requestInfo.sessionKey && this.sessionBindings.has(requestInfo.sessionKey));
     const preferred = this._preferredAccount(profile, excludedIndexes, requestInfo);
     if (preferred) {

package/src/config.js

@@ -111,7 +111,7 @@ export function createDefaultConfig() {
       enabled: true,
       maxWaitMs: 24 * 60 * 60 * 1000,    // hard ceiling for non-streaming/capacity holds; streaming uses streamHoldMaxMs
       autoMaxWaitMs: null,               // 5h/session-cap hold (null = maxWaitMs)
-      capacityMaxWaitMs: 15 * 60 * 1000, // upstream 529/overload — stays short, never governed by the others
+      capacityMaxWaitMs: 15 * 60 * 1000, // short cap for NON-streaming capacity holds + the concurrency-cap clamp. A STREAMING capacity/throttle hold uses maxWaitMs (held on the heartbeat until capacity frees), NOT this.
       weeklyMaxWaitMs: 24 * 60 * 60 * 1000, // legacy bound; streaming holds use streamHoldMaxMs
       // Streaming hold ceiling: how long a streaming session is held ALIVE on the
       // heartbeat waiting for any account to free up. 7d so a session is never

package/src/index.js

@@ -139,6 +139,10 @@ async function serverCommand() {
 
 async function supervisorCommand() {
   const { createServer } = await import('node:net');
+  // Test-only: the restart integration test delivers a GROUP SIGUSR2 to drive the
+  // worker's restart path; ignore it on the supervisor so the group signal doesn't
+  // kill it (default SIGUSR2 action is terminate). Gated — never active normally.
+  if (process.env.MAXPOOL_TEST_RESTART_SIGNAL === '1') process.on('SIGUSR2', () => {});
   const config = await loadOrCreateConfig();
   initEventLog(config, { manageRotation: true }); // supervisor is the single rotation owner
   const port = config.proxy.port;
@@ -733,8 +737,19 @@ async function serverWorkerCommand() {
   restartController = new RestartController({
     pauseAdmission: () => accountManager.setAdmissionPaused(true),
     restartNow: requestReload,
+    // Configurable so ops can tune the bounded pre-restart drain, and so the
+    // integration test can exercise the force-restart path without a 10s wait.
+    ...(Number.isFinite(config.restartDrainTimeoutMs) ? { drainTimeoutMs: config.restartDrainTimeoutMs } : {}),
   });
 
+  // Test-only: drive the real interactive restart path (the TUI `r` key) headless,
+  // so the end-to-end "restart while requests are in flight completes bounded and
+  // the server comes back" is provable without allocating a pty. Gated — never
+  // active in normal runs.
+  if (process.env.MAXPOOL_TEST_RESTART_SIGNAL === '1') {
+    process.on('SIGUSR2', () => restartController?.requestRestart());
+  }
+
   const shutdownGracefully = (reason, options = {}) => {
     // A terminal-close shutdown must NEVER exit non-zero: the supervisor reads a
     // fast non-zero exit as a crash and respawns a fresh worker — onto the now-dead

package/src/server.js

@@ -746,6 +746,24 @@ async function forwardRequest(
       }
       if (errorType === 'invalid_thinking_signature') {
         console.log(`[Maxpool] Non-retryable Anthropic thinking signature error on "${account.name}"`);
+        // Fail-safe: if this request had been MIGRATED to a different Claude account
+        // this turn (cross-account thinking rebalance), the rejected block was issued
+        // by the PRE-MIGRATION account. Revert the session there, exclude the failed
+        // target, and retry PINNED to the issuer — so a rejected cross-account replay
+        // self-heals instead of poisoning the session into a 400 loop. Signatures are
+        // portable in practice (verified 2026-07-02); this only fires if Anthropic
+        // ever account-binds them.
+        if (lease.migratedFromName && requestInfo.sessionKey
+          && canRetryBufferedBody && retryCount + 1 < maxAttempts && !res.headersSent) {
+          accountManager.revertSessionBinding(requestInfo.sessionKey, lease.migratedFromName);
+          excludedIndexes.add(account.index);
+          console.log(`[Maxpool] Thinking-signature fail-safe: reverting session to issuer "${lease.migratedFromName}" and retrying`);
+          return forwardRequest(
+            req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir,
+            retryConfig, queueConfig, { ...requestInfo, pinnedAccountName: lease.migratedFromName },
+            canRetryBufferedBody, canQueueBufferedBody, excludedIndexes,
+          );
+        }
       }
 
       ctx.status = upstreamRes.status;
@@ -934,6 +952,38 @@ function formatRetryDuration(seconds) {
   return `${s}s`;
 }
 
+/**
+ * Hold-window ceiling for a queued request. The soonest-recovery oracle
+ * (nextRetryForRequest) is the real governor — this is the backstop ceiling that
+ * bounds how long a request may WAIT for that recovery.
+ *
+ *   non-streaming (no heartbeat)  → short cap (would die on the client's own timeout)
+ *   streaming + capacity/429/throttle → maxWaitMs (24h): HELD on the SSE heartbeat
+ *       until an account frees. This is the "hold, don't fail" fix — a throttle clears
+ *       in seconds and a 5h session-cap in hours, both far under 24h; only a multi-day
+ *       weekly reset exceeds it and error-fasts with the honest "add an account"
+ *       message. Previously capped at the short capacityMaxWaitMs (15m), which failed
+ *       the MOST-clearly-temporary throttle sooner than an ordinary per-account 429.
+ *   streaming + other (quota)     → streamHoldMaxMs (7d)
+ *   retryPlanCause==='concurrency_cap' → clamped to the short capacity window (a LOCAL
+ *       transient: a slot frees in seconds; never spin a multi-day hold on it).
+ */
+function computeQueueWindowMs({
+  cause, stream, retryPlanCause,
+  maxWaitMs, capacityMaxWaitMs, nonStreamMaxWaitMs, streamHoldMaxMs,
+}) {
+  let windowMs;
+  if (!stream) {
+    windowMs = cause === 'capacity' ? Math.min(nonStreamMaxWaitMs, capacityMaxWaitMs) : nonStreamMaxWaitMs;
+  } else if (cause === 'capacity') {
+    windowMs = maxWaitMs;
+  } else {
+    windowMs = streamHoldMaxMs;
+  }
+  if (retryPlanCause === 'concurrency_cap') windowMs = Math.min(windowMs, capacityMaxWaitMs);
+  return windowMs;
+}
+
 function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRecoverSoon = true) {
   const thinking = requestInfo.requiresAnthropicThinkingIntegrity
     || accountManager._requiresAnthropicThinkingIntegrity?.(requestInfo);
@@ -958,7 +1008,7 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
   return `All ${n} accounts exhausted. Retry in ${retryAfter}s.`;
 }
 
-export const __serverTest = { unavailableMessage, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, describeRequest };
+export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, describeRequest };
 
 async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
   if (!upstreamRes.body) return '';
@@ -1213,33 +1263,15 @@ async function queueAndRetry(
   const streamHoldMaxMs = queueConfig.streamHoldMaxMs == null
     ? 7 * 24 * 60 * 60 * 1000
     : Math.max(0, Number(queueConfig.streamHoldMaxMs) || 0);
-  // Pick the hold ceiling:
-  //   capacity (upstream 529/overload) → its own short cap, never a long hold
-  //   non-streaming (no heartbeat)      → short cap (would die on client timeout)
-  //   streaming                         → up to streamHoldMaxMs (7d), kept alive
-  //                                       by the heartbeat
-  let queueWindowMs;
-  if (cause === 'capacity') {
-    queueWindowMs = Math.min(maxWaitMs, capacityMaxWaitMs);
-  } else if (!requestInfo.stream) {
-    queueWindowMs = nonStreamMaxWaitMs;
-  } else {
-    queueWindowMs = streamHoldMaxMs;
-  }
-  // A non-streaming request has no heartbeat regardless of cause, so it must
-  // never outlast nonStreamMaxWaitMs even under capacity (it would occupy a
-  // slot 3x its documented cap with nothing to reap it).
-  if (!requestInfo.stream) queueWindowMs = Math.min(queueWindowMs, nonStreamMaxWaitMs);
-
-  // A pure concurrency-cap block (every account healthy but all in-flight/global
-  // slots busy — NOT a quota/rate-limit reset) is a LOCAL capacity transient. Bound
-  // it by the short capacity window, never the multi-day streaming hold: a slot
-  // frees as active requests finish (seconds–minutes), and if the fleet stays
-  // saturated past the window the request sheds load (error-fast) instead of
-  // spinning a queue slot for up to streamHoldMaxMs (7d) — the soft-deadlock guard.
-  if (retryPlan.cause === 'concurrency_cap') {
-    queueWindowMs = Math.min(queueWindowMs, capacityMaxWaitMs);
-  }
+  const queueWindowMs = computeQueueWindowMs({
+    cause,
+    stream: Boolean(requestInfo.stream),
+    retryPlanCause: retryPlan.cause,
+    maxWaitMs,
+    capacityMaxWaitMs,
+    nonStreamMaxWaitMs,
+    streamHoldMaxMs,
+  });
 
   if (queueWindowMs <= 0) return finishQueuedStreamIfNeeded(res, requestInfo, honestMessage);
 

package/src/tui.js

@@ -925,8 +925,19 @@ export class TUI {
     if (a.enabled !== false && upstreamBlocking && a.status === 'active') {
       effectiveStatus = a.inFlight > 0 ? 'probing' : 'waiting';
     }
+    // Anthropic is actively REJECTING this account right now (e.g. a per-model weekly
+    // sub-limit the general utilization % doesn't expose). It's unusable — surface
+    // that instead of a benign green "active", so a low weekly % (the bar keeps its
+    // true value) is never misread as available headroom.
+    // Keys on a.status (not effectiveStatus) so a rejected account reads 'blocked'
+    // even inside an upstream-throttle window (where it would otherwise show
+    // probing/waiting) — a rejected account is unusable, not part of the recovery.
+    if (a.enabled !== false && a.quota?.unifiedStatus === 'rejected' && a.status === 'active') {
+      effectiveStatus = 'blocked';
+    }
     switch (effectiveStatus) {
       case 'active':    status = isCur ? green('active') : 'active'; break;
+      case 'blocked':   status = red('blocked'); break;
       case 'probing':   status = green('probing'); break;
       case 'waiting':   status = yellow('waiting'); break;
       case 'paused':    status = yellow('paused'); break;