maxpool 1.0.7 → 1.2.0

package/README.md

@@ -299,8 +299,11 @@ TEAMCLAUDE_CONFIG=./my-config.json maxpool server
     "maxWaitMs": 86400000,
     "autoMaxWaitMs": null,
     "capacityMaxWaitMs": 900000,
+    "weeklyMaxWaitMs": 86400000,
+    "nonStreamMaxWaitMs": 300000,
+    "maxConcurrentQueued": 64,
+    "maxQueuedBytes": 1073741824,
     "maxQueuedBodyBytes": 268435456,
-    "weeklyMaxWaitMs": 0,
     "pollMs": 1000
   },
   "shutdown": {
@@ -338,7 +341,10 @@ TEAMCLAUDE_CONFIG=./my-config.json maxpool server
 | `queue.autoMaxWaitMs` | Optional shorter auto-queue cap. Set to `null` or omit it to use `queue.maxWaitMs`; set a number for interactive sessions where you prefer fast errors |
 | `queue.capacityMaxWaitMs` | Separate cap for repeated upstream 5xx/overload failures; defaults to 15m so broken providers do not park requests for 24h |
 | `queue.maxQueuedBodyBytes` | Maximum request body Maxpool will hold in memory while waiting for capacity before the request has been sent upstream; defaults to 256 MiB |
-| `queue.weeklyMaxWaitMs` | Optional cap for weekly-limit waits. Defaults to `0`, so weekly exhaustion fails fast instead of parking requests for days |
+| `queue.weeklyMaxWaitMs` | How long to hold a request when every account is at its weekly (7d) cap. Defaults to 24h — but the early-exit gates on each account's REAL reset time, so it only waits when a reset genuinely lands inside the window and errors honestly otherwise. (Was `0` = fail-fast, which killed sessions the instant all accounts hit their weekly cap.) |
+| `queue.nonStreamMaxWaitMs` | Max hold for non-streaming requests; defaults to 5m. They have no SSE keepalive, so a longer hold would die on the client timeout anyway |
+| `queue.maxConcurrentQueued` | Backpressure: max requests held waiting at once; defaults to 64. Beyond it, new waiters get a clear "queue full" error instead of growing the heap |
+| `queue.maxQueuedBytes` | Backpressure: max aggregate buffered request-body bytes across all held requests; defaults to 1 GiB |
 | `queue.pollMs` | How often queued requests check for a recovered account/provider |
 | `queue.heartbeatMs` | SSE heartbeat interval for queued streaming requests; defaults to 10s so Claude Code keeps the queued connection alive |
 | `shutdown.drainTimeoutMs` | Maximum time quit/Ctrl-C waits for active requests before exiting |
@@ -368,7 +374,7 @@ The weekly usage bar shows raw upstream utilization and reset timing. Reset-awar
 11. In the `all` profile only, if all Claude accounts are unavailable, provider fallbacks are tried by priority: GLM before Kimi
 12. If all eligible accounts/providers are temporarily unavailable for a temporary reason (5h/session limit, provider cooldown, short 429), the proxy queues the request and retries when one recovers
 13. Repeated upstream 5xx/overload failures use the shorter `capacityMaxWaitMs` cap, not the long quota wait
-14. Weekly exhaustion and non-retryable 4xx errors fail fast by default; if the queue wait expires, returns 429 with the soonest retry time
+14. Weekly (7d) exhaustion is held and retried like a 5h cap, up to `weeklyMaxWaitMs`, but only when an account's real reset lands inside the window; if the soonest reset is beyond it (or unknown), it returns 429 promptly with an honest message naming the soonest reset rather than spinning. Non-retryable 4xx errors still fail fast
 15. Temporary OAuth refresh failures cool the account down and queue/fail over; invalid refresh credentials disable only that account and require login
 16. In an interactive terminal, the server runs under a foreground supervisor so confirmed Restart (`r`) can drain and restart without detaching the replacement TUI
 17. When Restart is confirmed, new upstream admission pauses immediately. Existing upstream requests finish, queued requests cannot deadlock restart, and their sockets close during relaunch so Claude Code reconnects automatically

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.0.7",
+  "version": "1.2.0",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/account-manager.js

@@ -34,14 +34,20 @@ const DEFAULT_SCHEDULER = {
   weeklyCriticalThreshold: 0.95,
   weeklyExhaustedThreshold: 0.985,
   weeklyBurnDebtWeight: 0.6,
-  // Routing-cost tuning (lower cost = preferred). Quota scarcity is the primary
-  // signal; recent-load spread breaks ties between equally-scarce accounts so
-  // sequential traffic rotates instead of funnelling onto one account.
-  scarcityWeight: 6,          // multiplies quota scarcity (pace overage, 0..~1)
-  spreadShareWeight: 3,       // multiplies an account's share of recent fleet load (0..1)
-  recoveryRampWeight: 4,      // decaying penalty applied to a just-recovered account
-  recoveryRampMs: 5 * 60_000, // how long the post-recovery ramp lasts
-  spreadWindowMs: 15 * 60_000,// rolling window used to measure recent per-account load
+  // Routing-cost tuning (lower cost = preferred). The goal is to AVOID
+  // short-term (rate/concurrency) throttling by spreading load across healthy
+  // accounts. So in-flight concurrency is the DOMINANT term, with a steep
+  // per-account soft cap; burn-pace is only a soft de-preference (never a
+  // bench); quota "use-it-or-lose-it" is intentionally a minor signal here.
+  concurrencyWeight: 2,            // multiplies in-flight load (activeWeight+reqWeight) — dominant
+  perAccountConcurrencyTarget: 3,  // D: soft per-account in-flight target; past it, capPenalty bites
+  capPenaltyWeight: 10,            // steep penalty per unit of in-flight depth past D (throttle safety floor)
+  paceCostWeight: 1.5,            // soft de-preference of accounts burning ahead of pace (was the ×6 term)
+  scarcityWeight: 6,              // legacy; superseded by paceCostWeight (kept so old configs don't error)
+  spreadShareWeight: 3,           // multiplies an account's share of recent fleet load (0..1)
+  recoveryRampWeight: 4,          // decaying penalty applied to a just-recovered account
+  recoveryRampMs: 5 * 60_000,     // how long the post-recovery ramp lasts
+  spreadWindowMs: 15 * 60_000,    // rolling window used to measure recent per-account load
 };
 const LOAD_EVENT_MAX_AGE_MS = 60 * 60 * 1000;
 const WEEK_MS = 7 * 24 * 60 * 60 * 1000;
@@ -173,6 +179,7 @@ export class AccountManager {
       waiting: [],
       lastAdmissionAt: 0,
       rampUntil: 0,
+      bytes: 0, // aggregate buffered body bytes across all held requests
     };
     this.admissionPaused = false;
   }
@@ -197,6 +204,7 @@ export class AccountManager {
     let soonestTemporary = Infinity;
     let temporaryCause = null;
     let soonestWeekly = Infinity;
+    let weeklyUnknownReset = 0; // weekly-exhausted accounts whose reset time we don't know yet
     let matchingRoutes = 0;
     const reasons = {};
 
@@ -244,6 +252,11 @@ export class AccountManager {
       } else if (retry.cause === 'weekly_exhausted' && retry.retryAt) {
         const ms = retry.retryAt - Date.now();
         if (ms < soonestWeekly) soonestWeekly = ms;
+      } else if (retry.cause === 'weekly_exhausted' && !retry.retryAt) {
+        // Weekly-capped but we haven't learned the reset time (cold start /
+        // probe failure). We cannot estimate a wait — flag it so the caller
+        // emits an honest "reset time unknown" error instead of waiting forever.
+        weeklyUnknownReset++;
       }
     }
 
@@ -267,6 +280,16 @@ export class AccountManager {
       };
     }
 
+    if (weeklyUnknownReset > 0) {
+      return {
+        available: false,
+        retryAfterMs: Infinity,
+        cause: 'weekly_reset_unknown',
+        reasons,
+        matchingRoutes,
+      };
+    }
+
     return {
       available: false,
       retryAfterMs: Infinity,
@@ -437,7 +460,10 @@ export class AccountManager {
     if (this.getGlobalInFlight() >= this.scheduler.safetyMaxGlobalActive) return false;
     if (account.status === 'exhausted' || account.status === 'error') return false;
     if (this._isSessionQuotaUnavailable(account)) return false;
-    const weeklyState = this._weeklyState(account);
+    // Gate on RAW weekly usage, not pace-adjusted: an account with real
+    // headroom (e.g. 69% used, resets in days) must stay in the healthy-spread
+    // pool even if it's burning fast. Pace is a soft SCORE cost, never a bench.
+    const weeklyState = this._weeklyRawState(account);
     if (weeklyState === 'exhausted') return false;
     if (weeklyState === 'critical' && !options.allowWeeklyCritical) return false;
     if (weeklyState === 'reserve' && !options.allowWeeklyReserve) return false;
@@ -574,13 +600,41 @@ export class AccountManager {
     });
   }
 
-  registerQueuedRequest(requestInfo = {}) {
+  // Drop wedged head tickets (deadline passed or explicitly marked dead) so a
+  // single orphaned waiter cannot block every other request behind it. Cheap;
+  // safe to call before every head check.
+  _reapStaleQueueHead() {
+    const q = this.queueState;
+    const now = Date.now();
+    let guard = 0;
+    while (q.waiting.length && guard++ < 10_000) {
+      const head = q.waiting[0];
+      const stale = head.dead === true || (head.deadlineAt && now > head.deadlineAt);
+      if (!stale) break;
+      q.waiting.shift();
+      q.bytes = Math.max(0, q.bytes - (head.bytes || 0));
+    }
+  }
+
+  // Register a waiter. Returns the ticket, or null if a backpressure limit
+  // (maxConcurrentQueued / maxQueuedBytes) would be exceeded — the caller then
+  // rejects the request with a "queue full" error instead of holding it.
+  registerQueuedRequest(requestInfo = {}, opts = {}) {
     if (requestInfo.queueTicket) return requestInfo.queueTicket;
+    this._reapStaleQueueHead();
+    const bytes = Math.max(0, Number(opts.bytes) || 0);
+    const { maxConcurrentQueued, maxQueuedBytes } = opts;
+    if (maxConcurrentQueued != null && this.queueState.waiting.length >= maxConcurrentQueued) return null;
+    if (maxQueuedBytes != null && this.queueState.waiting.length > 0
+      && this.queueState.bytes + bytes > maxQueuedBytes) return null;
     const ticket = {
       id: this.queueState.nextId++,
       queuedAt: Date.now(),
+      bytes,
+      deadlineAt: opts.deadlineAt || null,
     };
     this.queueState.waiting.push(ticket);
+    this.queueState.bytes += bytes;
     requestInfo.queueTicket = ticket;
     return ticket;
   }
@@ -588,10 +642,12 @@ export class AccountManager {
   canAdmitQueuedRequest(requestInfo = {}) {
     const ticket = requestInfo.queueTicket;
     if (!ticket) return true;
+    this._reapStaleQueueHead();
     if (this.queueState.waiting[0]?.id !== ticket.id) return false;
     const now = Date.now();
     if (now < this.queueState.rampUntil && now - this.queueState.lastAdmissionAt < 250) return false;
     this.queueState.waiting.shift();
+    this.queueState.bytes = Math.max(0, this.queueState.bytes - (ticket.bytes || 0));
     this.queueState.lastAdmissionAt = now;
     requestInfo.queueTicket = null;
     requestInfo.queueAdmitted = true;
@@ -602,7 +658,10 @@ export class AccountManager {
     const ticket = requestInfo.queueTicket;
     if (!ticket) return;
     const index = this.queueState.waiting.findIndex(entry => entry.id === ticket.id);
-    if (index >= 0) this.queueState.waiting.splice(index, 1);
+    if (index >= 0) {
+      this.queueState.waiting.splice(index, 1);
+      this.queueState.bytes = Math.max(0, this.queueState.bytes - (ticket.bytes || 0));
+    }
     requestInfo.queueTicket = null;
   }
 
@@ -1075,8 +1134,21 @@ export class AccountManager {
   _scoreAccount(account, requestInfo = {}, ctx = null) {
     const now = ctx?.now ?? Date.now();
     const reqWeight = Math.max(1, requestInfo.weight || 1);
-    const concurrency = account.activeWeight + reqWeight;
-    const scarcity = this._accountScarcity(account, now) * this.scheduler.scarcityWeight;
+    const inflight = account.activeWeight + reqWeight;
+
+    // DOMINANT term: in-flight concurrency. Short-term throttling is driven by
+    // how many requests pile on one account, so least-loaded-first spread is
+    // the primary objective.
+    const concurrency = inflight * this.scheduler.concurrencyWeight;
+
+    // Steep soft cap past depth D — the throttle safety floor. No single
+    // account absorbs a deep concurrent burst no matter how "cheap" it looks.
+    const capPenalty = this.scheduler.capPenaltyWeight
+      * Math.max(0, inflight - this.scheduler.perAccountConcurrencyTarget);
+
+    // Burn-pace COST only (demoted from the old dominant scarcity×6 term): a
+    // soft de-preference of accounts burning ahead of an even pace. Never a bench.
+    const paceCost = this._accountScarcity(account, now) * this.scheduler.paceCostWeight;
 
     const fleetRecentWeight = ctx?.fleetRecentWeight ?? 0;
     const recentWeight = this._loadSummary(account, this.scheduler.spreadWindowMs, now).weight;
@@ -1089,7 +1161,7 @@ export class AccountManager {
     // probed and learned (matches the legacy unknown-quota exploration nudge).
     const explorationBonus = account.quota.unified7dReset == null ? -0.5 : 0;
 
-    return concurrency + scarcity + spread + ramp + failurePenalty + explorationBonus;
+    return concurrency + capPenalty + paceCost + spread + ramp + failurePenalty + explorationBonus;
   }
 
   /**

package/src/config.js

@@ -81,13 +81,20 @@ export function createDefaultConfig() {
     shutdown: {
       drainTimeoutMs: 15_000,
     },
+    // When every account is rate-limited, hold the request and retry until one
+    // frees up, instead of erroring and killing the session. Only error if
+    // nothing recovers within the window below (the early-exit gates on each
+    // account's REAL reset time, so a generous bound never spins pointlessly).
     queue: {
       enabled: true,
-      maxWaitMs: 24 * 60 * 60 * 1000,
-      autoMaxWaitMs: null,
-      capacityMaxWaitMs: 15 * 60 * 1000,
-      maxQueuedBodyBytes: 256 * 1024 * 1024,
-      weeklyMaxWaitMs: 0,
+      maxWaitMs: 24 * 60 * 60 * 1000,    // hard ceiling for any hold
+      autoMaxWaitMs: null,               // 5h/session-cap hold (null = maxWaitMs)
+      capacityMaxWaitMs: 15 * 60 * 1000, // upstream 529/overload — stays short, never governed by the others
+      weeklyMaxWaitMs: 24 * 60 * 60 * 1000, // weekly (7d) cap hold; was 0 (fail-fast) — that killed sessions on weekly cap
+      nonStreamMaxWaitMs: 5 * 60 * 1000, // non-streaming requests have no keepalive; cap their wait
+      maxConcurrentQueued: 64,           // backpressure: max requests held at once
+      maxQueuedBytes: 1024 * 1024 * 1024, // backpressure: max aggregate buffered body bytes (1 GiB)
+      maxQueuedBodyBytes: 256 * 1024 * 1024, // per-request cap on a queueable body
       pollMs: 1000,
       heartbeatMs: 10_000,
     },

package/src/server.js

@@ -19,7 +19,19 @@ const DEFAULT_QUEUE = {
   maxWaitMs: 24 * 60 * 60 * 1000,
   autoMaxWaitMs: null,
   capacityMaxWaitMs: 15 * 60 * 1000,
-  weeklyMaxWaitMs: 0,
+  // Weekly (7d) cap hold. A generous bound is SAFE because the early-exit
+  // gates on the REAL reset time (unified7dReset - now): it only waits when a
+  // reset genuinely lands inside the window, and errors honestly otherwise.
+  // 0 here was the bug — it fail-fast-killed sessions the instant every
+  // account hit its weekly cap, instead of waiting for the soonest reset.
+  weeklyMaxWaitMs: 24 * 60 * 60 * 1000,
+  // Non-streaming requests have no SSE heartbeat to keep them alive, so a long
+  // hold would die on the client timeout anyway. Cap their wait conservatively.
+  nonStreamMaxWaitMs: 5 * 60 * 1000,
+  // Backpressure: holds used to be 0ms, now they can be hours. Bound the queue
+  // so 22 retrying agents can't grow the heap without limit.
+  maxConcurrentQueued: 64,
+  maxQueuedBytes: 1024 * 1024 * 1024, // 1 GiB aggregate across all held bodies
   pollMs: 1000,
   heartbeatMs: 10_000,
 };
@@ -825,6 +837,14 @@ function hasEligibleRoute(accountManager, requestInfo = {}, excludedIndexes = ne
   return accountManager.hasAvailableRoute?.(requestInfo, excludedIndexes) || false;
 }
 
+function formatRetryDuration(seconds) {
+  const s = Math.max(0, Math.round(Number(seconds) || 0));
+  if (s >= 86400) return `${Math.round(s / 86400)}d`;
+  if (s >= 3600) return `${Math.round(s / 3600)}h`;
+  if (s >= 60) return `${Math.round(s / 60)}m`;
+  return `${s}s`;
+}
+
 function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRecoverSoon = true) {
   const thinking = requestInfo.requiresAnthropicThinkingIntegrity
     || accountManager._requiresAnthropicThinkingIntegrity?.(requestInfo);
@@ -834,7 +854,10 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
   // account is at its own 5h/weekly limit. A short "retry in Ns" would be a lie;
   // tell the user the real fix.
   if (!willRecoverSoon) {
-    const base = `No Claude account can take this request — all ${n} are at their 5h or weekly limit. Add another Claude account or wait for a quota reset.`;
+    const eta = Number.isFinite(retryAfter) && retryAfter > 0
+      ? ` Soonest reset in ~${formatRetryDuration(retryAfter)}, beyond the hold window.`
+      : '';
+    const base = `No Claude account can take this request — all ${n} are at their 5h or weekly limit.${eta} Add another Claude account or wait for a quota reset.`;
     return thinking
       ? `${base} GLM/Kimi fallback is unavailable because this session contains Anthropic signed thinking blocks; start a fresh non-thinking session to use them.`
       : base;
@@ -1062,29 +1085,64 @@ async function queueAndRetry(
     ? autoMaxWaitMs
     : Math.max(0, Number(queueConfig.capacityMaxWaitMs) || 0);
   const weeklyMaxWaitMs = Math.max(0, Number(queueConfig.weeklyMaxWaitMs) || 0);
+  const nonStreamMaxWaitMs = queueConfig.nonStreamMaxWaitMs == null
+    ? 5 * 60_000
+    : Math.max(0, Number(queueConfig.nonStreamMaxWaitMs) || 0);
   const retryPlan = accountManager.nextRetryForRequest?.(requestInfo, new Set()) || {
     retryAfterMs: Infinity,
     cause: 'unavailable',
   };
-  const queueWindowMs = retryPlan.cause === 'weekly_exhausted'
-    ? Math.min(maxWaitMs, weeklyMaxWaitMs)
-    : cause === 'capacity'
-      ? Math.min(maxWaitMs, capacityMaxWaitMs)
-    : Math.min(maxWaitMs, autoMaxWaitMs);
-  if (queueWindowMs <= 0) return finishQueuedStreamIfNeeded(res, requestInfo, 'No retry window is available.');
+
+  // Honest, cause-/thinking-aware message used for every give-up path below.
+  const honestMessage = unavailableMessage(
+    accountManager, requestInfo,
+    Math.ceil((Number.isFinite(retryPlan.retryAfterMs) ? retryPlan.retryAfterMs : 0) / 1000),
+    false,
+  );
+
+  // Weekly-capped but the reset time is unknown (cold start / probe failure):
+  // we can't estimate a wait, so don't pretend to — error honestly now.
+  if (retryPlan.cause === 'weekly_reset_unknown') {
+    return finishQueuedStreamIfNeeded(res, requestInfo, honestMessage);
+  }
+
+  // Pick the wait window. Capacity (upstream 529/overload) MUST stay on its own
+  // short cap even when it coincides with weekly exhaustion — never let a
+  // transient overload inherit the long weekly bound.
+  let queueWindowMs = cause === 'capacity'
+    ? Math.min(maxWaitMs, capacityMaxWaitMs)
+    : retryPlan.cause === 'weekly_exhausted'
+      ? Math.min(maxWaitMs, weeklyMaxWaitMs)
+      : Math.min(maxWaitMs, autoMaxWaitMs);
+  // Non-streaming requests have no SSE heartbeat, so a long hold would die on
+  // the client timeout. Cap them so we never promise a wait we can't deliver.
+  if (!requestInfo.stream) queueWindowMs = Math.min(queueWindowMs, nonStreamMaxWaitMs);
+
+  if (queueWindowMs <= 0) return finishQueuedStreamIfNeeded(res, requestInfo, honestMessage);
 
   const retryAfterMs = retryPlan.retryAfterMs;
   if (!Number.isFinite(retryAfterMs) || retryAfterMs > queueWindowMs) {
-    return finishQueuedStreamIfNeeded(res, requestInfo, 'No route is expected to recover within the configured queue window.');
+    return finishQueuedStreamIfNeeded(res, requestInfo, honestMessage);
   }
 
   requestInfo.queueStartedAt ||= Date.now();
-  accountManager.registerQueuedRequest?.(requestInfo);
+  const ticket = accountManager.registerQueuedRequest?.(requestInfo, {
+    bytes: body?.length || 0,
+    deadlineAt: requestInfo.queueStartedAt + queueWindowMs,
+    maxConcurrentQueued: queueConfig.maxConcurrentQueued,
+    maxQueuedBytes: queueConfig.maxQueuedBytes,
+  });
+  if (ticket === null) {
+    // Backpressure: too many requests already waiting / too many bytes buffered.
+    // Reject honestly instead of growing the heap unbounded.
+    return finishQueuedStreamIfNeeded(res, requestInfo,
+      'Maxpool queue is full — too many requests are already waiting for capacity. Try again shortly.');
+  }
   const elapsed = Date.now() - requestInfo.queueStartedAt;
   const remaining = queueWindowMs - elapsed;
   if (remaining <= 0) {
     accountManager.removeQueuedRequest?.(requestInfo);
-    return finishQueuedStreamIfNeeded(res, requestInfo, 'The Maxpool queue wait expired.');
+    return finishQueuedStreamIfNeeded(res, requestInfo, honestMessage);
   }
 
   ctx.account = '(queued)';
@@ -1096,7 +1154,7 @@ async function queueAndRetry(
   if (!available) {
     if (res.destroyed || req.destroyed) return true;
     accountManager.removeQueuedRequest?.(requestInfo);
-    return finishQueuedStreamIfNeeded(res, requestInfo, 'The Maxpool queue wait expired.');
+    return finishQueuedStreamIfNeeded(res, requestInfo, honestMessage);
   }
 
   return forwardRequest(
@@ -1163,7 +1221,10 @@ async function waitForAvailableRoute(req, res, accountManager, requestInfo, queu
       ) return true;
 
       const remaining = maxWaitMs - (Date.now() - startedAt);
-      await sleep(Math.min(pollMs, remaining));
+      // Jitter the poll so a synchronized weekly-reset event doesn't re-align
+      // every waiter's poll into the same instant (thundering scan).
+      const jittered = pollMs * (0.8 + Math.random() * 0.4);
+      await sleep(Math.min(jittered, remaining));
     }
 
     return accountManager.hasAvailableRoute(requestInfo, new Set())