maxpool 1.5.1 → 1.5.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.5.1",
+  "version": "1.5.3",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/account-manager.js

@@ -48,6 +48,10 @@ const DEFAULT_SCHEDULER = {
   safetyMaxGlobalActive: 150,
   cooldownMs: 30_000,
   maxCooldownMs: 15 * 60_000,
+  // Fixed cooldown for NETWORK-class failures (lost connectivity / token-refresh
+  // fetch-failed). Short + non-escalating so the fleet auto-recovers seconds after
+  // connectivity returns, instead of the exponential maxCooldownMs bench.
+  networkCooldownMs: 5_000,
   weeklySoftThreshold: 0.65,
   weeklyReserveThreshold: 0.85,
   weeklyCriticalThreshold: 0.95,
@@ -250,48 +254,53 @@ export class AccountManager {
 
     for (const account of this.accounts) {
       if (excludedIndexes.has(account.index)) continue;
-      if (!this._matchesRequest(account, profile, requestInfo)) {
-        // A signed-thinking request behind the TRANSIENT FIFO queue-fairness gate
-        // (a queue already exists and this newcomer hasn't registered a ticket yet
-        // — the `!requestInfo.queueTicket` clause in _matchesRequest) must HOLD,
-        // not error-fast. Such a request has NO fallback: providers are barred
-        // (signed thinking), so shedding it with a 429 KILLS the session — the
-        // reproduced "all N are at their 5h or weekly limit" false kill
-        // (2026-06-27) while healthy accounts sat idle behind the queue. The
-        // instant it registers a ticket the gate lifts and it waits its FIFO turn
-        // behind the existing waiters (which drain as the healthy accounts finish),
-        // so a bounded re-poll hold is the right answer.
-        // Scoped to THINKING requests on purpose: a non-thinking request can fall
-        // back to a provider or be cheaply retried, so the fairness gate keeps
-        // shedding it as backpressure (never grow the queue past its cap under a
-        // pure-concurrency burst). admissionPaused (restart/shutdown shed) and
-        // upstream-throttle keep their existing behavior (throttle is handled by
-        // the early return above). NOTE: queueAndRetry does NOT special-case
-        // 'queued_behind_fairness' in its window switch, so a STREAMING thinking
-        // hold rides streamHoldMaxMs (kept alive by the heartbeat) — intended: a
-        // signed-thinking stream should wait its turn, bounded by the per-ticket
-        // deadlineAt + _reapStaleQueueHead + the client's own disconnect, not the
-        // 15-min concurrency-cap clamp.
-        const thinkingFairnessGated = this._requiresAnthropicThinkingIntegrity(requestInfo)
-          && !this.admissionPaused
-          && !this._isUpstreamThrottleBlocking()
-          && this.queueState.waiting.length > 0
-          && !requestInfo.queueTicket
-          && !requestInfo.queueAdmitted;
-        if (thinkingFairnessGated
-          && account.type !== 'provider'
-          && this._isRequestCompatible(account, profile, requestInfo)
-          && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
-          soonestBoundedHold = Math.min(soonestBoundedHold, BOUNDED_REPOLL_HOLD_MS);
-          if (!boundedHoldCause) boundedHoldCause = 'queued_behind_fairness';
-        } else if (account.type === 'provider' && this._requiresAnthropicThinkingIntegrity(requestInfo)) {
+      const matches = this._matchesRequest(account, profile, requestInfo);
+
+      // A signed-thinking request behind the TRANSIENT FIFO queue-fairness gate
+      // (a queue already exists and this newcomer hasn't registered a ticket yet
+      // — the `!requestInfo.queueTicket` clause in _matchesRequest) must still be
+      // CONSIDERED for retry timing, not skipped. Such a request has NO fallback:
+      // providers are barred (signed thinking), so dropping the account here — and
+      // losing its REAL cooldown / rate-limit / 5h reset — collapses the oracle to
+      // Infinity and KILLS the session. Two reproduced kills (2026-06-27): a queue
+      // forms, then either (a) healthy accounts sit idle behind it, or (b) a
+      // network blip has cooled EVERY account, and the thinking newcomer dies with
+      // the false "all N are at their 5h or weekly limit". The gate lifts the
+      // instant the request registers a ticket, so bypass it for retry timing here.
+      // Still respected: admissionPaused (restart/shutdown shed → stays terminal),
+      // upstream-throttle (handled by the early return above), and structural
+      // profile / provider-thinking compatibility. Scoped to THINKING on purpose: a
+      // non-thinking newcomer can fall back to a provider or be cheaply retried, so
+      // the gate keeps shedding it as backpressure (never grow the queue past its
+      // cap under a pure-concurrency burst).
+      const fairnessOnlyBlock = !matches
+        && this._requiresAnthropicThinkingIntegrity(requestInfo)
+        && account.type !== 'provider'
+        && this._isRequestCompatible(account, profile, requestInfo)
+        && !this.admissionPaused
+        && !this._isUpstreamThrottleBlocking()
+        && this.queueState.waiting.length > 0
+        && !requestInfo.queueTicket
+        && !requestInfo.queueAdmitted;
+
+      if (!matches && !fairnessOnlyBlock) {
+        if (account.type === 'provider' && this._requiresAnthropicThinkingIntegrity(requestInfo)) {
           note('provider_fallback_disabled_signed_thinking');
         }
         continue;
       }
 
       matchingRoutes++;
-      if (this._isAvailable(account, { allowWeeklyReserve: true })) {
+
+      // NEVER claim available:0 for a fairness-gated account — _selectNext still
+      // refuses it (the gate), so an available verdict would desync the oracle from
+      // selection and spin the caller. A healthy gated account holds a bounded
+      // re-poll (queued_behind_fairness); a transiently-blocked one (cooldown /
+      // rate-limit / 5h cap) falls through so its REAL short-term reset drives a
+      // finite hold. It must NEVER contribute the WEEKLY reset (days) — that is the
+      // multi-day-hang the bounded path fences off; see the !fairnessOnlyBlock
+      // guards on the weekly branches below.
+      if (!fairnessOnlyBlock && this._isAvailable(account, { allowWeeklyReserve: true })) {
         return {
           available: true,
           retryAfterMs: 0,
@@ -300,10 +309,15 @@ export class AccountManager {
           matchingRoutes,
         };
       }
+      if (fairnessOnlyBlock && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
+        soonestBoundedHold = Math.min(soonestBoundedHold, BOUNDED_REPOLL_HOLD_MS);
+        if (!boundedHoldCause) boundedHoldCause = 'queued_behind_fairness';
+        continue;
+      }
 
       const retry = this._retryInfo(account);
       note(retry.cause);
-      if (retry.weeklyCritical && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
+      if (!fairnessOnlyBlock && retry.weeklyCritical && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
         return {
           available: true,
           retryAfterMs: 0,
@@ -334,10 +348,15 @@ export class AccountManager {
         // hold when no weekly-critical account contributed it.
         if (retry.weeklyCritical) boundedHoldCause = 'weekly_critical';
         else if (!boundedHoldCause) boundedHoldCause = 'concurrency_cap';
-      } else if (retry.cause === 'weekly_exhausted' && retry.retryAt) {
+      } else if (!fairnessOnlyBlock && retry.cause === 'weekly_exhausted' && retry.retryAt) {
+        // A fairness-gated account NEVER contributes the weekly reset: holding a
+        // thinking session for days behind the queue is the over-correction we
+        // avoid (a weekly-exhausted gated fleet stays terminal → honest error,
+        // matching the no-newcomer-vs-queue distinction). Only its short-term reset
+        // (above) or the bounded hold may fire.
         const ms = retry.retryAt - Date.now();
         if (ms < soonestWeekly) soonestWeekly = ms;
-      } else if (retry.cause === 'weekly_exhausted' && !retry.retryAt) {
+      } else if (!fairnessOnlyBlock && retry.cause === 'weekly_exhausted' && !retry.retryAt) {
         // Weekly-capped but we haven't learned the reset time (cold start /
         // probe failure). We cannot estimate a wait — flag it so the caller
         // emits an honest "reset time unknown" error instead of waiting forever.
@@ -1683,9 +1702,28 @@ export class AccountManager {
     console.log(`[Maxpool] Account "${account.name}" disabled after HTTP ${status} (${reason})`);
   }
 
-  markTransientFailure(accountIndex, reason = 'transient_error') {
+  markTransientFailure(accountIndex, reason = 'transient_error', { network = false } = {}) {
     const account = this.accounts[accountIndex];
     if (!account) return;
+
+    if (network) {
+      // A network-class failure (lost connectivity / token-refresh `fetch failed`)
+      // is a FLEET-WIDE condition, not this account's fault — every account fails at
+      // once. Use a SHORT FIXED cooldown so the whole fleet retries within seconds of
+      // connectivity returning and recovers AUTOMATICALLY, never the exponential
+      // 15-min bench that stranded the fleet long after a multi-hour outage and forced
+      // a manual restart (2026-06-29 hotel-network nightly cutoff). Deliberately does
+      // NOT bump consecutiveFailures: a network blip must not poison the scoring
+      // penalty (_scoreAccount) or prime the next REAL per-account failure for the max
+      // cooldown — so the counter stays a pure request-health signal and needs no reset.
+      account.failedRequests++;
+      account.lastError = reason;
+      account.lastErrorAt = Date.now();
+      account.cooldownUntil = Date.now() + this.scheduler.networkCooldownMs;
+      console.log(`[Maxpool] Account "${account.name}" cooling down for ${Math.ceil(this.scheduler.networkCooldownMs / 1000)}s after ${reason} (network — short fixed, auto-recovers)`);
+      return;
+    }
+
     const failures = Math.max(1, account.consecutiveFailures + 1);
     const cooldown = Math.min(
       this.scheduler.maxCooldownMs,
@@ -1797,7 +1835,11 @@ export class AccountManager {
         // a failed proactive refresh shouldn't kill a still-valid token
         if (!account.expiresAt || Date.now() >= account.expiresAt) {
           if (err.retryable) {
-            this.markTransientFailure(accountIndex, `token_refresh_${err.status || 'network'}`);
+            // A retryable refresh failure with NO HTTP status is a network/connectivity
+            // failure (fetch failed / timeout) → short fixed cooldown so it auto-recovers
+            // when the network returns. A retryable HTTP status (429 / 5xx from the OAuth
+            // endpoint) is server-side → keep the exponential backoff.
+            this.markTransientFailure(accountIndex, `token_refresh_${err.status || 'network'}`, { network: !err.status });
           } else {
             this.markAuthFailed(accountIndex, err.status || 401, 'token_refresh_failed');
           }

package/src/config.js

@@ -59,6 +59,10 @@ export function createDefaultConfig() {
       safetyMaxGlobalActive: 150,
       cooldownMs: 30_000,
       maxCooldownMs: 15 * 60_000,
+      // Fixed cooldown for network-class failures (lost connectivity / token-refresh
+      // fetch-failed) — short + non-escalating so the fleet auto-recovers seconds after
+      // connectivity returns, never the exponential maxCooldownMs bench.
+      networkCooldownMs: 5_000,
       // Weekly (7d) quota tiers — how aggressively to de-prioritise an account
       // as its weekly usage climbs. Each is a fraction (0..1) of the weekly
       // limit. Below soft = full speed; soft..reserve = mild penalty;

package/src/oauth.js

@@ -17,6 +17,11 @@ const DEFAULT_CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e';
 export async function refreshAccessToken(refreshToken, endpoint = DEFAULT_TOKEN_ENDPOINT) {
   const maxRetries = 2;
   const baseDelayMs = 500;
+  // Bound each refresh POST so a hung connect during an outage can't pin the
+  // single-flight _refreshPromise indefinitely (which would stall that account's
+  // recovery). Comfortably above a normal refresh latency; a timeout is classified
+  // as a network error below and retried / short-cooled.
+  const perAttemptTimeoutMs = 10_000;
 
   for (let attempt = 0; attempt <= maxRetries; attempt++) {
     try {
@@ -37,6 +42,7 @@ export async function refreshAccessToken(refreshToken, endpoint = DEFAULT_TOKEN_
           refresh_token: refreshToken,
           client_id: DEFAULT_CLIENT_ID,
         }),
+        signal: AbortSignal.timeout(perAttemptTimeoutMs),
       });
 
       if (!res.ok) {
@@ -60,6 +66,7 @@ export async function refreshAccessToken(refreshToken, endpoint = DEFAULT_TOKEN_
     } catch (err) {
       const isNetworkError = err instanceof Error &&
         (err.message.includes('fetch failed') ||
+          err.name === 'TimeoutError' || err.name === 'AbortError' ||
           (err.code === 'ECONNRESET' || err.code === 'ECONNREFUSED' ||
            err.code === 'ETIMEDOUT' || err.code === 'UND_ERR_CONNECT_TIMEOUT'));
 

package/src/server.js

@@ -855,7 +855,10 @@ async function forwardRequest(
         err.message.includes('terminated'));
 
     if (isTransient) {
-      accountManager.markTransientFailure(account.index, err.code || err.message || 'network_error');
+      // Network-class (ECONNRESET / fetch failed / timeout / terminated): short fixed
+      // cooldown, no exponential escalation — the fleet auto-recovers seconds after
+      // connectivity returns instead of being benched for up to 15 min.
+      accountManager.markTransientFailure(account.index, err.code || err.message || 'network_error', { network: true });
       accountManager.releaseAccount(lease);
       excludedIndexes.add(account.index);
       if (canRetryBufferedBody && retryCount + 1 < maxAttempts && !res.headersSent) {
@@ -1158,7 +1161,18 @@ async function queueAndRetry(
     }
     return false;
   }
-  if (cause === 'network' || cause === 'proxy') return false;
+  // A 'proxy' (non-transient upstream) error fails fast. A 'network' error (the
+  // upstream fetch threw — ECONNRESET / ETIMEDOUT / a VPN or internet blip) is the
+  // MOST transient failure there is, and must NOT kill a live session: when the
+  // request is a STREAMING, replayable one (heartbeat keeps it alive, buffered body
+  // can be re-sent) and ≥1 account will recover on a finite schedule, hold it and
+  // resume when connectivity returns — exactly the "a network drop shouldn't break
+  // my session" goal (2026-06-27). The hold-vs-error gate below still error-fasts if
+  // nextRetryForRequest reports no finite recovery (all accounts genuinely gone), so
+  // a real outage doesn't spin. A non-streaming network failure (no keepalive) still
+  // fails fast — it would die on the client's own timeout anyway.
+  if (cause === 'proxy') return false;
+  if (cause === 'network' && (!requestInfo.stream || !canQueueBufferedBody)) return false;
 
   const maxWaitMs = Math.max(0, Number(queueConfig.maxWaitMs) || 0);
   const autoMaxWaitMs = queueConfig.autoMaxWaitMs == null
@@ -1176,11 +1190,16 @@ async function queueAndRetry(
   };
 
   // Honest, cause-/thinking-aware message used for every give-up path below.
-  const honestMessage = unavailableMessage(
-    accountManager, requestInfo,
-    Math.ceil((Number.isFinite(retryPlan.retryAfterMs) ? retryPlan.retryAfterMs : 0) / 1000),
-    false,
-  );
+  // A 'network'-cause hold that ultimately gives up means connectivity never came
+  // back within the window — say THAT (check your internet), not the misleading
+  // "all accounts at their quota limit" (the accounts are fine; the network isn't).
+  const honestMessage = cause === 'network'
+    ? 'Could not connect to Claude (network error) and connectivity did not return in time. Check your internet connection and try again. This is not an account quota issue.'
+    : unavailableMessage(
+      accountManager, requestInfo,
+      Math.ceil((Number.isFinite(retryPlan.retryAfterMs) ? retryPlan.retryAfterMs : 0) / 1000),
+      false,
+    );
 
   // Weekly-capped but the reset time is unknown (cold start / probe failure):
   // we can't estimate a wait, so don't pretend to — error honestly now.