maxpool 1.24.7 → 1.24.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.24.7",
+  "version": "1.24.9",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/account-manager.js

@@ -219,6 +219,15 @@ const DEFAULT_SCHEDULER = {
   // undiscounted, as do the hard gates (safetyMaxActivePerAccount, usage cap,
   // cooldowns) — those are the anti-dogpile backstops, not price signals.
   fastRefillDiscount: 0.6,        // 0 = off (full cost), 0.6 = discount up to 60% of the two balancing terms
+  // RESET-AWARE PREFERENCE (2026-10-08, owner): an account holding a usable, unexpired
+  // limit reset (Claude cedar_ember grant / z.ai card) has capacity its siblings lack —
+  // when it reaches its wall the prober redeems the reset and it refills (reset-policy
+  // R1). So its BALANCING costs (pace, utilization, spread) are discounted: it soaks
+  // more traffic and spends the reset instead of letting it expire. The SAFETY terms
+  // (capPenalty, reserve/critical, ramp, failure) and every hard gate stay undiscounted,
+  // so it is never dogpiled and never preferred over an unhealthy-looking pick.
+  // 0 = off. 0.35 ≈ a 1.5x share at otherwise-equal health.
+  resetBoostDiscount: 0.35,
   fastRefillFadeUtil: 0.65,       // discount reaches 0 at this session utilization (weeklySoftThreshold)
   recoveryRampWeight: 4,          // decaying penalty applied to a just-recovered account
   recoveryRampMs: 5 * 60_000,     // how long the post-recovery ramp lasts
@@ -2770,6 +2779,31 @@ export class AccountManager {
    *   - ramp:        ease a just-recovered account back in instead of slamming it
    *   - failures:    direct per-account backoff after errors
    */
+  /**
+   * Does this account hold a banked reset that can actually lift its BINDING limit?
+   * Claude grants: usable, not paused, unexpired. z.ai cards: unexpired five-hour cards
+   * always count; weekly cards count only when the account HAS a weekly limit (a legacy
+   * plan has none — z.ai support, 2026-10-03 — so a weekly card there lifts nothing).
+   * Disabled accounts never count (owner gate R0). Display state only otherwise.
+   */
+  _holdsUsableReset(account, now = Date.now()) {
+    if (!account || account.enabled === false) return false;
+    const grants = account.resetGrants?.grants || [];
+    if (grants.some(g => g && g.usableNow !== false && !g.paused && (g.endsAt == null || g.endsAt > now))) return true;
+    const cards = account.resetCards;
+    if (!cards) return false;
+    const live = c => c && !c.expired && (c.expiresAt == null || c.expiresAt > now);
+    if ((cards.fiveHour || []).some(live)) return true;
+    const hasWeeklyLimit = account.quota?.providerWk != null || account.quota?.providerWkReset != null;
+    return hasWeeklyLimit && (cards.weekly || []).some(live);
+  }
+
+  _resetMultiplier(account, now = Date.now()) {
+    const d = Number(this.scheduler.resetBoostDiscount) || 0;
+    if (d <= 0) return 1;
+    return this._holdsUsableReset(account, now) ? 1 - Math.min(d, 0.9) : 1;
+  }
+
   _scoreAccount(account, requestInfo = {}, ctx = null, criticalUnlock = null) {
     const now = ctx?.now ?? Date.now();
     const reqWeight = Math.max(1, requestInfo.weight || 1);
@@ -2826,7 +2860,9 @@ export class AccountManager {
     // FAST-REFILL DISCOUNT: applied to the pace and utilization terms for an
     // account whose only cap is a fast-refilling session window — see the
     // DEFAULT_SCHEDULER block for the full rationale (refillMult computed above).
-    const paceCost = this._accountScarcity(account, now) * this.scheduler.paceCostWeight * refillMult;
+    // RESET-AWARE: discounts the three BALANCING terms only (see resetBoostDiscount).
+    const resetMult = this._resetMultiplier(account, now);
+    const paceCost = this._accountScarcity(account, now) * this.scheduler.paceCostWeight * refillMult * resetMult;
 
     // RAW utilization cost — direct, not pace-adjusted. The pace cost above discounts
     // by how far into the window you are, so an account at 80% with 2h left is only
@@ -2834,7 +2870,7 @@ export class AccountManager {
     // benching, but wrong for load balancing: an account at 80% should be clearly less
     // attractive than one at 10% even if both are "on pace". Measured 2026-08-10: cc at
     // 80% scored 52.30 vs glm at 10% at 52.15 — a 0.15 gap drowned by round-robin.
-    const utilizationCost = this._rawUtilization(account) * this.scheduler.utilizationWeight * refillMult;
+    const utilizationCost = this._rawUtilization(account) * this.scheduler.utilizationWeight * refillMult * resetMult;
 
     // Per-model weekly de-preference: an account whose scoped weekly for THIS
     // request's model (e.g. Fable) is high-but-not-exhausted is a poor pick for
@@ -2860,7 +2896,7 @@ export class AccountManager {
     // 2.3x early in its window, fading to parity at the same ses 0.65 the multiplier
     // already uses. A multiplier of 1 makes every discounted term byte-identical to
     // pre-2026-08-25 behaviour; reserve/critical/ramp/failure are never discounted.
-    const spread = share * this.scheduler.spreadShareWeight * refillMult;
+    const spread = share * this.scheduler.spreadShareWeight * refillMult * resetMult;
 
     const ramp = this._recoveryRamp(account, now);
     const reserveCost = this._reserveCost(account, now, weeklyState);
@@ -3344,7 +3380,13 @@ export class AccountManager {
     }
 
     q.lastProbeOkAt = Date.now();
-    // A successful probe clears any recorded failure — freshness confirmed.
+    // A successful probe clears any recorded failure — freshness confirmed — including
+    // the org-403 streak (a healthy probe proves the org still accepts OAuth).
+    q.consecutiveOrgGone = 0;
+    if (account.subscriptionGone) {
+      account.subscriptionGone = false;
+      console.log(`[Maxpool] "${account.name}" subscription check passed — un-benched (the org-403 was transient)`);
+    }
     q.lastProbeError = null;
     q.lastProbeErrorAt = null;
     q.lastProbeErrorStatus = null;
@@ -3381,6 +3423,20 @@ export class AccountManager {
     // worth having; the diagnosis that motivated it was wrong.)
     q.consecutiveProbeFailures = (q.consecutiveProbeFailures || 0) + 1;
     const n = q.consecutiveProbeFailures;
+    // ORG-403 LATCH REQUIRES ITS OWN STRIKES (2026-10-08, false "no sub" on
+    // max@gomokka.com). `n` mixes EVERY probe failure — network timeouts first among
+    // them — so after a flapping-network morning one genuine org-403 can arrive with
+    // n already >= 3 and bench a LIVE subscription (measured: latch fired at "x6"
+    // with exactly one org-403 in the whole week's log; 104 successful requests the
+    // same day). The subscription verdict must count only consecutive org-403s.
+    const isOrgGone = status === 403
+      && /not allowed for this organization|disabled.*subscription|subscription.*disabled|organization has disabled/i.test(String(message));
+    if (isOrgGone) {
+      q.consecutiveOrgGone = (q.consecutiveOrgGone || 0) + 1;
+    } else {
+      q.consecutiveOrgGone = 0;   // any OTHER outcome breaks the org-gone streak
+    }
+    const orgN = q.consecutiveOrgGone;
     // A SUSTAINED ORG-403 is the subscription being gone (canceled Max plan lapsing
     // server-side — measured 2026-09-18/20: "OAuth authentication is currently not
     // allowed for this organization" on 2solarmax@ and privacy@, both canceled Sep 16).
@@ -3388,11 +3444,10 @@ export class AccountManager {
     // pure waste. Latch subscriptionGone (3 strikes, like the 401 rule): the prober
     // skips the account, the TUI says why, and routing treats it as unavailable. A
     // successful re-auth clears it (updateAccountTokens).
-    if (status === 403
-      && /not allowed for this organization|disabled.*subscription|subscription.*disabled|organization has disabled/i.test(String(message))
-      && n >= 3 && !account.subscriptionGone) {
+    if (isOrgGone && orgN >= 3 && !account.subscriptionGone) {
       account.subscriptionGone = true;
-      console.error(`[Maxpool] "${account.name}" subscription disabled at the organization (HTTP 403 x${n}) — benching it. Re-enable after re-subscribing, or remove the account (a → d).`);
+      account._subRecheckAt = Date.now();   // first recheck one full window after latching
+      console.error(`[Maxpool] "${account.name}" subscription disabled at the organization (HTTP 403 x${orgN}) — benching it. Re-enable after re-subscribing, or remove the account (a → d).`);
     }
     // A SUSTAINED 401 is dead credentials, not a blip. Latch refreshDead so (a) the
     // prober stops re-POSTing a rejected token every 60s forever — measured 2026-08-10:
@@ -3438,7 +3493,7 @@ export class AccountManager {
   }
 
   /** Store the latest reset-card listing for an account (prober piggyback).
-   *  Cards are display/decision state only — routing never reads them. */
+   *  Read by redeem policy AND by _scoreAccount's reset-aware preference. */
   applyResetCards(accountIndex, cards) {
     const account = this.accounts[accountIndex];
     if (!account || !cards) return;
@@ -4009,7 +4064,7 @@ export class AccountManager {
     // A dead refresh token (invalid_grant) is PERMANENT until browser re-auth —
     // never auto-retry it. Without this the prober re-POSTs the rejected token every
     // ~60s forever (hammering Anthropic's OAuth endpoint). Cleared on re-login.
-    if (account.refreshDead || account.subscriptionGone) return false;
+    if (account.refreshDead || (account.subscriptionGone && !account._subRecheck)) return false;
 
     // A DISABLED account never spends its single-use refresh token. The prober still
     // READS its quota by design (you disable an exhausted account and still want to

package/src/prober.js

@@ -12,6 +12,10 @@ import { listResetCards, redeemResetCard } from './zai-reset-cards.js';
 import { listResetGrants, claimResetGrant } from './claude-reset-grants.js';
 import { decideZai, decideClaude } from './reset-policy.js';
 
+// How often a subscription-latched account is rechecked (one probe + at most one
+// token refresh per window — cheap, and bounded so a truly canceled plan isn't hammered).
+export const SUB_RECHECK_MS = 30 * 60_000;
+
 export class Prober {
   constructor(accountManager, { intervalMs = 0, probeFn = fetchUsage, providerProbeFn = fetchProviderUsage, grantsFn = listResetGrants, timeoutMs = 10_000, log = console.log, usageGapMs = null } = {}) {
     this.am = accountManager;
@@ -245,7 +249,19 @@ export class Prober {
     // endpoint 403s before answering anything, so probing is pure waste until the org
     // accepts OAuth again. Skipping also stops the every-60s hammer that ran 700+ times
     // on 2solarmax@ between 2026-09-18 and 09-20.
-    if (account.subscriptionGone) return { ok: false, status: 403 };
+    // SELF-CLEAR RECHECK (2026-10-08): the latch used to be terminal — the account was
+    // skipped here AND by ensureTokenFresh, so a single transient org-403 benched a live
+    // subscription until manual re-auth. Recheck it at a low cadence: a 200 clears the
+    // latch (applyUsageData), another org-403 keeps it benched.
+    if (account.subscriptionGone) {
+      // A latch set outside recordProbeError (state restore, tests, older builds) has no
+      // stamp: treat NOW as the latch moment so the first recheck waits a full window.
+      if (!account._subRecheckAt) account._subRecheckAt = Date.now();
+      const since = Date.now() - account._subRecheckAt;
+      if (since < SUB_RECHECK_MS) return { ok: false, status: 403 };
+      account._subRecheckAt = Date.now();
+      account._subRecheck = true;
+    }
     try {
       // Claude banked resets (cedar_ember) piggyback on the oauth poll — same
       // OWNER GATE as z.ai cards: a DISABLED account is never listed or claimed.
@@ -288,6 +304,8 @@ export class Prober {
     } catch (e) { // best-effort; never let a probe throw
       this.am.recordProbeError?.(account.index, e?.message || String(e), null);
       return { ok: false, status: null };
+    } finally {
+      account._subRecheck = false;
     }
   }
 

package/src/server.js

@@ -39,6 +39,9 @@ const isNetworkCode = c => Boolean(c) && NETWORK_ERROR_CODES.has(c);
 const NETWORK_SOAK_NONSTREAM_MS = Math.max(5_000,
   Number(process.env.MAXPOOL_NETWORK_SOAK_NONSTREAM_MS) || 45_000);
 const QUEUE_KEEPALIVE = 'event: ping\ndata: {}\n\n';
+// Claude Code's absolute per-stream ceiling (binary-verified 2026-10-07: idle timeout =
+// min(env-or-300s-floor, 1800000)). Nothing resets it; the stream dies at 30 min.
+const CLI_STREAM_CEILING_MS = 25 * 60_000;   // bound network holds strictly below it
 // 240s, strictly BELOW Claude Code's hard 300s stall floor. At the old 300_000 the two
 // timers were a dead heat and the client always won — maxpool's clock starts when a chunk
 // is READ from upstream, strictly before the client parses it. Result: `upstream idle
@@ -1732,12 +1735,19 @@ function computeQueueWindowMs({
   // (the `cc` alias sets 3h) otherwise licenses a multi-hour hold on a connection that is
   // simply gone. Error-fast + client reconnect beats an unattended hold.
   // Network holds are NOT special-cased short any more. maxpool already re-polls every ~1s
-  // and each retry issues a FRESH fetch, so a hold IS "keep probing, resume the moment any
-  // route returns" — exactly what an unattended agent needs to survive a connectivity blip.
-  // Failing fast at 2 minutes handed the turn to Claude Code's retry loop, which is the
-  // thing that loses accumulated work. Visibility is paid for by logging/TUI, not by
-  // truncating the wait.
-  if (cause === 'network' && networkMaxWaitMs != null) windowMs = Math.min(windowMs, Math.max(networkMaxWaitMs, streamClientToleranceMs || 0));
+  // Network holds are NOT special-cased short any more — but they must stay under the
+  // CLI's ABSOLUTE stream ceiling (30 min: Claude Code kills the stream at
+  // min(idle-env, 1800000ms) regardless of bytes flowing — pings reset the idle
+  // watchdog, NOT this ceiling; 49 client-abandoned 30-33-min holds measured
+  // 2026-10-07 when the client's 3h idle header licensed a 3h network hold).
+  // A hold past the ceiling is worse than erroring: the client that started the
+  // request is already gone when capacity returns. The client's own retry loop
+  // (which survives outages natively) takes over after our fast error.
+  // streamClientToleranceMs may be 3h from x-maxpool-client-stream-idle-ms, so it
+  // cannot bound a NETWORK hold — only a capacity/quota hold with real content.
+  if (cause === 'network') {
+    windowMs = Math.min(windowMs, CLI_STREAM_CEILING_MS);
+  }
   return windowMs;
 }
 
@@ -2573,8 +2583,18 @@ async function queueAndRetry(
   // floor applies with nothing resetting it — we must give up well before that so the user
   // sees a real message instead of a client-side timeout.
   if (cause === 'network') {
+    // The client's ABSOLUTE stream ceiling (Claude Code binary, 2026-10-07: idle timeout
+    // is min(env/300s floor, 1800000ms)) kills a stream at 30 min NO MATTER WHAT bytes
+    // flow — pings reset the idle watchdog, not this ceiling. Measured 2026-10-07: 49
+    // client-abandoned holds clustered at 1700-2100s (30-33 min) while streamHoldMaxMs
+    // licensed 7-day holds; the user saw "Waiting for API response · will retry in X"
+    // across sessions with no recovery because the hold outlived every client that
+    // started it. Cap network holds at 25 min: when the network is actually back the
+    // hold resumes in seconds; when it is not, error-fast lets the CLI's OWN retry
+    // (which survives outages and reconnects — the native behavior the owner compared
+    // against) take over instead of dying on our silent 30-min ceiling.
     const budgetMs = requestInfo.stream
-      ? Math.max(60_000, Number(queueConfig.networkMaxWaitMs) || 120_000)
+      ? Math.min(Math.max(60_000, Number(queueConfig.networkMaxWaitMs) || 120_000), CLI_STREAM_CEILING_MS)
       : NETWORK_SOAK_NONSTREAM_MS;
     requestInfo.networkSoakDeadline ||= Date.now() + budgetMs;
     if (Date.now() >= requestInfo.networkSoakDeadline) {