maxpool 1.5.79 → 1.5.86

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.5.79",
+  "version": "1.5.86",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/account-manager.js

@@ -111,6 +111,7 @@ const DEFAULT_SCHEDULER = {
   perAccountConcurrencyTarget: 3,  // D: soft per-account in-flight target; past it, capPenalty bites
   capPenaltyWeight: 10,            // steep penalty per unit of in-flight depth past D (throttle safety floor)
   paceCostWeight: 1.5,            // soft de-preference of accounts burning ahead of pace (was the ×6 term)
+  utilizationWeight: 3,           // RAW utilization cost — drives load balancing in the mid-range
   scarcityWeight: 6,              // legacy; superseded by paceCostWeight (kept so old configs don't error)
   // Reserve-account OVERFLOW model. A weekly-RESERVE account (util 0.85-0.95) used to
   // sit idle behind a healthy-only first pass; now it's eligible in the first pass but
@@ -140,22 +141,30 @@ const DEFAULT_SCHEDULER = {
   crossAccountThinkingMigration: true,
   // Cross-PROVIDER fallback policy for 'cc all' (profile=all), i.e. whether a session
   // may be served by a provider FAMILY other than its home (Claude ↔ GLM ↔ Kimi).
-  //   'never'         — (DEFAULT) a Claude session never spills onto a provider. NOTE:
-  //                     this governs the Claude→provider direction ONLY — it is NOT a
-  //                     same-family pin (a provider-origin session still reaches Claude).
-  //   'when-exhausted'— home family preferred; a Claude session falls back to GLM/Kimi
-  //                     only once all Claude accounts are unavailable, and a GLM session
-  //                     may fall to Kimi once GLM is exhausted.
-  //   'always'        — providers peer with Claude for a Claude/unknown session
-  //                     (load-balanced, not last-resort).
-  // DEFAULT is 'never' (2026-07-24): a Claude Code session stays on Anthropic. Routing
-  // Claude→GLM/Kimi proved unreliable (the coding legs 403/429 + ignore the model id),
-  // so cross-routing OUT of Claude is OFF by default; re-enable via the TUI routing
-  // cycle or config. This governs ONLY the Claude→provider direction.
-  // INVARIANT (all policies): a GLM/Kimi-origin session NEVER routes to an Anthropic
-  // account — Anthropic 400s on a non-`srvtoolu_` server_tool_use id; that direction
-  // is unfixable, not policy-tunable.
+  //
+  // SUPERSEDED by `routingMode` below — kept for backward-compat migration only. An
+  // old config carrying `crossProviderFallbackPolicy` but no `routingMode` is upgraded
+  // at boot: 'always' → 'balance', 'when-exhausted'/'never' → 'prefer-claude'. The
+  // per-provider `providers[<key>].claudeFallback` field still controls the same thing
+  // for its one provider under the legacy modes.
   crossProviderFallbackPolicy: 'never',
+  // ROUTING MODE — the single knob that governs how sessions are distributed across
+  // the account pool. Replaces the hidden per-session binding that made 'always' not
+  // actually balance. Reported 2026-08-10: with cross-provider 'always' set, 20
+  // long-lived sessions that happened to start on the same Anthropic account hammered
+  // it at 82% while two GLM accounts at 2%/9% sat idle — because 'always' only governed
+  // the FIRST request; after that the session was pinned to one account.
+  //
+  //   'balance'         — score EVERY request across the full pool. No session binding.
+  //                       Accounts drain evenly. The behaviour 'always' should always
+  //                       have been.
+  //   'prefer-claude'   — score every request, but Anthropic accounts outrank providers
+  //                       unless they are all loaded/exhausted.
+  //   'prefer-zai'      — GLM accounts preferred; Claude/Kimi fill overflow.
+  //   'prefer-kimi'     — Kimi preferred; Claude/GLM fill overflow.
+  //   'sticky'          — the old behaviour, made explicit. Sessions stay on the account
+  //                       they first land on until it goes hot, then rebalance.
+  routingMode: 'sticky',
   // The OTHER cross direction, independent of the policy above: may a provider-origin
   // (GLM/Kimi) session cross to the OTHER provider (GLM↔Kimi)? Default ON — both legs are
   // lenient and accept each other's ids, and it's the reliable direction the user wants
@@ -244,6 +253,20 @@ function parseResetHeader(value) {
 export class AccountManager {
   constructor(accounts, switchThreshold = 0.90, schedulerOptions = {}, dependencies = {}) {
     this.scheduler = { ...DEFAULT_SCHEDULER, ...schedulerOptions };
+    // Migrate the legacy single-value policy to the new mode if the caller (config)
+    // set `crossProviderFallbackPolicy` but not `routingMode`. The per-provider
+    // `providers[<key>].claudeFallback` still works under the prefer-* modes.
+    // IMPORTANT: when schedulerOptions is empty (no explicit crossProviderFallbackPolicy),
+    // the DEFAULT_SCHEDULER value 'never' triggers migration. But the old behaviour
+    // under 'never' WAS sticky pinning — so the default must map to 'sticky', not
+    // 'prefer-claude'. Only an EXPLICIT 'always' in the caller's config changes the mode.
+    if (!schedulerOptions.routingMode) {
+      const leg = schedulerOptions.crossProviderFallbackPolicy;
+      if (leg === 'always') this.scheduler.routingMode = 'balance';
+      else if (leg === 'when-exhausted') this.scheduler.routingMode = 'prefer-claude';
+      // 'never' or unset → sticky (the historical default: sessions pin to one account)
+      else this.scheduler.routingMode = 'sticky';
+    }
     this._refreshAccessToken = dependencies.refreshAccessToken || refreshAccessToken;
     this.accounts = accounts.map((acct, index) => ({
       index,
@@ -1230,7 +1253,29 @@ export class AccountManager {
    *  score loop), so a healthy bound account never ping-pongs. */
   _isBoundAccountHot(account) {
     return this._isSessionQuotaUnavailable(account)
-      || ['reserve', 'critical', 'exhausted'].includes(this._weeklyPaceState(account));
+      || ['reserve', 'critical', 'exhausted'].includes(this._weeklyPaceState(account))
+      // SESSION-window pressure counts too. _isSessionQuotaUnavailable only fires at
+      // switchThreshold (0.90), and the weekly bands don't see the 5h window at all —
+      // so an account at 82% of a 5h window was "not hot" and every bound session
+      // stayed on it while idle accounts sat at 2%. Measured 2026-08-10: Anthropic at
+      // Ses 82% / Wk 66% hammered flat-out beside two GLM accounts at 2% and 9%.
+      // Uses the SOFT band (0.65), not reserve (0.85): the point is to shed load while
+      // there is still headroom, not at the cliff edge. _shouldRebalanceBoundSession
+      // still requires a clearly-cheaper, strictly-healthier target, so a hot account
+      // with no better alternative keeps its sessions — this only opens the question.
+      || this._sessionWindowUsage(account) >= this.scheduler.weeklySoftThreshold;
+  }
+
+  /** Fraction of the SESSION (5h) window consumed, across both quota shapes.
+   *  Anthropic reports unified5h; a provider reports providerSes. Returns 0 when
+   *  unknown — an unreadable window must never make an account look hot. */
+  _sessionWindowUsage(account) {
+    const q = account?.quota;
+    if (!q) return 0;
+    const vals = [];
+    if (q.unified5h != null) vals.push(clamp01(q.unified5h));
+    if (q.providerSes != null) vals.push(clamp01(q.providerSes));
+    return vals.length ? Math.max(...vals) : 0;
   }
 
   /** Decide whether a bound session should leave its (hot) account THIS request.
@@ -1334,6 +1379,18 @@ export class AccountManager {
     // candidate loop lands on the least-loaded healthy account, spreading the move.
     if ((WEEKLY_TIER[this._weeklyRawState(bound)] ?? 0) >= WEEKLY_TIER.reserve) return true;
 
+    // Same absolute escape for the SESSION window. The tier test below compares WEEKLY
+    // tiers, so a bound account burning down its 5h window can never satisfy it while
+    // its weekly is merely soft — which is exactly how an account at Ses 82% kept every
+    // session while idle accounts sat at 2%. Requires a MATERIALLY cheaper target so
+    // this can't churn between two similarly-loaded accounts.
+    // No score margin here, for the reason the weekly escape documents directly above:
+    // boundScore*0.5 on an IDLE near-cap account sits below every candidate's minimum
+    // score, so the margin is unsatisfiable and pins the session to the account it is
+    // meant to relieve. The candidate loop has already kept only RAW-healthy targets,
+    // and preserving the last of a 5h window beats concurrency spread.
+    if (this._sessionWindowUsage(bound) >= this.scheduler.weeklyReserveThreshold) return true;
+
     // Otherwise the trigger was PACE-only on a RAW-healthy account — a fast-burner
     // that still has real absolute headroom (RAW soft but pace reserve/critical,
     // e.g. 79% used resetting in ~3.5d). Keep the conservative gate so it isn't
@@ -1430,10 +1487,15 @@ export class AccountManager {
     const c = { total: 0, quota: 0, transient: 0, network: 0, disabled: 0, error: 0, other: 0 };
     for (const a of this.accounts) {
       if (a.type === 'provider') continue;
+      // DISABLED accounts are out of the serving pool — counting them in `total`
+      // made the 429 say "all 9 Claude accounts are momentarily busy" when ONE
+      // enabled account was reconnecting and EIGHT were disabled with dead tokens
+      // (reported 2026-08-13). The user reads 9-busy as fleet saturation; the
+      // truth is 1-busy + 8-off. Track them for the message, exclude from cause math.
+      if (a.enabled === false) { c.disabled++; continue; }
       c.total++;
       const cause = this._retryInfo(a, model)?.cause || 'unavailable';
-      if (cause === 'disabled') c.disabled++;
-      else if (cause === 'error') c.error++;
+      if (cause === 'error') c.error++;
       else if (QUOTA.has(cause)) c.quota++;
       else if (TRANSIENT.has(cause)) {
         c.transient++;
@@ -1443,7 +1505,7 @@ export class AccountManager {
           && (a.cooldownUntil - Date.now()) <= this.scheduler.networkCooldownMs) c.network++;
       } else c.other++;
     }
-    const eligible = c.total - c.disabled - c.error;
+    const eligible = c.total - c.error;
     c.dominant = eligible <= 0 ? 'none'
       : c.quota >= Math.max(1, Math.ceil(eligible / 2)) ? 'quota'
         : c.transient > 0 ? 'transient' : 'other';
@@ -1531,7 +1593,15 @@ export class AccountManager {
         return preferred;
       }
     }
-    const bound = this._boundAccount(requestInfo.sessionKey, profile, excludedIndexes, requestInfo);
+    // Session binding only applies in 'sticky' mode. Every other mode scores each
+    // request independently — the whole point of 'balance' is that 20 sessions that
+    // happened to start on the same account do NOT keep hammering it while others
+    // idle. Warmup-pull still works under the non-sticky modes because it keys on
+    // `_isWarming`, not on a binding.
+    const isStickyMode = this.scheduler.routingMode === 'sticky';
+    const bound = isStickyMode
+      ? this._boundAccount(requestInfo.sessionKey, profile, excludedIndexes, requestInfo)
+      : null;
     if (bound) {
       // Warmup-pull: onboard a freshly-ADDED account (added mid-session, no reload)
       // by DIRECTLY re-homing this migration-safe session onto the warming account,
@@ -1787,18 +1857,52 @@ export class AccountManager {
   // (10/20) → fallback-only. A foreign session is provider-only regardless.
   _effectivePriority(account, requestInfo = {}) {
     const base = Number.isFinite(account.priority) ? account.priority : 0;
+    const mode = this.scheduler.routingMode;
+    const incompatible = this._effectiveIncompatible(requestInfo).incompatible;
+    // An incompatible session is pinned to its provider family — no mode overrides that.
+    if (incompatible) {
+      const isHome = account.type === 'provider'
+        && (mode === 'prefer-zai' ? account.provider === 'zai'
+          : mode === 'prefer-kimi' ? account.provider === 'kimi' : true);
+      return isHome ? 0 : base;
+    }
+    // Balance: every account peers at priority 0. Accounts are ranked by score alone.
+    if (mode === 'balance') return 0;
+    // Prefer-* modes: the preferred family sits at 0, everything else at its base
+    // priority (10 for GLM, 20 for Kimi). So the preferred family is chosen first and
+    // the others only when every preferred account is unavailable (the score loop's
+    // pass-1 admits reserve accounts, so a loaded preferred account DOES give way).
+    if (mode === 'prefer-claude') {
+      return account.type === 'provider' ? base : 0;
+    }
+    if (mode === 'prefer-zai') {
+      return account.provider === 'zai' ? 0 : base;
+    }
+    if (mode === 'prefer-kimi') {
+      return account.provider === 'kimi' ? 0 : base;
+    }
+    // Sticky: legacy behaviour — the 'always' policy promoted providers to 0.
     if (account.type === 'provider'
-        && this._claudeFallbackFor(account.provider) === 'always'
-        && !this._effectiveIncompatible(requestInfo).incompatible) {
+        && this._claudeFallbackFor(account.provider) === 'always') {
       return 0;
     }
     return base;
   }
 
+  setProviderRoutingMode(mode) {
+    if (!['balance', 'prefer-claude', 'prefer-zai', 'prefer-kimi', 'sticky'].includes(mode)) return false;
+    this.scheduler.routingMode = mode;
+    console.log(`[Maxpool] Routing mode set to "${mode}"`);
+    return true;
+  }
+
+  // Legacy shim — the TUI routing screen still cycles this. Maps to the new mode.
   setCrossProviderFallbackPolicy(policy) {
     if (!['never', 'when-exhausted', 'always'].includes(policy)) return false;
     this.scheduler.crossProviderFallbackPolicy = policy;
-    console.log(`[Maxpool] Cross-provider fallback policy set to "${policy}"`);
+    // Map to the new mode so the binding/priority logic agrees.
+    this.scheduler.routingMode = policy === 'always' ? 'balance' : policy === 'when-exhausted' ? 'prefer-claude' : 'sticky';
+    console.log(`[Maxpool] Cross-provider fallback policy set to "${policy}" (routing mode: ${this.scheduler.routingMode})`);
     return true;
   }
 
@@ -1872,15 +1976,20 @@ export class AccountManager {
 
     // Compatible session — includes Kimi and GLM-without-server-tools, whose regular
     // tool_use ids pass Anthropic's loose validation, AND ordinary Claude sessions.
-    // Claude is eligible + preferred (priority 0). Provider fallback is the SAFE
-    // direction (lenient providers accept Anthropic ids/signatures) and is
-    // policy-gated ONLY: 'never' keeps a Claude session on Claude; 'when-exhausted'
-    // lets providers serve as a priority-fallback; 'always' peers them
-    // (_effectivePriority). Signed thinking no longer bars providers here — but its
-    // live MIGRATION stays Claude-only (see the rebalance guard).
-    // PER-PROVIDER Claude→provider gate (GLM and Kimi steer independently). Unset ⇒
-    // inherits the legacy global policy, so behavior is unchanged on upgrade.
-    if (account.type === 'provider' && this._claudeFallbackFor(account.provider) === 'never') return false;
+    // Under the new routing modes, providers are eligible whenever the mode allows
+    // them to peer (balance + prefer-{zai,kimi}) or serve as fallback
+    // (prefer-claude + sticky). The legacy per-provider `claudeFallback: 'never'`
+    // gate only applies under 'sticky' — the old behaviour it was written for.
+    if (account.type === 'provider') {
+      const mode = this.scheduler.routingMode;
+      // Balance and prefer-{zai,kimi}: providers are always eligible (scored, not gated).
+      if (mode === 'balance' || mode === 'prefer-zai' || mode === 'prefer-kimi') return true;
+      // Prefer-claude: providers serve as overflow. Still eligible — priority handles the
+      // preference; an available Claude account always wins on priority.
+      if (mode === 'prefer-claude') return true;
+      // Sticky: the legacy per-provider gate applies — this is the mode it was written for.
+      if (this._claudeFallbackFor(account.provider) === 'never') return false;
+    }
     return true;
   }
 
@@ -2057,6 +2166,14 @@ export class AccountManager {
     // soft de-preference of accounts burning ahead of an even pace. Never a bench.
     const paceCost = this._accountScarcity(account, now) * this.scheduler.paceCostWeight;
 
+    // RAW utilization cost — direct, not pace-adjusted. The pace cost above discounts
+    // by how far into the window you are, so an account at 80% with 2h left is only
+    // "slightly ahead of pace" → tiny cost. That's right for avoiding premature
+    // benching, but wrong for load balancing: an account at 80% should be clearly less
+    // attractive than one at 10% even if both are "on pace". Measured 2026-08-10: cc at
+    // 80% scored 52.30 vs glm at 10% at 52.15 — a 0.15 gap drowned by round-robin.
+    const utilizationCost = this._rawUtilization(account) * this.scheduler.utilizationWeight;
+
     // Per-model weekly de-preference: an account whose scoped weekly for THIS
     // request's model (e.g. Fable) is high-but-not-exhausted is a poor pick for
     // that model — shed its load toward healthier accounts BEFORE the hard bench
@@ -2083,7 +2200,7 @@ export class AccountManager {
     // default) learns the real number within a cycle. `probing`/requalify still
     // flags a never-seen account for learning — that path is unchanged.
 
-    return concurrency + capPenalty + paceCost + scopedPace + spread + ramp + reserveCost + failurePenalty;
+    return concurrency + capPenalty + paceCost + utilizationCost + scopedPace + spread + ramp + reserveCost + failurePenalty;
   }
 
   /**
@@ -2137,6 +2254,29 @@ export class AccountManager {
     return scarcity;
   }
 
+  /** RAW utilization (0..1) — not pace-adjusted. Reads the same fields as
+   *  _accountScarcity but WITHOUT the elapsed-fraction discount. This is the signal
+   *  the load balancer needs: an account at 80% is more expensive than one at 10%,
+   *  full stop. */
+  _rawUtilization(account) {
+    const q = account?.quota;
+    if (!q) return 0;
+    // SESSION windows use raw utilization — headroom is consumed immediately and an
+    // 80%-used 5h window is genuinely more expensive than a 10%-used one right now.
+    let util = 0;
+    if (q.unified5h != null) util = Math.max(util, clamp01(q.unified5h));
+    if (q.providerSes != null) util = Math.max(util, clamp01(q.providerSes));
+    // WEEKLY windows use PACE-ADJUSTED utilization (via _windowScarcity), not raw —
+    // a 79% account resetting in 2h has plenty of headroom and should be cheap to
+    // spend (the use-it-or-lose-it principle). Using raw weekly would break that.
+    // The pace cost already carries this signal; here we add only the session signal
+    // the pace cost was too weak to express.
+    if (q.tokensLimit != null && q.tokensLimit > 0 && q.tokensRemaining != null) {
+      util = Math.max(util, 1 - q.tokensRemaining / q.tokensLimit);
+    }
+    return util;
+  }
+
   _windowScarcity(util, resetMs, windowLen, now = Date.now()) {
     const used = clamp01(util);
     if (!resetMs || resetMs <= now) return used; // unknown / just-reset → face value
@@ -3050,7 +3190,7 @@ export class AccountManager {
       // Default model maps — match what `cc all` sends via headers. Users can override
       // per-provider in config. Without these z.ai returns [1210 Invalid API parameter].
       const defaultModelMap = entry.provider === 'zai'
-        ? { opus: 'glm-5.2', sonnet: 'glm-5.2', haiku: 'glm-5.1', default: 'glm-5.2' }
+        ? { opus: 'glm-5.3', sonnet: 'glm-5.3', haiku: 'glm-5.3', default: 'glm-5.3' }
         : null;
       const defaultModel = entry.provider === 'kimi' ? 'kimi-k3' : null;
       this.upsertRuntimeAccount({

package/src/index.js

@@ -985,6 +985,13 @@ async function serverWorkerCommand() {
     pauseAdmission: () => accountManager.setAdmissionPaused(true),
     resumeAdmission: () => accountManager.setAdmissionPaused(false),
     restartNow: requestReload,
+    // The pre-drain only earns its cost on the COLD path (the socket closes, so an
+    // in-flight request would be severed). Seamless keeps serving them post-baton.
+    isSeamless: () => {
+      const forceCold = process.env.MAXPOOL_TEST_FORCE_COLD_RESTART === '1'
+        || process.env.MAXPOOL_TUI_COLD_RESTART === '1';
+      return !forceCold && reloadStrategy({ supervised }) === 'seamless';
+    },
     // Configurable so ops can tune the bounded pre-restart drain, and so the
     // integration test can exercise the force-restart path without a 10s wait.
     ...(Number.isFinite(config.restartDrainTimeoutMs) ? { drainTimeoutMs: config.restartDrainTimeoutMs } : {}),

package/src/restart-controller.js

@@ -12,16 +12,31 @@ export class RestartController {
     // heartbeat — pins the restart on "Restart pending…" forever (the stuck-`r`
     // bug). Dropped requests reconnect after restart, as the message promises.
     drainTimeoutMs = 10_000,
+    // When the reload is SEAMLESS, the old worker keeps serving its in-flight
+    // requests through the baton handoff (releaseBatonAndDrain, 60s+ cap) — so
+    // pre-draining here buys nothing and costs every OTHER session a hard 503 for
+    // the whole window. With ~30 sessions running 30-60s requests the 10s drain
+    // NEVER completes naturally; it times out every time. So it was 10 guaranteed
+    // seconds of "Maxpool is restarting" across the fleet in exchange for zero
+    // drained requests — the reported 503 storm. Returns true when the restart
+    // path will hand off seamlessly, in which case we skip straight to the swap.
+    isSeamless = () => false,
     setTimeoutFn = setTimeout,
     clearTimeoutFn = clearTimeout,
+    setIntervalFn = setInterval,
+    clearIntervalFn = clearInterval,
   }) {
     this.pauseAdmission = pauseAdmission;
     this.resumeAdmission = resumeAdmission;
     this.restartNow = restartNow;
     this.log = log;
     this.drainTimeoutMs = drainTimeoutMs;
+    this.isSeamless = isSeamless;
     this.setTimeoutFn = setTimeoutFn;
     this.clearTimeoutFn = clearTimeoutFn;
+    this.setIntervalFn = setIntervalFn;
+    this.clearIntervalFn = clearIntervalFn;
+    this._progressTimer = null;
     this.activeRequests = new Set();
     this.upstreamRequests = new Set();
     this.pending = false;
@@ -48,6 +63,7 @@ export class RestartController {
     this.pending = false;
     this.restarting = false;
     if (this._drainTimer) { this.clearTimeoutFn(this._drainTimer); this._drainTimer = null; }
+    if (this._progressTimer) { this.clearIntervalFn(this._progressTimer); this._progressTimer = null; }
     this.resumeAdmission();
     return true;
   }
@@ -77,9 +93,33 @@ export class RestartController {
       return;
     }
 
+    // Seamless handoff → do not pre-drain (see isSeamless above). The in-flight
+    // requests finish on the OLD worker after the baton passes; holding admission
+    // here would only 503 the rest of the fleet for nothing.
+    if (this.isSeamless()) {
+      this.log(`[Maxpool] Restarting now — ${this.upstreamRequests.size} in-flight request(s) finish on the current version; new requests go to the updated one.`);
+      this._restart();
+      return;
+    }
+
     this.pending = true;
     const queuedOrIdle = Math.max(0, this.activeRequests.size - this.upstreamRequests.size);
     this.log(`[Maxpool] Restart pending; admission paused while ${this.upstreamRequests.size} upstream request(s) finish (up to ${Math.round(this.drainTimeoutMs / 1000)}s). ${queuedOrIdle} queued/idle request(s) will reconnect after restart.`);
+    // Progress ticks while draining. Without these the TUI shows one line and then
+    // silence for the whole window, which reads as "nothing happened" — the reported
+    // complaint. A countdown makes the wait legible and bounded.
+    this._tick = 0;
+    const tickMs = 2000;
+    this._progressTimer = this.setIntervalFn?.(() => {
+      if (!this.pending || this.restarting) return;
+      this._tick += tickMs;
+      const left = Math.max(0, Math.ceil((this.drainTimeoutMs - this._tick) / 1000));
+      const n = this.upstreamRequests.size;
+      if (n === 0) return;   // _maybeRestart is about to fire
+      this.log(`[Maxpool] Restarting — waiting for ${n} request(s) to finish (${left}s left)…`);
+    }, tickMs);
+    this._progressTimer?.unref?.();
+
     // Force the restart if the drain overruns — never hang on a long stream.
     this._drainTimer = this.setTimeoutFn(() => {
       if (!this.pending || this.restarting) return;
@@ -102,6 +142,10 @@ export class RestartController {
       this.clearTimeoutFn(this._drainTimer);
       this._drainTimer = null;
     }
+    if (this._progressTimer) {
+      this.clearIntervalFn(this._progressTimer);
+      this._progressTimer = null;
+    }
     this.restartNow();
   }
 }

package/src/secret-resolver.js

@@ -10,13 +10,59 @@
  */
 import { execFile } from 'node:child_process';
 import { promisify } from 'node:util';
+import { readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
 
 const execFileAsync = promisify(execFile);
 
 const DEFAULT_PROJECT = 'mokka-business-automations';
 
-export async function resolveSecret(secretName, { project = DEFAULT_PROJECT, timeoutMs = 45_000 } = {}) {
+// A single `gcloud secrets versions access` costs ~17-33s on this machine (python
+// interpreter start + an ADC round-trip), and running them CONCURRENTLY makes it
+// worse, not better: measured 2026-08-10, 5 secrets in parallel → 3 of 5 hit the
+// 45s timeout; the same 5 sequentially → 4 of 5. Either way providers boot as
+// "secret-unresolved", which is what stranded `kimi max@gomokka.com` and let the
+// header path create a duplicate `kimi-fallback` row beside it.
+//
+// The workspace already maintains a local plaintext cache of these same secrets
+// (written by scripts/load-secrets.sh, mode 0600, sanctioned by the secrets-directory
+// rule). Reading it is sub-millisecond and needs no auth, so try it FIRST and fall
+// back to gcloud only for what it does not carry.
+let cacheMemo;
+let cachePathMemo;
+
+function cachePath() {
+  if (!cachePathMemo) cachePathMemo = join(homedir(), '.claude', '.credentials-cache');
+  return cachePathMemo;
+}
+
+/** Test seam: drop the memo so a re-read picks up a freshly-written cache. */
+export function __resetSecretCache() { cacheMemo = undefined; cachePathMemo = undefined; }
+
+function readCredentialCache() {
+  if (cacheMemo !== undefined) return cacheMemo;
+  try {
+    const parsed = JSON.parse(readFileSync(cachePath(), 'utf8'));
+    cacheMemo = (parsed && typeof parsed === 'object') ? parsed : null;
+  } catch {
+    cacheMemo = null;   // absent/unreadable/corrupt → gcloud path, never a crash
+  }
+  return cacheMemo;
+}
+
+function fromCache(secretName) {
+  const cache = readCredentialCache();
+  const v = cache?.[secretName];
+  return (typeof v === 'string' && v.trim()) ? v.trim() : null;
+}
+
+export async function resolveSecret(secretName, { project = DEFAULT_PROJECT, timeoutMs = 45_000, useCache = true } = {}) {
   if (!secretName || typeof secretName !== 'string') return null;
+  if (useCache) {
+    const cached = fromCache(secretName);
+    if (cached) return cached;
+  }
   try {
     const { stdout } = await execFileAsync(
       'gcloud',
@@ -42,8 +88,22 @@ export async function resolveSecret(secretName, { project = DEFAULT_PROJECT, tim
 export async function resolveSecrets(secretNames, opts = {}) {
   const unique = [...new Set(secretNames.filter(Boolean))];
   if (!unique.length) return {};
-  const entries = await Promise.all(
-    unique.map(async name => [name, await resolveSecret(name, opts)]),
-  );
-  return Object.fromEntries(entries.filter(([, v]) => v != null));
+
+  // Cache hits first — free, and on this machine that is usually all of them.
+  const out = {};
+  const misses = [];
+  for (const name of unique) {
+    const cached = opts.useCache === false ? null : fromCache(name);
+    if (cached) out[name] = cached;
+    else misses.push(name);
+  }
+
+  // Only genuine misses pay gcloud, and SEQUENTIALLY — concurrent invocations
+  // contend and time each other out (measured: 3 of 5 failed in parallel vs 1 of 5
+  // sequentially). Sequential over a handful of misses is strictly better here.
+  for (const name of misses) {
+    const v = await resolveSecret(name, { ...opts, useCache: false });
+    if (v != null) out[name] = v;
+  }
+  return out;
 }

package/src/server.js

@@ -211,7 +211,7 @@ export function createProxyServer(accountManager, config, hooks = {}) {
             type: 'error',
             error: {
               type: 'restart_in_progress',
-              message: 'Maxpool is restarting. Retry immediately.',
+              message: 'Maxpool is restarting — finishing in-flight requests first. This retries automatically; your session is not lost.',
             },
           }));
           return;
@@ -247,7 +247,7 @@ export function createProxyServer(accountManager, config, hooks = {}) {
           type: 'error',
           error: {
             type: 'restart_in_progress',
-            message: 'Maxpool is restarting. Retry immediately.',
+            message: 'Maxpool is restarting — finishing in-flight requests first. This retries automatically; your session is not lost.',
           },
         }));
         return;
@@ -1555,7 +1555,9 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
     return `This session is too large for the GLM/Kimi fallbacks (their ~256K limit) — it needs a 1M-context Claude account, and they're all busy right now.${eta} It sends as soon as one frees; /compact shortens the session if you'd rather not wait.`;
   }
 
-  const claudeCount = accountManager.accounts.filter(a => a.type !== 'provider').length;
+  // ENABLED only — a disabled account is not "at its limit", it is off; counting it
+  // made a 1-enabled-account pool report "all 9 accounts at their limit".
+  const claudeCount = accountManager.accounts.filter(a => a.type !== 'provider' && a.enabled !== false).length;
   // Only name the providers when this pool HAS them AND they are actually allowed to
   // serve this request. With crossProviderFallbackPolicy 'never' (the default) they are
   // barred by POLICY, not saturated — saying they are "at their limit" is a lie that
@@ -1592,12 +1594,20 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
   if (census && census.dominant === 'transient') {
     const netly = census.network > 0;
     const what = netly
-      ? `${census.network} of ${census.total} Claude accounts are in a brief reconnect cooldown`
-      : `all ${census.total} Claude accounts are momentarily busy`;
-    return `No account can take this request right now — ${what}, not out of quota. Retry in ~${formatRetryDuration(retryAfter)}; it clears on its own.${providersBarredHint}`;
+      ? `${census.network} of ${census.total} Claude account${census.total === 1 ? '' : 's'} are in a brief reconnect cooldown`
+      : census.total === 1
+        ? 'the only enabled Claude account is momentarily busy'
+        : `all ${census.total} Claude accounts are momentarily busy`;
+    // When most of the pool is DISABLED, "1 account busy" begs the real question —
+    // name the disabled share so the fix (re-enable/re-auth) is visible in the message
+    // instead of reading like a fleet that mysteriously shrank.
+    const offNote = census.disabled > 0
+      ? ` (${census.disabled} account${census.disabled === 1 ? '' : 's'} disabled)`
+      : '';
+    return `No account can take this request right now — ${what}, not out of quota${offNote}. Retry in ~${formatRetryDuration(retryAfter)}; it clears on its own.${providersBarredHint}`;
   }
 
-  return `No account can take this request right now — all ${claudeCount} Claude accounts${providersClause} are ${waitLong ? 'at their limit' : 'momentarily at their limit'}. Retry in ~${formatRetryDuration(retryAfter)}.${providersBarredHint}`;
+  return `No account can take this request right now — all ${claudeCount} Claude account${claudeCount === 1 ? '' : 's'}${providersClause} are ${waitLong ? 'at their limit' : 'momentarily at their limit'}. Retry in ~${formatRetryDuration(retryAfter)}.${providersBarredHint}`;
 }
 
 // A provider (GLM/Kimi) rejecting a request whose token count exceeds its context
@@ -2612,9 +2622,9 @@ function prepareRuntimeProviders(accountManager, headers) {
 
   const zaiToken = headerValue(headers, 'x-maxpool-zai-token');
   if (zaiToken && !configTokens.has(zaiToken)) {
-    const opus = headerValue(headers, 'x-maxpool-zai-opus-model') || headerValue(headers, 'x-maxpool-zai-model') || 'glm-5.2';
+    const opus = headerValue(headers, 'x-maxpool-zai-opus-model') || headerValue(headers, 'x-maxpool-zai-model') || 'glm-5.3';
     const sonnet = headerValue(headers, 'x-maxpool-zai-sonnet-model') || headerValue(headers, 'x-maxpool-zai-model') || opus;
-    const haiku = headerValue(headers, 'x-maxpool-zai-haiku-model') || 'glm-5.1';
+    const haiku = headerValue(headers, 'x-maxpool-zai-haiku-model') || 'glm-5.3';
     accountManager.upsertRuntimeAccount({
       name: 'glm-fallback',
       type: 'provider',
@@ -2739,6 +2749,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
   let committed = res.headersSent;
   let readFailed = false;
 
+
   // We're now committed to streaming a real upstream response body onto this
   // response — there is no more failover for this forward. Stop the queue
   // heartbeat (if this was a resumed held stream) BEFORE the first real byte, so
@@ -2854,6 +2865,20 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
     reader.cancel().catch(() => {});
     if (!readFailed) {
       if (!committed && !res.headersSent) res.writeHead(status, responseHeaders);
+      // Truncation sentinel FIRST (before end()) — a write after end() throws and
+      // would be swallowed, silently producing the truncated stream this guards.
+      // Fires ONLY on a genuinely truncated stream: a non-empty residual SSE buffer
+      // means the upstream died mid-event (an event without its '\n\n' terminator).
+      // Complete-event streams without a terminal message_stop (z.ai compat shape,
+      // test fixtures) end with an EMPTY residual and pass through unchanged.
+      if ((committed || res.headersSent) && sseBuffer.trim() && !res.destroyed && !res.writableEnded) {
+        try {
+          res.write(`event: error\ndata: ${JSON.stringify({
+            type: 'error',
+            error: { type: 'api_error', message: 'Upstream closed the stream before the response completed. Retry the message.' },
+          })}\n\n`);
+        } catch { /* client already gone */ }
+      }
       if (!res.writableEnded) res.end();
     }
   }

package/src/tui.js

@@ -426,7 +426,12 @@ export class TUI {
     } else if (k === 'r') {
       this._confirm(
         'Restart Maxpool?',
-        'Pause new requests, drain active work, then start the updated server.',
+        // Say what actually happens to the user's sessions. The old text ("pause new
+        // requests, drain active work") was accurate but hid the consequence: with
+        // several long requests in flight, NEW requests get a 503 for up to the drain
+        // window, and every running session sees a retry. Reported 2026-08-10: "I click
+        // restart, then yes, and NOTHING happens" — while ~30 sessions were 503-ing.
+        this._restartConfirmDetail(),
         // Do NOT stop the TUI here — let the reload path own it: cold restart stops
         // it in restartWorkerNow, a seamless reload in releaseBatonAndDrain (on
         // MSG_RELEASE). If the reload ROLLS BACK (new worker fails to boot), neither
@@ -558,6 +563,22 @@ export class TUI {
     this.mode = 'normal';
   }
 
+  /** Live description of what pressing restart will do RIGHT NOW — counts the
+   *  in-flight requests that must drain and states the worst-case pause, so the
+   *  confirm is honest about the cost instead of hiding it behind "drain active work". */
+  _restartConfirmDetail() {
+    const inFlight = this.active?.size || 0;
+    const sessions = new Set([...(this.active?.values() || [])].map(r => r.sessionKey).filter(Boolean)).size;
+    const drainSec = Math.round((this.config?.restartDrainTimeoutMs ?? 10_000) / 1000);
+    if (inFlight === 0) {
+      return 'Nothing is in flight, so this restarts immediately. Running sessions reconnect on their next request.';
+    }
+    const s = sessions === 1 ? '' : 's';
+    return `${inFlight} request${inFlight === 1 ? '' : 's'} from ${sessions} session${s} are in flight. `
+      + `They finish first (up to ${drainSec}s), and during that window NEW requests get a "restarting" retry. `
+      + 'Sessions are not lost — they reconnect automatically.';
+  }
+
   _keyAccounts(k) {
     if (k === 'a') {
       // ONE entry point for every account type. Previously `l` (browser) and `k`
@@ -783,12 +804,16 @@ export class TUI {
       );
     } else if (k === 'p' && this.am.accounts.some(account => account.type !== 'provider')) {
       this._startSelection('prefer');
+    } else if (k === 'f' || k === 'F' || k === 'm' || k === 'M') {
+      // Cycle the named routing mode — reversible + non-destructive, so no confirm
+      // dialog (unlike restart/delete). `f` is kept for muscle memory from the old
+      // cross-provider knob this replaced; `m` is the mnemonic (mode).
+      this._cycleRoutingMode();
     } else if (k === 'g' || k === 'k') {
+      // Per-provider Claude→provider fallback. Only affects routing under `sticky`
+      // mode — under balance/prefer-* the mode itself controls eligibility. Still
+      // safe to set (it'll apply if you switch back to sticky).
       this._cycleProviderClaudeFallback(k === 'g' ? 'zai' : 'kimi');
-    } else if (k === 'f' || k === 'F') {
-      // Cycle the cross-provider fallback policy in place — reversible + non-destructive,
-      // so no confirm dialog (unlike restart/delete). Accept F too (Shift-f muscle memory).
-      this._cycleCrossProviderPolicy();
     } else if (k === 'esc' || k === 'q') {
       this.mode = 'normal';
     }
@@ -822,14 +847,38 @@ export class TUI {
     apply();
   }
 
-  async _cycleCrossProviderPolicy() {
-    const order = ['never', 'when-exhausted', 'always'];
-    const cur = this.am._crossProviderFallbackPolicy();
-    const next = order[(order.indexOf(cur) + 1) % order.length];
-    this.am.setCrossProviderFallbackPolicy(next);
-    this.config.scheduler = { ...(this.config.scheduler || {}), crossProviderFallbackPolicy: next };
+  /** The five named routing modes, in cycle order. Each carries the plain-English
+   *  line the header shows, so the name the operator picks and the behaviour they
+   *  get are described by ONE string — the old `never/when-exhausted/always` knob
+   *  read as "load balance across everything" and did nothing of the sort. */
+  static ROUTING_MODES = [
+    { id: 'balance', label: 'Balance all',
+      help: 'Score every request across Claude, GLM and Kimi. Busiest accounts get less.' },
+    { id: 'prefer-claude', label: 'Prefer Claude',
+      help: 'Claude first; GLM/Kimi pick up the overflow when Claude is loaded.' },
+    { id: 'prefer-zai', label: 'Prefer GLM',
+      help: 'GLM first; Claude/Kimi pick up the overflow when GLM is loaded.' },
+    { id: 'prefer-kimi', label: 'Prefer Kimi',
+      help: 'Kimi first; Claude/GLM pick up the overflow when Kimi is loaded.' },
+    { id: 'sticky', label: 'One account per session',
+      help: 'Each session stays on the account it started on. No spreading.' },
+  ];
+
+  _routingModeDef(id) {
+    return TUI.ROUTING_MODES.find(m => m.id === id) || TUI.ROUTING_MODES[0];
+  }
+
+  async _cycleRoutingMode() {
+    const modes = TUI.ROUTING_MODES;
+    const cur = this.am.scheduler?.routingMode || 'sticky';
+    const next = modes[(modes.findIndex(m => m.id === cur) + 1) % modes.length];
+    this.am.setProviderRoutingMode?.(next.id);
+    this.config.scheduler = { ...(this.config.scheduler || {}), routingMode: next.id };
+    // Clear the stale legacy policy so it can't contradict the new mode in
+    // /maxpool/status or be read by code that still references it.
+    delete this.config.scheduler.crossProviderFallbackPolicy;
     await this.saveConfig(this.config);
-    this._addLog(`Cross-provider fallback: ${next}`);
+    this._addLog(`Routing: ${next.label} — ${next.help}`);
   }
 
   _keySelect(k) {
@@ -912,7 +961,20 @@ export class TUI {
     const nonProv = [];
     const prov = [];
     this.am.accounts.forEach((a, i) => (a.type === 'provider' ? prov : nonProv).push(i));
-    return [...nonProv, ...prov];
+    // GROUP providers by family (all GLM together, all Kimi together). Providers land
+    // in the config array in the order they were ADDED, so a GLM account added after a
+    // Kimi rendered below it — same-provider accounts split across the table (reported
+    // 2026-08-17: "accounts need to be grouped by provider... you shouldn't be mixing
+    // Anthropic with Z.ai and Moonshot"). Stable: group by first-seen order, preserving
+    // the within-family order and never reordering OAuth rows.
+    const familyFirstSeen = new Map();
+    for (const idx of prov) {
+      const fam = this.am.accounts[idx].provider || 'other';
+      if (!familyFirstSeen.has(fam)) familyFirstSeen.set(fam, []);
+      familyFirstSeen.get(fam).push(idx);
+    }
+    const provGrouped = [...familyFirstSeen.values()].flat();
+    return [...nonProv, ...provGrouped];
   }
 
   _selectableIndexes(action) {
@@ -1400,19 +1462,35 @@ export class TUI {
       const how = this._autoUpdateOn() ? 'applying automatically · or press u now' : 'press u to update now';
       lines.push('  ' + yellow(`↑ Update available: v${v.current} → v${v.latest}`) + dim(`  ·  ${how}`));
     }
-    const routing = this.am.routingMode === 'preferred'
-      ? `Manual preference: ${this.am.preferredAccountName} (automatic failover)`
-      : 'Automatic load balancing';
+    // Routing header: name the mode + show the one-line description the operator
+    // picked when cycling. Under `preferred`, show the manual-preference line (still
+    // available via the 'p' key on the routing screen).
+    let routing;
+    if (this.am.routingMode === 'preferred') {
+      routing = `Manual preference: ${this.am.preferredAccountName} (automatic failover)`;
+    } else {
+      // Label ONLY here. The one-line description belongs on the routing screen (and in
+      // the log line when the mode changes) — inlining it here pushed the provider-volume
+      // and cross-provider fragments off the right edge at 120 columns.
+      routing = this._routingModeDef(this.am.scheduler?.routingMode || 'sticky').label;
+    }
     // Cross-provider fallback policy — only meaningful when GLM/Kimi providers are in
     // the pool (profile=all). never=strict pin (yellow), when-exhausted=default (cyan),
     // always=peer (green).
     const hasProviders = this.am.accounts.some(a => a.type === 'provider');
+    const mode = this.am.scheduler?.routingMode || 'sticky';
     let xpText = '';
+    // Only show the cross-provider fragment under sticky — under balance/prefer-*, the
+    // MODE controls routing and claudeFallback is inert. Showing it there made the
+    // header read "Balance all · Cross-provider: always" which looked like two
+    // conflicting settings when only the first one does anything.
     if (hasProviders) {
-      xpText = this._crossProviderText();
+      // Cross-provider fragment is sticky-only — under other modes it is inert and
+      // reading "always" next to "Balance all" looked like two conflicting controls.
+      if (mode === 'sticky') xpText = this._crossProviderText();
       // Overflow visibility: when GLM/Kimi actually served requests recently, surface the
-      // volume so provider traffic isn't a mystery — it's overflow that spilled here while
-      // OAuth was saturated. Reuses the SAME 15m load window as the per-row "15m Nr" column.
+      // volume so provider traffic isn't a mystery. This is DATA, not a control — shown
+      // in every mode. Reuses the SAME 15m load window as the per-row "15m Nr" column.
       const now = Date.now();
       let provReq = 0;
       for (const a of this.am.accounts) {
@@ -1732,7 +1810,7 @@ export class TUI {
       if (headerFresh) return '';
     }
     const s = q.lastProbeErrorStatus;
-    if (s === 429) return `  ${yellow('stale·rate-limited')}`;
+    if (s === 429) return `  ${yellow('stale·probe throttled')}`;
     if (s) return `  ${yellow('stale·probe ' + s)}`;
     return `  ${dim('stale')}`;
   }
@@ -1808,12 +1886,14 @@ export class TUI {
       case 'addtype':
         return ` ${bold('1')}-${bold('4')} pick a type  ${bold('Esc')} Back`;
       case 'routing': {
-        // Show the CURRENT cross-provider policy inline so pressing f visibly changes it
-        // right here at the footer (the policy also renders in the header, far from the
-        // keypress — the "f does nothing" report).
-        const g = this.am._claudeFallbackFor?.('zai') || 'never';
-        const k = this.am._claudeFallbackFor?.('kimi') || 'never';
-        return ` ${bold('a')} Automatic  ${bold('p')} Preference  ${bold('g')} GLM: ${cyan(g)} ↻  ${bold('k')} Kimi: ${cyan(k)} ↻  ${bold('Esc')} Back`;
+        const mode = this._routingModeDef(this.am.scheduler?.routingMode || 'sticky');
+        const hasProviders = this.am.accounts.some(a => a.type === 'provider');
+        // g/k controls only affect routing under sticky — hide them under other modes
+        // so the operator never cycles a knob that does nothing.
+        const provPart = (hasProviders && mode.id === 'sticky')
+          ? `  ${bold('g')} GLM: ${cyan(this.am._claudeFallbackFor?.('zai') || 'never')}  ${bold('k')} Kimi: ${cyan(this.am._claudeFallbackFor?.('kimi') || 'never')}`
+          : '';
+        return ` ${bold('f')} Routing: ${cyan(mode.label)} ↻${provPart}  ${bold('p')} Manual preference  ${bold('Esc')} Back`;
       }
       case 'select': {
         const act = this.selAction === 'prefer'