maxpool 1.3.0 → 1.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/account-manager.js

@@ -9,6 +9,16 @@ import { refreshAccessToken, isTokenExpiringSoon } from './oauth.js';
 // an Infinity session-kill / error-fast.
 const BOUNDED_REPOLL_HOLD_MS = 60_000;
 
+// Session rebalancing (issue #1): a bound session may migrate OFF a hot account
+// onto a much-healthier one, but ONLY on a thinking-safe request (see
+// _migrationSafeForRequest). These margins force a CLEAR, flap-stable win so a
+// session can never ping-pong between two similarly-loaded accounts.
+const REBALANCE_SCORE_MARGIN = 0.5;   // candidate score must be ≤ 50% of the bound account's
+const REBALANCE_MIN_ABS_GAP = 0.5;    // …with a small absolute floor so near-zero scores don't micro-churn
+// Weekly-pressure tiers, healthiest first. A fresh (unknown) account is the best
+// migration target; migration requires the candidate be a STRICTLY healthier tier.
+const WEEKLY_TIER = { unknown: 0, normal: 0, soft: 1, reserve: 2, critical: 3, exhausted: 4 };
+
 function emptyQuota() {
   return {
     // Standard API rate limits (API key accounts)
@@ -860,6 +870,62 @@ export class AccountManager {
       || ['reserve', 'critical', 'exhausted'].includes(this._weeklyRawState(account));
   }
 
+  // ── Session rebalancing (issue #1) ────────────────────────────────────────
+  // A bound session normally sticks to its account (continuity + Anthropic signed-
+  // thinking signature validity). These let it migrate OFF a hot account onto fresh
+  // capacity, but only when it's provably safe and clearly worth it.
+
+  /** Per-request safety gate: a request is safe to migrate to a DIFFERENT account
+   *  iff its body carries NO signed thinking (replaying a signed thinking block to
+   *  another account → non-retryable "invalid signature"). Uses the PER-REQUEST
+   *  body signal only — never the session-sticky policy — and fails CLOSED on any
+   *  body we couldn't fully scan (non-JSON / parse error → bodyThinkingScanned unset). */
+  _migrationSafeForRequest(requestInfo = {}) {
+    return requestInfo.bodyThinkingScanned === true
+      && requestInfo.requiresAnthropicThinkingIntegrity !== true;
+  }
+
+  /** Flap-stable "hot": weekly reserve/critical or 5h-cap pressure — signals that
+   *  do NOT flip the instant a request migrates (unlike live in-flight, left to the
+   *  score loop). A healthy bound account is never hot, so it never migrates → no
+   *  ping-pong. */
+  _isBoundAccountHot(account) {
+    return this._isNearQuota(account);
+  }
+
+  /** Decide whether a bound session should leave its (hot) account THIS request.
+   *  All gates must hold: thinking-safe + not mid queue-admission + bound is hot +
+   *  a genuinely-healthy alternative that is BOTH much cheaper AND a strictly
+   *  healthier weekly tier (so concurrency jitter alone can never trigger a move). */
+  _shouldRebalanceBoundSession(bound, profile, excludedIndexes, requestInfo, scoringCtx) {
+    if (!this._migrationSafeForRequest(requestInfo)) return false;
+    if (requestInfo.queueTicket || requestInfo.queueAdmitted) return false;
+    if (!this._isBoundAccountHot(bound)) return false;
+
+    const boundScore = this._scoreAccount(bound, requestInfo, scoringCtx);
+    const boundTier = WEEKLY_TIER[this._weeklyRawState(bound)] ?? 0;
+
+    let bestScore = Infinity;
+    let bestTier = Infinity;
+    for (const account of this.accounts) {
+      if (account.index === bound.index) continue;
+      if (excludedIndexes.has(account.index)) continue;
+      if (!this._matchesRequest(account, profile, requestInfo)) continue;
+      // Genuinely-healthy alternatives only (normal/soft/unknown weekly).
+      if (!this._isAvailable(account, { allowWeeklyReserve: false, allowWeeklyCritical: false })) continue;
+      const score = this._scoreAccount(account, requestInfo, scoringCtx);
+      if (score < bestScore) {
+        bestScore = score;
+        bestTier = WEEKLY_TIER[this._weeklyRawState(account)] ?? 0;
+      }
+    }
+    if (!Number.isFinite(bestScore)) return false; // no healthy alternative
+
+    return bestScore <= boundScore * REBALANCE_SCORE_MARGIN
+      && (boundScore - bestScore) >= REBALANCE_MIN_ABS_GAP
+      && bestTier < boundTier;
+  }
+
   _retryInfo(account) {
     const now = Date.now();
     const q = account.quota || {};
@@ -989,7 +1055,13 @@ export class AccountManager {
       }
     }
     const bound = this._boundAccount(requestInfo.sessionKey, profile, excludedIndexes, requestInfo);
-    if (bound && !this._hasHigherPriorityAvailable(bound, profile, excludedIndexes, requestInfo)) return bound;
+    if (bound
+      && !this._hasHigherPriorityAvailable(bound, profile, excludedIndexes, requestInfo)
+      && !this._shouldRebalanceBoundSession(bound, profile, excludedIndexes, requestInfo, scoringCtx)) {
+      return bound;
+    }
+    // Else fall through to the candidate score loop, which re-homes the session
+    // onto the best healthy account via _bindSession on acquire.
 
     const weeklyPasses = hasBinding
       ? [
@@ -1106,6 +1178,16 @@ export class AccountManager {
     if (!binding.homeName || priority < binding.homePriority) {
       binding.homeName = account.name;
       binding.homePriority = priority;
+    } else if (priority === binding.homePriority && account.name !== binding.homeName) {
+      // Same-priority move. If the previous home is still AVAILABLE we left it by
+      // CHOICE (session rebalancing off a hot-but-usable account) → re-home, so
+      // _boundAccount (which prefers homeName) doesn't snap the session straight
+      // back to the hot home next request. If the old home is UNAVAILABLE this is a
+      // FAILOVER → keep homeName so the session returns to it once it recovers.
+      const oldHome = this.accounts.find(a => a.name === binding.homeName);
+      if (oldHome && this._isAvailable(oldHome, { allowWeeklyReserve: true })) {
+        binding.homeName = account.name;
+      }
     }
     binding.currentName = account.name;
     this.sessionBindings.set(sessionKey, binding);

package/src/server.js

@@ -930,7 +930,7 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
   return `All ${n} accounts exhausted. Retry in ${retryAfter}s.`;
 }
 
-export const __serverTest = { unavailableMessage, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat };
+export const __serverTest = { unavailableMessage, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, describeRequest };
 
 async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
   if (!upstreamRes.body) return '';
@@ -1375,6 +1375,11 @@ function describeRequest(req, body) {
     if (requiresAnthropicThinkingIntegrity(json)) {
       info.requiresAnthropicThinkingIntegrity = true;
     }
+    // We fully scanned this body for signed-thinking content. Only a successfully
+    // scanned, thinking-free body is safe to migrate to another account (session
+    // rebalancing); an unparsed body leaves this false → treated as NOT safe
+    // (fail-closed) so we never replay a signed thinking block to a new account.
+    info.bodyThinkingScanned = true;
   } catch {
     // Non-JSON requests are rare; body size still gives a useful load signal.
   }

package/src/tui.js

@@ -93,16 +93,37 @@ function statusColor(status) {
   return String(status);
 }
 
+/** Short live countdown to a timestamp, SINGLE-unit so it stays ≤3 chars
+ *  ("41s"/"5m"/"23h"/"2d") and never overflows the status column: seconds under a
+ *  minute (the common throttle cooldown), then minute/hour/day. '' once elapsed.
+ *  Accepts a numeric ms timestamp (live account field) or an ISO string (snapshot). */
+function countdown(ts) {
+  if (!ts) return '';
+  const target = typeof ts === 'string' ? Date.parse(ts) : ts;
+  const ms = target - Date.now();
+  if (!Number.isFinite(ms) || ms <= 0) return '';
+  if (ms < 60_000) return `${Math.ceil(ms / 1000)}s`;
+  if (ms < 3_600_000) return `${Math.ceil(ms / 60_000)}m`;
+  if (ms < 86_400_000) return `${Math.ceil(ms / 3_600_000)}h`;
+  return `${Math.ceil(ms / 86_400_000)}d`;
+}
+
 function loadText(load) {
   const cur = load?.current || {};
   const m15 = load?.last15m || {};
   const h1 = load?.last1h || {};
-  const current = `${cur.inFlight || 0}/${cur.activeWeight || 0}`;
+  // "Now" = what this account is handling right THIS moment: N in-flight requests
+  // (and their combined weight ~ payload size, the scheduler's load input). Kept
+  // distinct from the "15m"/"1h" THROUGHPUT counts that follow, which the old
+  // "Load X/Y" label collided with.
+  const inflight = cur.inFlight || 0;
+  const weight = cur.activeWeight || 0;
+  const now = weight > 0 ? `Now ${inflight} (${weight}w)` : `Now ${inflight}`;
   const recent = `${m15.requests || 0}r`;
   const recentAvg = m15.avgMs != null ? ` ${formatMs(m15.avgMs)}` : '';
   const hour = `${h1.requests || 0}r`;
   const fails = (m15.failed || 0) > 0 ? ` ${red(`${m15.failed}f`)}` : '';
-  return `Load ${current}  15m ${recent}${recentAvg}${fails}  1h ${hour}`;
+  return `${now}  15m ${recent}${recentAvg}${fails}  1h ${hour}`;
 }
 
 function weeklyPolicyText(am, account) {
@@ -161,7 +182,7 @@ function bar(ratio, w = 10, resetTs) {
   return out;
 }
 
-export const __tuiTest = { formatReset, quotaLabel, bar, strip };
+export const __tuiTest = { formatReset, quotaLabel, bar, strip, loadText, countdown };
 
 function timestamp() {
   return new Date().toLocaleTimeString('en-US', { hour12: false });
@@ -906,12 +927,19 @@ export class TUI {
       case 'waiting':   status = yellow('waiting'); break;
       case 'paused':    status = yellow('paused'); break;
       case 'disabled':  status = gray('disabled'); break;
-      case 'throttled': status = yellow('throttled'); break;
+      case 'throttled': {
+        // A transient auto-recovering cooldown — show the remaining time (from
+        // rateLimitedUntil) so it reads as "recovering in Ns", not stuck.
+        const cd = countdown(a.rateLimitedUntil);
+        status = yellow(cd ? `throttled ${cd}` : 'throttled');
+        break;
+      }
       case 'exhausted': status = red('exhausted'); break;
       case 'error':     status = red('error'); break;
       default:          status = a.status || 'ready';
     }
-    status = rpad(status, 10);
+    // Widened from 10 to fit "throttled 59s" so the quota bars stay column-aligned.
+    status = rpad(status, 13);
 
     if (a.type === 'provider') {
       return this._renderProviderAcct(sel, cur, name, type, status, a);