maxpool 1.5.28 → 1.5.29

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.5.28",
+  "version": "1.5.29",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/index.js

@@ -23,6 +23,10 @@ import {
 const args = process.argv.slice(2);
 const command = args[0];
 const SERVER_RESTART_EXIT_CODE = 75;
+// If a seamless reload doesn't release us (→ take over) within this window, the new
+// worker rolled back — self-heal admission so we don't 503 forever. > the baton
+// readiness timeout (10s) + margin.
+const RELOAD_ROLLBACK_SELFHEAL_MS = Math.max(15_000, Number(process.env.MAXPOOL_RELOAD_SELFHEAL_MS) || 30_000);
 const SERVER_WORKER_ENV = 'MAXPOOL_SERVER_WORKER';
 // Set by the supervisor when it spawns a worker for a seamless reload: that
 // worker boots HEADLESS (plain logs, no writer lease) and waits for the baton.
@@ -683,6 +687,9 @@ async function serverWorkerCommand() {
   // so it can't flip the shared TTY's raw-mode/alt-screen back and clobber the new
   // worker's live TUI. Only the current terminal owner (or the supervisor) restores.
   let terminalHandedOff = false;
+  // Self-heal timer for a seamless reload that rolls back (new worker fails to boot,
+  // we're never released). Cleared on MSG_RELEASE; fires cancelRestart otherwise.
+  let reloadWatchdog = null;
 
   // Best-effort terminal restore on ANY abnormal exit path (uncaughtException,
   // a bare process.exit, a crash) so the user's shell is never left in raw mode
@@ -813,6 +820,20 @@ async function serverWorkerCommand() {
     if (!forceCold && reloadStrategy({ supervised }) === 'seamless') {
       try {
         process.send({ type: MSG_RELOAD_REQUEST });
+        // Arm the rollback self-heal: restartController already latched
+        // pending/restarting + paused admission (in _restart, before we got here).
+        // On a SUCCESSFUL reload the supervisor sends MSG_RELEASE (→ releaseBaton­
+        // AndDrain clears this) then we exit. On a ROLLBACK we're never released and
+        // never exit — so if this fires, resume serving instead of 503-ing forever.
+        if (reloadWatchdog) clearTimeout(reloadWatchdog);
+        reloadWatchdog = setTimeout(() => {
+          reloadWatchdog = null;
+          if (draining) return; // MSG_RELEASE already arrived → a real reload, not a rollback
+          if (restartController?.cancelRestart()) {
+            console.log('[Maxpool] Reload rolled back (new worker never took over) — resumed serving.');
+          }
+        }, RELOAD_ROLLBACK_SELFHEAL_MS);
+        reloadWatchdog.unref?.();
         return;
       } catch { /* IPC gone — fall through to abrupt restart */ }
     }
@@ -821,6 +842,7 @@ async function serverWorkerCommand() {
 
   restartController = new RestartController({
     pauseAdmission: () => accountManager.setAdmissionPaused(true),
+    resumeAdmission: () => accountManager.setAdmissionPaused(false),
     restartNow: requestReload,
     // Configurable so ops can tune the bounded pre-restart drain, and so the
     // integration test can exercise the force-restart path without a 10s wait.
@@ -1044,6 +1066,9 @@ async function serverWorkerCommand() {
     const releaseBatonAndDrain = async () => {
       if (draining) { try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ } return; }
       draining = true;
+      // MSG_RELEASE arrived → the new worker booted OK and this reload is really
+      // proceeding (not a rollback). Disarm the rollback self-heal.
+      if (reloadWatchdog) { clearTimeout(reloadWatchdog); reloadWatchdog = null; }
       if (syncTimer) clearInterval(syncTimer);
       // Stop accepting NEW connections; KEEP in-flight requests alive.
       server.maxpoolBeginDrain?.();

package/src/restart-controller.js

@@ -3,6 +3,7 @@ const NON_UPSTREAM_ROUTES = new Set(['(queued)', '(none available)']);
 export class RestartController {
   constructor({
     pauseAdmission,
+    resumeAdmission = () => {},
     restartNow,
     log = console.log,
     // Bounded drain: wait at most this long for in-flight upstream requests to
@@ -15,6 +16,7 @@ export class RestartController {
     clearTimeoutFn = clearTimeout,
   }) {
     this.pauseAdmission = pauseAdmission;
+    this.resumeAdmission = resumeAdmission;
     this.restartNow = restartNow;
     this.log = log;
     this.drainTimeoutMs = drainTimeoutMs;
@@ -33,6 +35,23 @@ export class RestartController {
     return true;
   }
 
+  /**
+   * Abort a restart that was requested but never completed because the process is
+   * STILL ALIVE — a seamless reload that rolled back (the new worker failed to boot,
+   * so the old worker was never released and never exited). Without this the latched
+   * `pending`/`restarting` flags make requestStarted() return false forever → the
+   * worker 503s EVERY request until a manual restart. Reset the flags + resume
+   * admission so it keeps serving. No-op if no restart is in progress.
+   */
+  cancelRestart() {
+    if (!this.pending && !this.restarting) return false;
+    this.pending = false;
+    this.restarting = false;
+    if (this._drainTimer) { this.clearTimeoutFn(this._drainTimer); this._drainTimer = null; }
+    this.resumeAdmission();
+    return true;
+  }
+
   requestRouted(id, account) {
     if (!this.activeRequests.has(id)) return false;
     if (NON_UPSTREAM_ROUTES.has(account)) this.upstreamRequests.delete(id);

package/src/server.js

@@ -25,6 +25,11 @@ const UPSTREAM_TTFB_MS = Math.max(5_000, Number(process.env.MAXPOOL_TTFB_MS) ||
 const STREAM_IDLE_MS = Math.max(30_000, Number(process.env.MAXPOOL_STREAM_IDLE_MS) || 300_000);  // max gap BETWEEN streamed chunks (reset per chunk)
 const UPSTREAM_BODY_MS = Math.max(30_000, Number(process.env.MAXPOOL_BODY_MS) || 300_000);       // non-streaming body read
 const CLIENT_DRAIN_MS = Math.max(5_000, Number(process.env.MAXPOOL_DRAIN_MS) || 60_000);         // max wait for a backpressured client to drain (half-open client → free the lease)
+// Backstop reaper idle ceiling. Floored WELL above the longest a legit NON-streaming
+// request can go silent (no heartbeat there): TTFB 120s + nonStreamMaxWaitMs 300s +
+// UPSTREAM_BODY_MS 300s ≈ 720s — so a slow-but-alive non-streaming request is never
+// reaped at completion. Streaming holds heartbeat, so they're safe at any ceiling.
+const REQUEST_IDLE_MAX_MS = Math.max(900_000, Number(process.env.MAXPOOL_REQUEST_IDLE_MAX_MS) || 1_200_000);
 
 const DEFAULT_QUEUE = {
   enabled: true,
@@ -177,6 +182,9 @@ export function createProxyServer(accountManager, config, hooks = {}) {
       // undefined (a body-read throw lands before describeRequest runs).
       const ctx = { account: null, status: null };
       let requestInfo = {};
+      // Backstop reaper (defense-in-depth for any UNKNOWN future hang the specific
+      // TTFB/idle/body/drain guards don't cover). See startIdleRequestReaper.
+      const reaperTimer = startIdleRequestReaper(res, reqId, REQUEST_IDLE_MAX_MS);
       try {
         // Buffer request body (needed for retry on 429)
         const bodyChunks = [];
@@ -218,6 +226,7 @@ export function createProxyServer(accountManager, config, hooks = {}) {
           });
         }
       } finally {
+        clearInterval(reaperTimer);
         hooks.onRequestEnd?.(reqId, {
           method: req.method, path: req.url,
           account: ctx.account, status: ctx.status,
@@ -1109,7 +1118,7 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
   return `All ${n} accounts exhausted. Retry in ${retryAfter}s.`;
 }
 
-export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, describeRequest, classifyRateLimit, detectTranscriptOrigin, isAnthropicIncompatBody, streamResponse };
+export const __serverTest = { unavailableMessage, computeQueueWindowMs, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat, describeRequest, classifyRateLimit, detectTranscriptOrigin, isAnthropicIncompatBody, streamResponse, startIdleRequestReaper };
 
 async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
   if (!upstreamRes.body) return '';
@@ -1760,6 +1769,30 @@ function mappedModel(originalModel, account) {
 /**
  * Stream an SSE response to the client, parsing usage data along the way.
  */
+/**
+ * Backstop reaper for a single request: force-abort it if it makes ZERO client-write
+ * progress for idleMs. IDLE-keyed via res.socket.bytesWritten (a legit streaming HOLD
+ * heartbeats → bytesWritten advances → never trips; only a genuinely stuck request
+ * with no writes is reaped). res.destroy() routes into the normal onClose→finally
+ * cleanup so the account lease + onRequestEnd release — never a direct releaseAccount
+ * (which has no idempotency guard). Clock/timer injectable for tests. Returns the
+ * interval handle (caller clearInterval()s it in the request's finally).
+ */
+function startIdleRequestReaper(res, reqId, idleMs, { now = Date.now, setIntervalFn = setInterval } = {}) {
+  let lastBytes = res.socket?.bytesWritten ?? 0;
+  let lastProgressAt = now();
+  const timer = setIntervalFn(() => {
+    const bytes = res.socket?.bytesWritten ?? lastBytes;
+    if (bytes !== lastBytes) { lastBytes = bytes; lastProgressAt = now(); return; }
+    if (now() - lastProgressAt >= idleMs && !res.writableEnded && !res.destroyed) {
+      console.error(`[Maxpool] Request ${reqId} — no write progress for ${Math.round(idleMs / 1000)}s (backstop reaper); force-aborting a stuck request to free its account slot`);
+      res.destroy();
+    }
+  }, Math.min(60_000, idleMs));
+  timer.unref?.();
+  return timer;
+}
+
 async function streamResponse(webStream, res, status, responseHeaders, accountIndex, accountManager, streamLog, requestInfo = {}, idleMs = STREAM_IDLE_MS) {
   const reader = webStream.getReader();
   const decoder = new TextDecoder();

package/src/tui.js

@@ -1,5 +1,5 @@
 import { createInterface } from 'node:readline';
-import { fetchProfile, loginOAuth } from './oauth.js';
+import { fetchProfile, loginOAuth, tokenFingerprint } from './oauth.js';
 import { appendEventLog } from './event-log.js';
 
 // ── ANSI helpers ─────────────────────────────────────────────
@@ -398,7 +398,12 @@ export class TUI {
       this._confirm(
         'Restart Maxpool?',
         'Pause new requests, drain active work, then start the updated server.',
-        () => { this.stop(); this.onRestart?.(); },
+        // Do NOT stop the TUI here — let the reload path own it: cold restart stops
+        // it in restartWorkerNow, a seamless reload in releaseBatonAndDrain (on
+        // MSG_RELEASE). If the reload ROLLS BACK (new worker fails to boot), neither
+        // fires, so the TUI keeps rendering instead of being stranded in plain-log
+        // mode. (The worker also self-heals admission — see index.js reload watchdog.)
+        () => { this.onRestart?.(); },
       );
     } else if (k === 'a') {
       this.mode = 'accounts';
@@ -725,6 +730,13 @@ export class TUI {
         throw error;
       }
       this._addLog(`Updated account "${name}"`);
+      // Confirm the fresh token actually reached DISK (saveConfig awaited above +
+      // the AM was updated before it, so disk now holds creds.refreshToken). This
+      // is the verifiable proof a re-auth "stuck" — its fp must match the on-disk
+      // token; a later "REJECTED sent fp=<other>" then means an upstream revocation,
+      // not a lost persist. console.log (not _addLog) so it's VISIBLE during the
+      // stopped-TUI login flow, mirroring the sibling "Persisted rotated token" line.
+      console.log(`[Maxpool] Re-authenticated "${name}" — fresh token persisted (fp=${tokenFingerprint(creds.refreshToken)})`);
       return { updated: true, name };
     } else {
       this.config.accounts.push(entry);
@@ -948,12 +960,11 @@ export class TUI {
       lines.push('');
       // Aligned column header (dim + underline so it reads as chrome, not data). It
       // labels the fixed columns that have no inline label; the Ses/Wk/Tok/Req and
-      // Now/15m/1h labels stay inline per row. A short glossary below expands the
-      // abbreviations the header + inline labels can't spell out.
+      // Now/15m/1h labels stay inline per row. It sits DIRECTLY above the rows — the
+      // abbreviation glossary is a FOOTER below the rows (a caption), never between
+      // the header and the data (a left-aligned sentence there reads as a broken,
+      // misaligned second header).
       lines.push(dimUnderline(acctHeader(W)));
-      if (W >= 88) {
-        lines.push(' ' + dim('Ses 5h · Wk 7d · Now in-flight (weight) · 15m/1h served (avg · f fails)'));
-      }
       const showBoth = W >= 70;
       const bw = showBoth
         ? Math.max(5, Math.min(20, Math.floor((W - 56) / 2)))
@@ -962,6 +973,11 @@ export class TUI {
       for (let i = 0; i < this.am.accounts.length; i++) {
         lines.push(this._renderAcct(i, bw, showBoth));
       }
+      // Glossary FOOTER (expands the abbreviations the header + inline labels can't
+      // spell out). Below the rows so it never breaks the header↔column alignment.
+      if (W >= 88) {
+        lines.push(' ' + dim('Ses 5h · Wk 7d · Now in-flight (weight) · 15m/1h served (avg · f fails)'));
+      }
     }
 
     // ── Activity header