maxpool 1.5.3 → 1.5.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.5.3",
+  "version": "1.5.5",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/config.js

@@ -28,6 +28,16 @@ export function getStatePath() {
   return cfg.endsWith('.json') ? cfg.replace(/\.json$/, '.state.json') : cfg + '.state';
 }
 
+/**
+ * Path to the persistent event log (a sibling of the config). Default-on; holds a
+ * rotating record of routing / network errors / cooldowns / reloads so incidents
+ * are investigable after the in-memory TUI feed has scrolled away.
+ */
+export function getLogPath() {
+  const cfg = getConfigPath();
+  return cfg.endsWith('.json') ? cfg.replace(/\.json$/, '.log') : cfg + '.log';
+}
+
 export function createDefaultConfig() {
   return {
     proxy: {

package/src/event-log.js

@@ -0,0 +1,122 @@
+// Persistent, rotating, low-overhead event log so incidents are investigable
+// after the fact (the TUI activity feed is in-memory only and scrolls away). It
+// mirrors the existing `[Maxpool] …` console stream — routing, network errors,
+// cooldowns, token refreshes, reloads, upstream-throttle — to a file that
+// survives the terminal. Logging must NEVER throw into the proxy path, so every
+// write is fire-and-forget and all errors are swallowed.
+
+import { appendFile, stat, rename, chmod } from 'node:fs/promises';
+
+const MAX_BYTES = 5 * 1024 * 1024;   // rotate at ~5 MB (one .1 backup kept)
+const ROTATE_CHECK_MS = 2000;        // rotation-owner size-check cadence
+// Cap each line below the platform PIPE_BUF (512 B on macOS) so concurrent
+// O_APPEND writes from coexisting processes during a reload stay atomic (never
+// interleave). 24-char ISO timestamp + space + body + newline must fit.
+const MAX_BODY = 480;
+const MAX_QUEUED = 10_000;            // bound the in-flight queue if the disk stalls
+
+let logPath = null;
+let manageRotation = false;           // SINGLE owner rotates (supervisor, or the lone unsupervised worker)
+let rotationTimer = null;
+let writeChain = Promise.resolve();
+let queued = 0;
+let dropped = 0;
+let origConsole = null;
+
+export function setEventLogPath(path, { manageRotation: owns = false } = {}) {
+  logPath = path || null;
+  manageRotation = Boolean(owns);
+  if (rotationTimer) { clearInterval(rotationTimer); rotationTimer = null; }
+  if (logPath) {
+    // 0600 even on a pre-existing file (appendFile's mode only applies on create).
+    chmod(logPath, 0o600).catch(() => {});
+    if (manageRotation) {
+      rotationTimer = setInterval(() => { rotateIfNeeded().catch(() => {}); }, ROTATE_CHECK_MS);
+      rotationTimer.unref?.();
+    }
+  }
+}
+
+/**
+ * Redact obvious secrets BEFORE anything reaches disk. We only ever log summaries
+ * (account names, statuses, error classes) — never raw tokens/headers/bodies — but
+ * this is defense-in-depth so a future log line that happens to include a token
+ * can't leak through the persisted file.
+ */
+export function redactSecrets(s) {
+  return String(s)
+    .replace(/sk-ant-[A-Za-z0-9._-]+/g, 'sk-ant-[redacted]')
+    .replace(/Bearer\s+[A-Za-z0-9._-]+/gi, 'Bearer [redacted]')
+    .replace(/(["']?(?:access|refresh)_?token["']?\s*[:=]\s*["']?)[A-Za-z0-9._-]{8,}/gi, '$1[redacted]')
+    .replace(/(["']?api[_-]?key["']?\s*[:=]\s*["']?)[A-Za-z0-9._-]{8,}/gi, '$1[redacted]');
+}
+
+export function appendEventLog(msg) {
+  if (!logPath || msg == null) return;
+  if (queued >= MAX_QUEUED) { dropped++; return; } // disk stalled — drop, don't grow memory unbounded
+  queued++;
+  writeChain = writeChain.then(() => writeLine(msg)).catch(() => {}).finally(() => { queued--; });
+}
+
+// Truncate to at most maxBytes UTF-8 bytes (not chars) without splitting a
+// multibyte codepoint, so the final line stays under PIPE_BUF for atomic O_APPEND.
+function capBytes(s, maxBytes) {
+  if (Buffer.byteLength(s) <= maxBytes) return s;
+  let str = Buffer.from(s).subarray(0, maxBytes - 3).toString('utf8'); // room for '…' (3 B)
+  if (str.endsWith('�')) str = str.slice(0, -1); // drop a partial trailing codepoint
+  return str + '…';
+}
+
+async function writeLine(rawMsg) {
+  const path = logPath;
+  if (!path) return;
+  try {
+    // The drop sentinel is its OWN append so neither write exceeds PIPE_BUF.
+    if (dropped > 0) {
+      const n = dropped; dropped = 0;
+      await appendFile(path, `${new Date().toISOString()} [event-log] dropped ${n} line(s) (disk stalled)\n`, { mode: 0o600 });
+    }
+    // Redaction + formatting on the DRAINED async chain, off the request hot path.
+    let body = redactSecrets(String(rawMsg)).replace(/\s*\n\s*/g, ' ');
+    body = capBytes(body, MAX_BODY);
+    await appendFile(path, `${new Date().toISOString()} ${body}\n`, { mode: 0o600 });
+  } catch { /* logging must never throw into the proxy */ }
+}
+
+// Rotation is run by a SINGLE owner's timer (never inside writeLine), so two
+// coexisting processes during a reload can't both rename and clobber each other's
+// archive. Appends are O_APPEND to the path, so a rename mid-stream just means the
+// next append re-creates the live file — no lost current lines.
+export async function rotateIfNeeded() {
+  const path = logPath;
+  if (!path) return;
+  const st = await stat(path).catch(() => null);
+  if (st && st.size > MAX_BYTES) await rename(path, `${path}.1`).catch(() => {});
+}
+
+/**
+ * Tee console.log/console.error to the event log while preserving their normal
+ * output (stdout in plain mode; the TUI separately re-points console at its ring
+ * buffer, whose _addLog also calls appendEventLog, so TUI-mode lines persist too).
+ */
+export function installConsoleMirror() {
+  if (origConsole) return; // idempotent
+  origConsole = { log: console.log.bind(console), error: console.error.bind(console) };
+  for (const method of ['log', 'error']) {
+    const orig = origConsole[method];
+    console[method] = (...args) => {
+      try { appendEventLog(args.map(a => (typeof a === 'string' ? a : String(a))).join(' ')); } catch { /* never */ }
+      orig(...args);
+    };
+  }
+}
+
+/** Test-only barrier: resolves once every queued append has settled. */
+export const flushEventLog = () => writeChain;
+
+/** Test-only: restore console + reset all module state so tests don't bleed. */
+export function __resetEventLogForTest() {
+  if (origConsole) { console.log = origConsole.log; console.error = origConsole.error; origConsole = null; }
+  if (rotationTimer) { clearInterval(rotationTimer); rotationTimer = null; }
+  logPath = null; manageRotation = false; writeChain = Promise.resolve(); queued = 0; dropped = 0;
+}

package/src/index.js

@@ -2,7 +2,8 @@
 
 import { spawn, spawnSync } from 'node:child_process';
 import { createInterface } from 'node:readline';
-import { loadOrCreateConfig, loadConfig, saveConfig, atomicConfigUpdate, getConfigPath, loadState, saveState, getStatePath, readGeneration, flushConfigWrites, flushStateWrites } from './config.js';
+import { loadOrCreateConfig, loadConfig, saveConfig, atomicConfigUpdate, getConfigPath, loadState, saveState, getStatePath, getLogPath, readGeneration, flushConfigWrites, flushStateWrites } from './config.js';
+import { setEventLogPath, installConsoleMirror } from './event-log.js';
 import { AccountManager } from './account-manager.js';
 import { createProxyServer } from './server.js';
 import { Prober } from './prober.js';
@@ -26,6 +27,34 @@ const SERVER_WORKER_ENV = 'MAXPOOL_SERVER_WORKER';
 // worker boots HEADLESS (plain logs, no writer lease) and waits for the baton.
 const SERVER_RELOAD_WORKER_ENV = 'MAXPOOL_RELOAD_WORKER';
 
+// Is maxpool attached to an interactive terminal? Governs SIGNAL semantics: in a
+// terminal a SIGHUP means "the window hung up → shut down" (never reload into a
+// headless orphan that outlives the terminal and squats the port); headless /
+// service mode keeps SIGHUP as the conventional reload trigger. MAXPOOL_FORCE_TTY
+// lets the (non-pty) test suite exercise the terminal-hangup path deterministically.
+function isInteractiveTerminal() {
+  return Boolean(process.stdout.isTTY) || process.env.MAXPOOL_FORCE_TTY === '1';
+}
+
+// Wire up the persistent event log (default-on; disable with `eventLog: false` in
+// config). Sets the path + tees console.log/error to disk so routing, network
+// errors, cooldowns and reloads survive the in-memory TUI feed for later triage.
+function initEventLog(config, { manageRotation = false } = {}) {
+  if (config?.eventLog === false) return;
+  try {
+    setEventLogPath(getLogPath(), { manageRotation });
+    installConsoleMirror();
+  } catch { /* logging must never block startup */ }
+}
+
+// Which reload path an in-process reload request takes. Interactive (a live TUI)
+// reloads via a FULL cold restart so the fresh worker re-renders the TUI — a
+// seamless reload worker boots headless and would strand the user in raw logs.
+// Headless/service keeps the zero-downtime baton (nothing visual to lose).
+function reloadStrategy({ supervised, useTUI }) {
+  return supervised && !useTUI ? 'seamless' : 'cold-restart';
+}
+
 switch (command) {
   case 'server':
     await serverCommand();
@@ -110,6 +139,7 @@ async function serverCommand() {
 async function supervisorCommand() {
   const { createServer } = await import('node:net');
   const config = await loadOrCreateConfig();
+  initEventLog(config, { manageRotation: true }); // supervisor is the single rotation owner
   const port = config.proxy.port;
   const host = config.proxy.host || '127.0.0.1';
 
@@ -149,9 +179,18 @@ async function supervisorCommand() {
   const forwardSignal = sig => { try { activeWorker?.child.kill(sig); } catch { /* ignore */ } };
   process.on('SIGINT', () => { /* delivered to the worker by the TTY group */ });
   process.on('SIGTERM', () => { /* delivered to the worker by the TTY group */ });
-  // SIGHUP (from `kill -HUP <supervisor-pid>`) reaches ONLY the supervisor →
-  // forward it so the worker requests a seamless reload.
-  process.on('SIGHUP', () => forwardSignal('SIGHUP'));
+  // SIGHUP: in a terminal, a window hangup delivers SIGHUP to the whole foreground
+  // group, so the worker ALSO received it and (interactive) is already shutting
+  // itself down — the supervisor must NOT forward a second signal, or the worker
+  // gets a doubled signal that force-quits it mid-drain (non-zero exit → a fast
+  // exit is then misread as a crash → respawn into a headless orphan, the exact
+  // bug). Only forward in headless/service mode, where SIGHUP is the conventional
+  // reload trigger and reaches the supervisor alone.
+  process.on('SIGHUP', () => { if (!isInteractiveTerminal()) forwardSignal('SIGHUP'); });
+  // A dead controlling terminal (window closed) makes writes to stdout/stderr emit
+  // EPIPE/EIO — swallow them so a shutdown-time log can't crash the supervisor.
+  process.stdout.on('error', () => {});
+  process.stderr.on('error', () => {});
   // A spawn failure / stray rejection must NOT kill the supervisor (it would
   // wedge the port and drop the service). Log and let the supervision loop or
   // the reload's own error handling recover.
@@ -407,6 +446,8 @@ function delay(ms) {
 
 async function serverWorkerCommand() {
   const config = await loadOrCreateConfig();
+  // The lone unsupervised worker owns rotation; under a supervisor, the supervisor does.
+  initEventLog(config, { manageRotation: typeof process.send !== 'function' });
 
   // --log-to <dir>
   const logTo = argValue('--log-to');
@@ -563,9 +604,15 @@ async function serverWorkerCommand() {
     console.error(`[Maxpool] Worker uncaughtException: ${err?.stack || err}`);
     restoreTerminal();
     // A reload worker must NEVER exit(1) (escapes the supervisor exit-75 loop).
-    // Stay alive so the supervisor's baton timeouts roll us back cleanly.
-    if (!isReloadWorker) process.exit(SERVER_RESTART_EXIT_CODE);
+    // Stay alive so the supervisor's baton timeouts roll us back cleanly. Also
+    // skip the exit-75 restart while DRAINING: a shutdown-time write to a dead
+    // terminal can throw here, and restarting then would respawn the very orphan
+    // a terminal-close shutdown is trying to avoid — let the drain finish instead.
+    if (!isReloadWorker && !draining) process.exit(SERVER_RESTART_EXIT_CODE);
   });
+  // Swallow EPIPE/EIO from writes to a hung-up terminal so they can't crash us.
+  process.stdout.on('error', () => {});
+  process.stderr.on('error', () => {});
   process.on('unhandledRejection', reason => {
     console.error(`[Maxpool] Worker unhandledRejection: ${reason}`);
   });
@@ -636,7 +683,17 @@ async function serverWorkerCommand() {
     // the respawned worker doesn't boot from a half-written state. Bounded so the
     // abrupt path stays fast even if a write hangs.
     Promise.race([
-      (async () => { await persistQuotaState(); await releaseLease(); await flushConfigWrites(); await flushStateWrites(); })(),
+      (async () => {
+        // Drain in-flight token refreshes before exit so the cold-respawned worker
+        // never boots mid-rotation of a single-use refresh token (→ invalid_grant
+        // → bricked account). Mirrors shutdownGracefully + the seamless baton's
+        // drainRefreshes — required now that interactive reload routes through here.
+        await releaseLease();
+        await accountManager.drainRefreshes();
+        await persistQuotaState(true);
+        await flushConfigWrites();
+        await flushStateWrites();
+      })(),
       delay(2000),
     ]).finally(() => process.exit(SERVER_RESTART_EXIT_CODE));
   };
@@ -646,7 +703,9 @@ async function serverWorkerCommand() {
   // (not supervised, no IPC), fall back to the abrupt restart.
   const requestReload = () => {
     if (draining) return;
-    if (supervised) {
+    // Interactive (live TUI) → full cold restart so the fresh worker re-renders the
+    // TUI; headless/service → zero-downtime seamless baton (nothing visual to lose).
+    if (reloadStrategy({ supervised, useTUI }) === 'seamless') {
       try {
         process.send({ type: MSG_RELOAD_REQUEST });
         return;
@@ -661,9 +720,15 @@ async function serverWorkerCommand() {
   });
 
   const shutdownGracefully = (reason, options = {}) => {
+    // A terminal-close shutdown must NEVER exit non-zero: the supervisor reads a
+    // fast non-zero exit as a crash and respawns a fresh worker — onto the now-dead
+    // terminal, re-creating the headless orphan. So even a drain-timeout or close
+    // error during a terminal close exits 0 (the terminal is gone; there's nothing
+    // to keep alive for).
+    const cleanExitCode = options.terminalClose ? 0 : 1;
     if (draining) {
       console.error(`\n[Maxpool] Force exiting with ${restartController.activeRequests.size} active request(s) still open.`);
-      process.exit(1);
+      process.exit(cleanExitCode);
     }
 
     draining = true;
@@ -704,14 +769,14 @@ async function serverWorkerCommand() {
 
     timeoutTimer = setTimeout(() => {
       console.error(`[Maxpool] Drain timeout after ${Math.ceil(drainTimeoutMs / 1000)}s; exiting with ${restartController.activeRequests.size} active request(s) still open.`);
-      finish(1);
+      finish(cleanExitCode);
     }, drainTimeoutMs);
     timeoutTimer.unref();
 
     server.close(err => {
       if (err) {
         console.error(`[Maxpool] Shutdown error: ${err.message}`);
-        finish(1);
+        finish(cleanExitCode);
         return;
       }
       console.log('[Maxpool] Shutdown complete.');
@@ -936,9 +1001,11 @@ async function serverWorkerCommand() {
 
   process.on('SIGINT', () => shutdownGracefully('SIGINT'));
   process.on('SIGTERM', () => shutdownGracefully('SIGTERM'));
-  // SIGHUP requests a seamless reload (the conventional "reload" signal). Under
-  // the supervisor this runs the baton; otherwise it falls back to exit-75.
-  process.on('SIGHUP', () => requestReload());
+  // SIGHUP: in a terminal this means the controlling window hung up (closed) →
+  // drain + exit cleanly so we never reload into a headless orphan that outlives
+  // the terminal and squats the port. Headless/service mode keeps SIGHUP as the
+  // conventional reload signal (baton under the supervisor, else exit-75).
+  process.on('SIGHUP', () => { if (isInteractiveTerminal()) shutdownGracefully('SIGHUP', { terminalClose: true }); else requestReload(); });
 }
 
 function logPlainServerStart({ host, port, accounts, threshold, config }) {

package/src/server.js

@@ -874,6 +874,9 @@ async function forwardRequest(
       );
       if (queued) return;
       ctx.status = 503;
+      // Record the user-facing failure explicitly so it's greppable in the event
+      // log (the in-memory TUI feed scrolls away). `err` is the last network error.
+      console.error(`[Maxpool] Returned connection_unavailable (503) after network errors on all routes (last: "${account.name}" — ${err.code || err.message})`);
       sendErrorResponse(res, requestInfo, 503, {
         type: 'error',
         error: {

package/src/tui.js

@@ -1,5 +1,6 @@
 import { createInterface } from 'node:readline';
 import { fetchProfile, loginOAuth } from './oauth.js';
+import { appendEventLog } from './event-log.js';
 
 // ── ANSI helpers ─────────────────────────────────────────────
 
@@ -284,6 +285,9 @@ export class TUI {
 
   _addLog(msg) {
     msg = msg.replace(/^\[Maxpool\]\s*/, '');
+    // Persist too — in TUI mode console is re-pointed here, bypassing the console
+    // mirror, so this is where TUI-mode lines reach the on-disk event log.
+    appendEventLog(msg);
     this.log.unshift({ t: timestamp(), msg });
     if (this.log.length > 200) this.log.length = 200;
     if (this.running) this.render();