maxpool 1.5.25 → 1.5.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxpool",
-  "version": "1.5.25",
+  "version": "1.5.27",
   "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
   "type": "module",
   "main": "src/index.js",

package/src/account-manager.js

@@ -1,4 +1,4 @@
-import { refreshAccessToken, isTokenExpiringSoon, modelFamily } from './oauth.js';
+import { refreshAccessToken, isTokenExpiringSoon, modelFamily, tokenFingerprint } from './oauth.js';
 
 // Bounded re-poll hold for an account blocked ONLY by a transient, self-clearing
 // condition whose exact recovery time is unknown: (a) a weekly-critical account
@@ -2298,8 +2298,24 @@ export class AccountManager {
         account.expiresAt = newTokens.expiresAt;
         account.status = 'active';
         account.cooldownUntil = null;
-        console.log(`[Maxpool] Token refreshed for account "${account.name}"`);
-        this._onTokenRefresh?.(accountIndex, newTokens);
+        console.log(`[Maxpool] Token refreshed for account "${account.name}" (rotated ${tokenFingerprint(account._refreshedFrom)} → ${tokenFingerprint(newTokens.refreshToken)})`);
+        // Persist-before-serve: the rotated single-use refresh token must be
+        // DURABLE on disk before we return true (before this request serves on the
+        // new access token). A non-graceful kill (SIGKILL/crash/OOM/terminal-close)
+        // between here and the disk write would otherwise leave the now-CONSUMED
+        // token on disk → next boot POSTs it → invalid_grant → forced re-auth.
+        // This minimizes the loss window to the write duration (it cannot be zero —
+        // the upstream consumes the old token the instant the POST returns); the
+        // fingerprint audit trail makes the irreducible residual diagnosable.
+        // persistTokenRefresh is bulletproofed to never throw, but the refresh has
+        // ALREADY succeeded — a persist anomaly must never be re-classified as a
+        // refresh failure (which would latch refreshDead on a working account), so
+        // guard the await too.
+        try {
+          await this._onTokenRefresh?.(accountIndex, newTokens);
+        } catch (persistErr) {
+          console.error(`[Maxpool] Token persist raised unexpectedly for "${account.name}": ${persistErr?.message || persistErr}`);
+        }
         return true;
       } catch (err) {
         console.error(`[Maxpool] Token refresh failed for "${account.name}": ${err.message}`);
@@ -2319,6 +2335,11 @@ export class AccountManager {
             // dead token. Set ONLY here (a rejected refresh), never in markAuthFailed
             // (shared with provider auth failures).
             account.refreshDead = true;
+            // Monitoring: name the exact token that was rejected + how to diagnose it
+            // from the persistent event log next time this recurs. (fp= is safe from
+            // the log's secret-redactor; refresh_token= would be redacted.)
+            const rejFp = tokenFingerprint(account._refreshedFrom);
+            console.error(`[Maxpool] Token refresh REJECTED for "${account.name}" (invalid_grant) — the refresh token maxpool sent (fp=${rejFp}) was not accepted. Diagnose from the event log: an earlier "rotated → ${rejFp}" that WAS "Persisted" ⇒ upstream revocation; NO persisted line for ${rejFp} ⇒ the rotation was lost across a restart (double-spend); the SAME source fp in two "rotated" lines in one window ⇒ two writers double-spent it. Re-login via the TUI ('l' key).`);
           }
           return false;
         }

package/src/index.js

@@ -8,7 +8,7 @@ import { SleepGuard } from './sleep-guard.js';
 import { AccountManager } from './account-manager.js';
 import { createProxyServer } from './server.js';
 import { Prober } from './prober.js';
-import { loginOAuth, fetchProfile, refreshAccessToken, isTokenExpiringSoon } from './oauth.js';
+import { loginOAuth, fetchProfile, refreshAccessToken, isTokenExpiringSoon, tokenFingerprint } from './oauth.js';
 import { TUI } from './tui.js';
 import { RestartController } from './restart-controller.js';
 import { resolveAccounts } from './account-config.js';
@@ -595,46 +595,63 @@ async function serverWorkerCommand() {
   // baton hands off and the new worker boots from the now-invalidated on-disk
   // token (B1/M3). New refreshes can't start without the lease (ensureTokenFresh
   // no-ops), so every callback here is from a legitimate lease-era refresh.
-  const persistTokenRefresh = (idx, newTokens) => {
+  //
+  // BULLETPROOF CONTRACT: ensureTokenFresh now AWAITs this (persist-before-serve),
+  // so it must NEVER throw or reject. A throw here — including from the synchronous
+  // prologue (findConfigAccount / addAccount) — would land in ensureTokenFresh's
+  // refresh-FAILURE catch and latch `refreshDead` on an account whose refresh POST
+  // actually SUCCEEDED (bricking a working account — strictly worse than the window
+  // this fix closes). So the ENTIRE body is wrapped, prologue included, and every
+  // exit resolves. Returns the awaitable persist promise.
+  const persistTokenRefresh = async (idx, newTokens) => {
     const account = accountManager.accounts[idx];
     if (!account) return;
-    // Keep config.accounts in sync so TUI saveConfig doesn't clobber fresh tokens
-    const memIdx = findConfigAccount(config, account);
-    if (memIdx >= 0) {
-      config.accounts[memIdx].accessToken = newTokens.accessToken;
-      config.accounts[memIdx].refreshToken = newTokens.refreshToken;
-      config.accounts[memIdx].expiresAt = newTokens.expiresAt;
-    }
-    atomicConfigUpdate(diskConfig => {
-      // Pick up any new accounts from disk so index matching stays correct
-      // (only add, don't refresh credentials — we're about to write the authoritative tokens)
-      for (const diskAcct of diskConfig.accounts) {
-        const known = (diskAcct.accountUuid && config.accounts.some(a => a.accountUuid === diskAcct.accountUuid))
-          || config.accounts.some(a => a.name === diskAcct.name);
-        if (!known) {
-          config.accounts.push(diskAcct);
-          accountManager.addAccount(diskAcct);
-        }
+    try {
+      // Keep config.accounts in sync so TUI saveConfig doesn't clobber fresh tokens
+      const memIdx = findConfigAccount(config, account);
+      if (memIdx >= 0) {
+        config.accounts[memIdx].accessToken = newTokens.accessToken;
+        config.accounts[memIdx].refreshToken = newTokens.refreshToken;
+        config.accounts[memIdx].expiresAt = newTokens.expiresAt;
       }
-      // Match by UUID first, then by name — index may have shifted
-      const cfgIdx = findConfigAccount(diskConfig, account);
-      if (cfgIdx >= 0) {
+      let skipped = false; // guard-skip or account-not-on-disk → nothing was persisted
+      await atomicConfigUpdate(diskConfig => {
+        // Pick up any new accounts from disk so index matching stays correct
+        // (only add, don't refresh credentials — we're about to write the authoritative tokens)
+        for (const diskAcct of diskConfig.accounts) {
+          const known = (diskAcct.accountUuid && config.accounts.some(a => a.accountUuid === diskAcct.accountUuid))
+            || config.accounts.some(a => a.name === diskAcct.name);
+          if (!known) {
+            config.accounts.push(diskAcct);
+            accountManager.addAccount(diskAcct);
+          }
+        }
+        // Match by UUID first, then by name — index may have shifted
+        const cfgIdx = findConfigAccount(diskConfig, account);
+        if (cfgIdx < 0) { skipped = true; return; }
         const onDisk = diskConfig.accounts[cfgIdx];
         // Generation guard: if the on-disk refresh token already advanced past
         // the token we rotated FROM, another writer beat us — skip the write so
         // we don't revert a fresher single-use token (the brick-the-account case).
         if (onDisk.refreshToken && onDisk.refreshToken !== account._refreshedFrom &&
             onDisk.refreshToken !== newTokens.refreshToken) {
+          skipped = true;
           return;
         }
         onDisk.accessToken = newTokens.accessToken;
         onDisk.refreshToken = newTokens.refreshToken;
         onDisk.expiresAt = newTokens.expiresAt;
+      });
+      // Monitoring: the rotated token is now DURABLE on disk. Correlate this fp
+      // with a later "REJECTED sent fp=" line to tell a lost-rotation double-spend
+      // (no matching Persisted line) from an upstream revocation (fp matches).
+      if (!skipped) {
+        console.log(`[Maxpool] Persisted rotated token for "${account.name}" (fp=${tokenFingerprint(newTokens.refreshToken)})`);
       }
-    }).catch(err => {
+    } catch (err) {
       if (err?.code === 'STALE_GENERATION') return; // another writer advanced; benign
-      console.error(`[Maxpool] Failed to save refreshed token: ${err.message}`);
-    });
+      console.error(`[Maxpool] Failed to save refreshed token for "${account.name}" (fp=${tokenFingerprint(newTokens?.refreshToken)}): ${err?.message || err}`);
+    }
   };
   accountManager.onTokenRefresh(persistTokenRefresh);
 

package/src/oauth.js

@@ -10,6 +10,19 @@ const DEFAULT_TOKEN_ENDPOINT = process.env.MAXPOOL_OAUTH_TOKEN_ENDPOINT
   || 'https://platform.claude.com/v1/oauth/token';
 const DEFAULT_CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e';
 
+/**
+ * A short, NON-REVERSIBLE fingerprint of a refresh token, for audit logging of
+ * the single-use-token rotation lifecycle. NEVER log the token itself — an
+ * 8-char sha256 prefix is irreversible yet enough to correlate a rotation
+ * ("rotated → fp / Persisted fp") with the token later rejected on boot
+ * ("REJECTED sent fp"), which is what distinguishes a lost-rotation double-spend
+ * from an upstream revocation. Returns 'none' for a falsy token.
+ */
+export function tokenFingerprint(token) {
+  if (!token) return 'none';
+  return createHash('sha256').update(String(token)).digest('hex').slice(0, 8);
+}
+
 /**
  * Refresh an expired OAuth access token using the refresh token.
  * Retries on 5xx and network errors with exponential backoff.

package/src/tui.js

@@ -18,11 +18,38 @@ const yellow = s => fg(33, s);
 const red = s => fg(31, s);
 const cyan = s => fg(36, s);
 const gray = s => fg(90, s);
+const dimUnderline = s => `${ESC}2;4m${s}${RESET}`;
 
 const ANSI_RE = /\x1b\[[0-9;]*m/g;
 const strip = s => s.replace(ANSI_RE, '');
 const vw = s => strip(s).length;
 
+// ── Accounts-table columns ───────────────────────────────────
+// Fixed column widths shared by the header row (acctHeader) AND every data row, so
+// the header labels stay aligned with the columns they name. The Account/Type/
+// Status/Quota start offsets (4/17/26/40) are pure functions of these widths + the
+// 4-col row prefix, independent of the quota-bar width.
+const NAME_W = 12;         // a.name.slice(0, NAME_W).padEnd(NAME_W)
+const TYPE_W = 8;          // a.type.padEnd(TYPE_W) — fits "provider"
+const STATUS_W = 13;       // rpad(status, STATUS_W) — fits "throttled 59s"
+const ROW_PREFIX = '    '; // ' ' + sel(1) + cur(1) + ' ' — 4 cols before the name
+
+/**
+ * Aligned column header for the accounts table. Names the three columns that carry
+ * NO inline label (Account / Type / Status) plus a group label over the two quota
+ * bars (Quota). The per-bar naming (Ses/Wk for OAuth, Tok/Req for API keys) and the
+ * load fields (Now/15m/1h) stay inline per row — they're row-type-dependent and
+ * begin at a variable offset — so the header stops after the Quota group label.
+ */
+function acctHeader(W) {
+  const quota = W >= 88 ? 'Quota (used% · resets-in)' : 'Quota';
+  return ROW_PREFIX
+    + 'Account'.padEnd(NAME_W) + ' '
+    + 'Type'.padEnd(TYPE_W) + ' '
+    + 'Status'.padEnd(STATUS_W) + ' '
+    + quota;
+}
+
 function rpad(s, w) {
   const gap = w - vw(s);
   return gap > 0 ? s + ' '.repeat(gap) : s;
@@ -194,7 +221,7 @@ function emptyBar(label, w = 10) {
   return `${ESC}100m${' '.repeat(lp)}${text}${' '.repeat(rp)}${RESET}`;
 }
 
-export const __tuiTest = { formatReset, quotaLabel, bar, emptyBar, strip, loadText, countdown };
+export const __tuiTest = { formatReset, quotaLabel, bar, emptyBar, strip, loadText, countdown, acctHeader, fitLine };
 
 function timestamp() {
   return new Date().toLocaleTimeString('en-US', { hour12: false });
@@ -663,12 +690,14 @@ export class TUI {
       const previous = this.config.accounts[idx];
       entry.enabled = previous.enabled;
       this.config.accounts[idx] = entry;
-      try {
-        await this.saveConfig(this.config);
-      } catch (error) {
-        this.config.accounts[idx] = previous;
-        throw error;
-      }
+      // Update the AccountManager to the FRESH tokens BEFORE persisting. The
+      // saveConfig writer (index.js) derives the ON-DISK tokens from the
+      // AccountManager, not from config.accounts — so persisting first writes the
+      // STALE pre-re-auth token (typically the dead one we're re-authing away from)
+      // to disk. The account then works in memory but boots DEAD on the next restart
+      // (a just-re-authed token rejected as invalid_grant). A later refresh does NOT
+      // heal disk (persistTokenRefresh's generation guard skips a disk token that
+      // != _refreshedFrom), so disk must be made correct right here.
       const amAcct = this.am.accounts.find(account =>
         (entry.accountUuid && account.accountUuid === entry.accountUuid) || account.name === name
       );
@@ -684,6 +713,17 @@ export class TUI {
         amAcct.refreshDead = false;
         if (amAcct.status === 'error') amAcct.status = 'active';
       }
+      try {
+        await this.saveConfig(this.config);
+      } catch (error) {
+        // Persist failed (e.g. disk full). The browser re-auth genuinely SUCCEEDED,
+        // so KEEP the AccountManager on the fresh tokens — reverting it to the dead
+        // token would brick a working in-memory account. Roll back only the
+        // config.accounts mirror and surface the error; disk self-heals on the next
+        // saveConfig (the AM-derived writer re-writes the fresh token).
+        this.config.accounts[idx] = previous;
+        throw error;
+      }
       this._addLog(`Updated account "${name}"`);
       return { updated: true, name };
     } else {
@@ -906,10 +946,13 @@ export class TUI {
       lines.push(yellow('  No accounts configured. Press [a] to add one.'));
     } else {
       lines.push('');
-      // Column legend — the per-row numbers are otherwise cryptic. Ses/Wk are the
-      // two quota bars; Now is live concurrency; 15m/1h are recent throughput.
+      // Aligned column header (dim + underline so it reads as chrome, not data). It
+      // labels the fixed columns that have no inline label; the Ses/Wk/Tok/Req and
+      // Now/15m/1h labels stay inline per row. A short glossary below expands the
+      // abbreviations the header + inline labels can't spell out.
+      lines.push(dimUnderline(acctHeader(W)));
       if (W >= 88) {
-        lines.push(' ' + dim('Ses/Wk = 5h/7d quota (used% · resets-in) · Now = in-flight (weight) · 15m/1h = requests served (avg latency · Nf=fails)'));
+        lines.push(' ' + dim('Ses 5h · Wk 7d · Now in-flight (weight) · 15m/1h served (avg · f fails)'));
       }
       const showBoth = W >= 70;
       const bw = showBoth
@@ -979,13 +1022,13 @@ export class TUI {
     const cur = isCur ? green('►') : ' ';
 
     // Name (bold if selected)
-    const rawName = a.name.slice(0, 12).padEnd(12);
+    const rawName = a.name.slice(0, NAME_W).padEnd(NAME_W);
     const name = isSel ? bold(rawName) : rawName;
 
     // Type — pad to 8 so "provider" (8 chars) doesn't overflow a 7-wide column and
     // shift the whole provider row (incl. its quota bars) 1 char out of alignment
     // with the "oauth"/"apikey" rows.
-    const type = gray(a.type.padEnd(8));
+    const type = gray(a.type.padEnd(TYPE_W));
 
     // Status
     let status;
@@ -1030,7 +1073,7 @@ export class TUI {
       default:          status = a.status || 'ready';
     }
     // Widened from 10 to fit "throttled 59s" so the quota bars stay column-aligned.
-    status = rpad(status, 13);
+    status = rpad(status, STATUS_W);
 
     if (a.type === 'provider') {
       return this._renderProviderAcct(sel, cur, name, type, status, a, bw, showBoth);