maxpool 1.24.90 → 1.24.92

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "maxpool",
3
- "version": "1.24.90",
3
+ "version": "1.24.92",
4
4
  "description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
@@ -1118,6 +1118,11 @@ export class AccountManager {
1118
1118
  * recorded (failedRequests, lastError) but never extends the
1119
1119
  * consecutiveFailures streak, so it cannot trip the hold-release
1120
1120
  * gate for the account's other conversations.
1121
+ * outcome.linkFault — a LINK-transport fault (link_ttfb / link_idle /
1122
+ * pool_link_unreachable): the Mac↔cloud link gave up, not the
1123
+ * account (2026-11-12: cloud rows showed 10-26 "failed" while the
1124
+ * cloud itself failed ≈6 of ~2,100). Records a link drop load
1125
+ * event and returns — no failedRequests, no streak, no error text.
1121
1126
  */
1122
1127
  releaseAccount(lease, outcome = {}) {
1123
1128
  if (!lease?.account) return;
@@ -1146,6 +1151,14 @@ export class AccountManager {
1146
1151
 
1147
1152
  if (outcome.neutral) return;
1148
1153
 
1154
+ // A link fault is not the account failing: tally it as a link drop (kept out of
1155
+ // avgMs too — a 120s give-up is not the account's latency) and stop here, before
1156
+ // every failure counter / error field.
1157
+ if (outcome.linkFault) {
1158
+ this._recordLoadEvent(account, lease, { success: false, linkFault: true });
1159
+ return;
1160
+ }
1161
+
1149
1162
  // A per-model weekly cap is NOT an account failure — the account is healthy for
1150
1163
  // every other model. Don't poison its failure counters / scoring penalty (mirror
1151
1164
  // the network-blip carve-out); the scoped bench is already set in markRateLimited.
@@ -1198,6 +1211,7 @@ export class AccountManager {
1198
1211
  durationMs: Math.max(0, now - lease.startedAt),
1199
1212
  weight: Math.max(1, lease.weight || 1),
1200
1213
  success: Boolean(outcome.success),
1214
+ linkFault: Boolean(outcome.linkFault),
1201
1215
  status: outcome.status || null,
1202
1216
  });
1203
1217
  this._pruneLoadEvents(account, now);
@@ -1216,14 +1230,21 @@ export class AccountManager {
1216
1230
  const since = now - windowMs;
1217
1231
  const events = (account.loadEvents || []).filter(e => e.at >= since);
1218
1232
  const requests = events.length;
1219
- const failed = events.filter(e => !e.success).length;
1233
+ // "failed" is what the operator reads as the ACCOUNT misbehaving; a link fault is
1234
+ // the pool's own tunnel to the team pool and gets its own field (2026-11-12).
1235
+ const failed = events.filter(e => !e.success && !e.linkFault).length;
1236
+ const linkFailed = events.filter(e => !e.success && e.linkFault).length;
1237
+ // avgMs excludes link faults (N1, pre-mortem r2): a 120s give-up on the link is not
1238
+ // this account's response time.
1239
+ const timed = events.filter(e => !e.linkFault);
1220
1240
  const weight = events.reduce((sum, e) => sum + (e.weight || 1), 0);
1221
- const durationMs = events.reduce((sum, e) => sum + (e.durationMs || 0), 0);
1241
+ const durationMs = timed.reduce((sum, e) => sum + (e.durationMs || 0), 0);
1222
1242
  return {
1223
1243
  requests,
1224
1244
  failed,
1245
+ linkFailed,
1225
1246
  weight,
1226
- avgMs: requests ? Math.round(durationMs / requests) : null,
1247
+ avgMs: timed.length ? Math.round(durationMs / timed.length) : null,
1227
1248
  };
1228
1249
  }
1229
1250
 
package/src/index.js CHANGED
@@ -42,7 +42,7 @@ net.setDefaultAutoSelectFamilyAttemptTimeout(
42
42
  import { Prober } from './prober.js';
43
43
  import { CapacityLedger } from './capacity-ledger.js';
44
44
  import { loginOAuth, fetchProfile, refreshAccessToken, isTokenExpiringSoon, tokenFingerprint, isLoginCancelled } from './oauth.js';
45
- import { TUI } from './tui.js';
45
+ import { TUI, writeTerminalSync, reclaimRawModeLogged } from './tui.js';
46
46
  import { RestartController } from './restart-controller.js';
47
47
  import { resolveAccounts } from './account-config.js';
48
48
  import { maybeCheckForUpdate, getCurrentVersion, markApplied, clearQuarantine, captureBootVersion } from './updater.js';
@@ -51,7 +51,7 @@ import {
51
51
  RELOAD_SWAPPED, RELOAD_ROLLED_BACK,
52
52
  MSG_LISTEN, MSG_RELEASE, MSG_TAKEOVER, MSG_PROBE_READY,
53
53
  MSG_RELOAD_REQUEST, MSG_READY, MSG_FAILED, MSG_RELEASED, MSG_PRIMARY, MSG_ROLLED_BACK,
54
- MSG_TTY_REASSERT, MSG_CONN,
54
+ MSG_TTY_REASSERT, MSG_CONN, describeOldWorkerExit,
55
55
  } from './reload-protocol.js';
56
56
 
57
57
  const args = process.argv.slice(2);
@@ -314,7 +314,7 @@ async function supervisorCommand() {
314
314
  // (exit alt-screen + show cursor + raw off) so the user's shell is clean.
315
315
  const restoreTerminalFromSupervisor = () => {
316
316
  try {
317
- if (process.stdout.isTTY) process.stdout.write('\x1b[?25h\x1b[?1049l');
317
+ if (process.stdout.isTTY) writeTerminalSync('\x1b[?25h\x1b[?1049l');
318
318
  if (process.stdin.isTTY) { try { process.stdin.setRawMode(false); } catch { /* ignore */ } }
319
319
  } catch { /* never throw */ }
320
320
  };
@@ -521,18 +521,25 @@ function reapOldWorker(worker, onExited = () => {}) {
521
521
  // SIGKILL only ever reaps a genuinely wedged worker, never cuts a live stream.
522
522
  const cap = RELOAD_DRAIN_MS + 30_000;
523
523
  let reaped = false;
524
+ let killed = false;
525
+ const swappedAt = Date.now();
524
526
  // `onExited` fires on BOTH paths (clean exit and SIGKILL): the terminal is clobbered
525
527
  // by the exit itself, however that exit came about.
526
528
  const finish = () => {
527
529
  if (reaped) return;
528
530
  reaped = true;
529
531
  clearTimeout(timer);
532
+ // B11: the update-window record — when the old worker actually exited (its exit resets the shared terminal).
533
+ // Event log ONLY (F6): the supervisor shares the terminal with the new worker's dashboard and is not
534
+ // stdout-muted, so a console line here would paint over the dashboard in the very window it measures.
535
+ try { appendEventLog(`[Maxpool] ${describeOldWorkerExit({ swappedAt, exitedAt: Date.now(), killed })}`); } catch { /* never break reaping */ }
530
536
  try { onExited(); } catch { /* never let the callback break reaping */ }
531
537
  };
532
538
  const timer = setTimeout(() => {
533
539
  if (reaped) return;
534
540
  console.error(`[Maxpool] Old worker outlived ${Math.ceil(cap / 1000)}s reload-drain cap; SIGKILL.`);
535
541
  try { worker.child.kill('SIGKILL'); } catch { /* ignore */ }
542
+ killed = true;
536
543
  finish();
537
544
  }, cap);
538
545
  timer.unref?.();
@@ -829,7 +836,10 @@ async function serverWorkerCommand() {
829
836
  // by spawned integration tests so a startup probe can't race their refresh/rotation
830
837
  // assertions (mirrors MAXPOOL_DISABLE_SLEEP_GUARD).
831
838
  const probeSeconds = process.env.MAXPOOL_DISABLE_QUOTA_PROBE === '1' ? 0 : (config.quotaProbeSeconds || 0);
832
- const prober = new Prober(accountManager, { intervalMs: probeSeconds * 1000 });
839
+ // Late-bound: routes probe lines through the TUI once it owns the console (a TUI
840
+ // render must be the only writer to the terminal, 2026-11-12).
841
+ const uiLog = (...a) => console.log(...a);
842
+ const prober = new Prober(accountManager, { intervalMs: probeSeconds * 1000, log: uiLog });
833
843
  // Tell the AM the probe cadence so the TUI can flag a scoped/provider tag whose
834
844
  // background probe has gone stale (> 3× interval since last success).
835
845
  accountManager.quotaProbeIntervalMs = probeSeconds * 1000;
@@ -979,26 +989,39 @@ async function serverWorkerCommand() {
979
989
  // Best-effort terminal restore on ANY abnormal exit path (uncaughtException,
980
990
  // a bare process.exit, a crash) so the user's shell is never left in raw mode
981
991
  // or the alt-screen. Idempotent and safe even when no TUI was running.
982
- const restoreTerminal = () => {
992
+ // `exiting`: only the process-exit path may turn blocking writes back on (r2 B1,
993
+ // 2026-11-12) — a TUI-owned stdout is NON-BLOCKING and its queued bytes are dropped
994
+ // at exit, so there the exit sequence goes out synchronously or the shell stays in
995
+ // the dashboard. A process that keeps running (a crash that does not exit) must
996
+ // stay non-blocking, or a terminal that stops reading freezes it again (red-team S-1).
997
+ const restoreTerminal = ({ exiting }) => {
983
998
  // Handed the TTY to a new worker (seamless TUI reload) → the new worker owns
984
999
  // it now; restoring here would clobber its live TUI. No-op.
985
1000
  if (terminalHandedOff) return;
986
1001
  try {
987
- if (tui?.running) { tui.stop(); return; }
988
- if (process.stdout.isTTY) process.stdout.write('\x1b[?25h\x1b[?1049l');
1002
+ if (tui?.running) { tui.stop({ exiting }); return; }
1003
+ if (process.stdout.isTTY) {
1004
+ if (exiting) writeTerminalSync('\x1b[?25h\x1b[?1049l');
1005
+ else process.stdout.write('\x1b[?25h\x1b[?1049l');
1006
+ }
989
1007
  if (process.stdin.isTTY) { try { process.stdin.setRawMode(false); } catch { /* ignore */ } }
990
1008
  } catch { /* never throw from a restore */ }
991
1009
  };
992
- process.on('exit', restoreTerminal);
1010
+ process.on('exit', () => restoreTerminal({ exiting: true }));
1011
+ // Test seam: SIGUSR2 throws from a timer, so a pty test can crash a worker for real.
1012
+ if (process.env.MAXPOOL_TEST_CRASH_SIGNAL === '1') {
1013
+ process.on('SIGUSR2', () => setImmediate(() => { throw new Error('maxpool test crash (MAXPOOL_TEST_CRASH_SIGNAL)'); }));
1014
+ }
993
1015
  process.on('uncaughtException', err => {
994
1016
  console.error(`[Maxpool] Worker uncaughtException: ${err?.stack || err}`);
995
- restoreTerminal();
996
1017
  // A reload worker must NEVER exit(1) (escapes the supervisor exit-75 loop).
997
1018
  // Stay alive so the supervisor's baton timeouts roll us back cleanly. Also
998
1019
  // skip the exit-75 restart while DRAINING: a shutdown-time write to a dead
999
1020
  // terminal can throw here, and restarting then would respawn the very orphan
1000
1021
  // a terminal-close shutdown is trying to avoid — let the drain finish instead.
1022
+ // Exiting → the 'exit' listener restores the terminal (synchronously, once).
1001
1023
  if (!isReloadWorker && !draining) process.exit(SERVER_RESTART_EXIT_CODE);
1024
+ restoreTerminal({ exiting: false });
1002
1025
  });
1003
1026
  // Swallow EPIPE/EIO from writes to a hung-up terminal so they can't crash us.
1004
1027
  process.stdout.on('error', () => {});
@@ -1077,7 +1100,7 @@ async function serverWorkerCommand() {
1077
1100
  draining = true;
1078
1101
  if (syncTimer) clearInterval(syncTimer);
1079
1102
  if (updateTimer) clearInterval(updateTimer);
1080
- if (tui?.running) { tui.stop(); }
1103
+ if (tui?.running) { tui.stop(); } // plain-log mode resumes for the drain messages
1081
1104
  console.log('\n[Maxpool] Restarting server now; queued requests will reconnect automatically.');
1082
1105
  server.closeAllConnections?.();
1083
1106
  // Best-effort: flush quota + settle in-flight refreshes/prober before exit so
@@ -1162,6 +1185,7 @@ async function serverWorkerCommand() {
1162
1185
  pauseAdmission: () => accountManager.setAdmissionPaused(true),
1163
1186
  resumeAdmission: () => accountManager.setAdmissionPaused(false),
1164
1187
  restartNow: requestReload,
1188
+ log: uiLog, // via the TUI's console, not straight to the terminal (T6)
1165
1189
  // The pre-drain only earns its cost on the COLD path (the socket closes, so an
1166
1190
  // in-flight request would be severed). Seamless keeps serving them post-baton.
1167
1191
  isSeamless: () => {
@@ -1502,6 +1526,10 @@ async function serverWorkerCommand() {
1502
1526
  const releaseBatonAndDrain = async () => {
1503
1527
  if (draining) { try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ } return; }
1504
1528
  draining = true;
1529
+ // Terminal hand-off, part 1 (red-team S-2): stop drawing and taking keys NOW and
1530
+ // let a slow terminal catch up WHILE steps 1-4 run — they can take many seconds,
1531
+ // and the supervisor waits only so long for MSG_RELEASED.
1532
+ const terminalCatchUp = tui?.running ? tui.beginHandOff({ timeoutMs: 1500 }) : null;
1505
1533
  // MSG_RELEASE arrived → the new worker booted OK and this reload is really
1506
1534
  // proceeding (not a rollback). Disarm the rollback self-heal.
1507
1535
  if (reloadWatchdog) { clearTimeout(reloadWatchdog); reloadWatchdog = null; }
@@ -1533,14 +1561,27 @@ async function serverWorkerCommand() {
1533
1561
  await flushConfigWrites();
1534
1562
  await flushStateWrites();
1535
1563
 
1536
- if (tui?.running) tui.stop(); // restore terminal before the new worker takes it
1564
+ // Muzzle our own remaining logs (in-flight requests completing) BEFORE the
1565
+ // hand-off's stop() gives console.log back to plain stdout, so none of them
1566
+ // print on the user's screen or paint the new worker's alt-screen.
1567
+ setConsoleStdoutSuppressed(true);
1568
+ // Terminal hand-off, part 2 (r2 B2, 2026-11-12): finish the catch-up started at
1569
+ // the top, then the exit sequence, each bounded (1.5s) — BEFORE releasing: the
1570
+ // new worker paints right after MSG_RELEASED, and its first frame must not
1571
+ // interleave with ours. Never turns blocking back on; keeps serving in-flight.
1572
+ if (terminalCatchUp) {
1573
+ await terminalCatchUp;
1574
+ appendEventLog(`[Maxpool] Reload: handing the terminal to the new worker (${process.stdout.writableLength || 0} bytes not yet shown)`);
1575
+ // keepRaw: the incoming worker puts the terminal in raw mode itself, and
1576
+ // switching it here would freeze this worker — still streaming in-flight
1577
+ // responses — for as long as the terminal is behind (TCSADRAIN; see TUI.stop).
1578
+ await tui.handOff({ timeoutMs: 1500, flushing: terminalCatchUp, keepRaw: true });
1579
+ }
1537
1580
  // The terminal is now HANDED OFF: the incoming worker will take the TTY at
1538
1581
  // MSG_TAKEOVER. Arm the guard UNCONDITIONALLY (even if the TUI was already
1539
1582
  // stopped) so this worker's later exit(0) restoreTerminal can't clobber the
1540
- // new worker's TUI. Also muzzle our own remaining drain-time logs (in-flight
1541
- // requests completing) so they don't paint the new worker's alt-screen.
1583
+ // new worker's TUI.
1542
1584
  terminalHandedOff = true;
1543
- setConsoleStdoutSuppressed(true);
1544
1585
  try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ }
1545
1586
 
1546
1587
  // (macOS) releaseLease() above dropped the wake-lock, but we still stream our
@@ -1612,25 +1653,15 @@ async function serverWorkerCommand() {
1612
1653
  await becomePrimary({ viaTakeover: true });
1613
1654
  } else if (msg?.type === MSG_TTY_REASSERT) {
1614
1655
  // The old worker has now EXITED, and its exit reset the shared terminal out
1615
- // of raw mode (libuv's uv_tty_reset_mode on process exit). Reclaim it.
1616
- //
1617
- // The toggle OFF then ON is load-bearing: Node short-circuits setRawMode(true)
1618
- // when it believes isRaw is already true — and it does believe that, because
1619
- // the clobber happened in ANOTHER process and left our flag untouched. A bare
1620
- // setRawMode(true) here is a silent no-op (verified on a real pty 2026-08-07).
1621
- if (process.stdin.isTTY) {
1622
- try {
1623
- process.stdin.setRawMode(false);
1624
- process.stdin.setRawMode(true);
1625
- process.stdin.resume();
1626
- } catch { /* terminal gone — nothing to reclaim */ }
1627
- }
1656
+ // of raw mode (libuv's uv_tty_reset_mode on process exit). Reclaim it —
1657
+ // without blocking on a terminal that is behind (reclaimRawMode).
1658
+ await reclaimRawModeLogged({ flush: tui?._nb ? o => tui.flushOutput(o) : null });
1628
1659
  // Re-assert the screen state too: the dying worker's alt-screen exit and
1629
1660
  // cursor-show can land after ours.
1630
1661
  if (tui?.running && process.stdout.isTTY) {
1631
1662
  try {
1632
1663
  process.stdout.write('\x1b[?1049h\x1b[?25l');
1633
- tui.render();
1664
+ tui.forceRender();
1634
1665
  } catch { /* ignore */ }
1635
1666
  }
1636
1667
  } else if (msg?.type === MSG_ROLLED_BACK) {
@@ -2550,5 +2581,6 @@ function formatDurationMs(ms) {
2550
2581
  function formatLoadWindow(load = {}) {
2551
2582
  const avg = load.avgMs != null ? ` avg ${formatDurationMs(load.avgMs)}` : '';
2552
2583
  const failed = load.failed ? `, ${load.failed} failed` : '';
2553
- return `${load.requests || 0} req${avg}${failed}`;
2584
+ const linkFailed = load.linkFailed ? `, ${load.linkFailed} link drop${load.linkFailed === 1 ? '' : 's'}` : '';
2585
+ return `${load.requests || 0} req${avg}${failed}${linkFailed}`;
2554
2586
  }
@@ -135,3 +135,9 @@ export async function runReloadBaton({
135
135
  // in-process can detect it; the TUI keeps rendering while the kernel line-buffers, so
136
136
  // every keystroke (r / q / p) is swallowed. Measured on a real pty 2026-08-07.
137
137
  export const MSG_TTY_REASSERT = 'tty-reassert';
138
+
139
+ /** One log line for the update-window record (release B, B11): when the old worker exited and how. */
140
+ export function describeOldWorkerExit({ swappedAt, exitedAt, killed = false }) {
141
+ const after = ((exitedAt - swappedAt) / 1000).toFixed(1);
142
+ return `reload: old worker exited at ${new Date(exitedAt).toISOString()} (${after}s after swap, ${killed ? 'SIGKILL after the drain cap' : 'clean exit'})`;
143
+ }