maxpool 1.24.90 → 1.24.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +24 -3
- package/src/index.js +61 -29
- package/src/reload-protocol.js +6 -0
- package/src/server.js +366 -29
- package/src/tui.js +212 -30
package/package.json
CHANGED
package/src/account-manager.js
CHANGED
|
@@ -1118,6 +1118,11 @@ export class AccountManager {
|
|
|
1118
1118
|
* recorded (failedRequests, lastError) but never extends the
|
|
1119
1119
|
* consecutiveFailures streak, so it cannot trip the hold-release
|
|
1120
1120
|
* gate for the account's other conversations.
|
|
1121
|
+
* outcome.linkFault — a LINK-transport fault (link_ttfb / link_idle /
|
|
1122
|
+
* pool_link_unreachable): the Mac↔cloud link gave up, not the
|
|
1123
|
+
* account (2026-11-12: cloud rows showed 10-26 "failed" while the
|
|
1124
|
+
* cloud itself failed ≈6 of ~2,100). Records a link drop load
|
|
1125
|
+
* event and returns — no failedRequests, no streak, no error text.
|
|
1121
1126
|
*/
|
|
1122
1127
|
releaseAccount(lease, outcome = {}) {
|
|
1123
1128
|
if (!lease?.account) return;
|
|
@@ -1146,6 +1151,14 @@ export class AccountManager {
|
|
|
1146
1151
|
|
|
1147
1152
|
if (outcome.neutral) return;
|
|
1148
1153
|
|
|
1154
|
+
// A link fault is not the account failing: tally it as a link drop (kept out of
|
|
1155
|
+
// avgMs too — a 120s give-up is not the account's latency) and stop here, before
|
|
1156
|
+
// every failure counter / error field.
|
|
1157
|
+
if (outcome.linkFault) {
|
|
1158
|
+
this._recordLoadEvent(account, lease, { success: false, linkFault: true });
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1149
1162
|
// A per-model weekly cap is NOT an account failure — the account is healthy for
|
|
1150
1163
|
// every other model. Don't poison its failure counters / scoring penalty (mirror
|
|
1151
1164
|
// the network-blip carve-out); the scoped bench is already set in markRateLimited.
|
|
@@ -1198,6 +1211,7 @@ export class AccountManager {
|
|
|
1198
1211
|
durationMs: Math.max(0, now - lease.startedAt),
|
|
1199
1212
|
weight: Math.max(1, lease.weight || 1),
|
|
1200
1213
|
success: Boolean(outcome.success),
|
|
1214
|
+
linkFault: Boolean(outcome.linkFault),
|
|
1201
1215
|
status: outcome.status || null,
|
|
1202
1216
|
});
|
|
1203
1217
|
this._pruneLoadEvents(account, now);
|
|
@@ -1216,14 +1230,21 @@ export class AccountManager {
|
|
|
1216
1230
|
const since = now - windowMs;
|
|
1217
1231
|
const events = (account.loadEvents || []).filter(e => e.at >= since);
|
|
1218
1232
|
const requests = events.length;
|
|
1219
|
-
|
|
1233
|
+
// "failed" is what the operator reads as the ACCOUNT misbehaving; a link fault is
|
|
1234
|
+
// the pool's own tunnel to the team pool and gets its own field (2026-11-12).
|
|
1235
|
+
const failed = events.filter(e => !e.success && !e.linkFault).length;
|
|
1236
|
+
const linkFailed = events.filter(e => !e.success && e.linkFault).length;
|
|
1237
|
+
// avgMs excludes link faults (N1, pre-mortem r2): a 120s give-up on the link is not
|
|
1238
|
+
// this account's response time.
|
|
1239
|
+
const timed = events.filter(e => !e.linkFault);
|
|
1220
1240
|
const weight = events.reduce((sum, e) => sum + (e.weight || 1), 0);
|
|
1221
|
-
const durationMs =
|
|
1241
|
+
const durationMs = timed.reduce((sum, e) => sum + (e.durationMs || 0), 0);
|
|
1222
1242
|
return {
|
|
1223
1243
|
requests,
|
|
1224
1244
|
failed,
|
|
1245
|
+
linkFailed,
|
|
1225
1246
|
weight,
|
|
1226
|
-
avgMs:
|
|
1247
|
+
avgMs: timed.length ? Math.round(durationMs / timed.length) : null,
|
|
1227
1248
|
};
|
|
1228
1249
|
}
|
|
1229
1250
|
|
package/src/index.js
CHANGED
|
@@ -42,7 +42,7 @@ net.setDefaultAutoSelectFamilyAttemptTimeout(
|
|
|
42
42
|
import { Prober } from './prober.js';
|
|
43
43
|
import { CapacityLedger } from './capacity-ledger.js';
|
|
44
44
|
import { loginOAuth, fetchProfile, refreshAccessToken, isTokenExpiringSoon, tokenFingerprint, isLoginCancelled } from './oauth.js';
|
|
45
|
-
import { TUI } from './tui.js';
|
|
45
|
+
import { TUI, writeTerminalSync, reclaimRawModeLogged } from './tui.js';
|
|
46
46
|
import { RestartController } from './restart-controller.js';
|
|
47
47
|
import { resolveAccounts } from './account-config.js';
|
|
48
48
|
import { maybeCheckForUpdate, getCurrentVersion, markApplied, clearQuarantine, captureBootVersion } from './updater.js';
|
|
@@ -51,7 +51,7 @@ import {
|
|
|
51
51
|
RELOAD_SWAPPED, RELOAD_ROLLED_BACK,
|
|
52
52
|
MSG_LISTEN, MSG_RELEASE, MSG_TAKEOVER, MSG_PROBE_READY,
|
|
53
53
|
MSG_RELOAD_REQUEST, MSG_READY, MSG_FAILED, MSG_RELEASED, MSG_PRIMARY, MSG_ROLLED_BACK,
|
|
54
|
-
MSG_TTY_REASSERT, MSG_CONN,
|
|
54
|
+
MSG_TTY_REASSERT, MSG_CONN, describeOldWorkerExit,
|
|
55
55
|
} from './reload-protocol.js';
|
|
56
56
|
|
|
57
57
|
const args = process.argv.slice(2);
|
|
@@ -314,7 +314,7 @@ async function supervisorCommand() {
|
|
|
314
314
|
// (exit alt-screen + show cursor + raw off) so the user's shell is clean.
|
|
315
315
|
const restoreTerminalFromSupervisor = () => {
|
|
316
316
|
try {
|
|
317
|
-
if (process.stdout.isTTY)
|
|
317
|
+
if (process.stdout.isTTY) writeTerminalSync('\x1b[?25h\x1b[?1049l');
|
|
318
318
|
if (process.stdin.isTTY) { try { process.stdin.setRawMode(false); } catch { /* ignore */ } }
|
|
319
319
|
} catch { /* never throw */ }
|
|
320
320
|
};
|
|
@@ -521,18 +521,25 @@ function reapOldWorker(worker, onExited = () => {}) {
|
|
|
521
521
|
// SIGKILL only ever reaps a genuinely wedged worker, never cuts a live stream.
|
|
522
522
|
const cap = RELOAD_DRAIN_MS + 30_000;
|
|
523
523
|
let reaped = false;
|
|
524
|
+
let killed = false;
|
|
525
|
+
const swappedAt = Date.now();
|
|
524
526
|
// `onExited` fires on BOTH paths (clean exit and SIGKILL): the terminal is clobbered
|
|
525
527
|
// by the exit itself, however that exit came about.
|
|
526
528
|
const finish = () => {
|
|
527
529
|
if (reaped) return;
|
|
528
530
|
reaped = true;
|
|
529
531
|
clearTimeout(timer);
|
|
532
|
+
// B11: the update-window record — when the old worker actually exited (its exit resets the shared terminal).
|
|
533
|
+
// Event log ONLY (F6): the supervisor shares the terminal with the new worker's dashboard and is not
|
|
534
|
+
// stdout-muted, so a console line here would paint over the dashboard in the very window it measures.
|
|
535
|
+
try { appendEventLog(`[Maxpool] ${describeOldWorkerExit({ swappedAt, exitedAt: Date.now(), killed })}`); } catch { /* never break reaping */ }
|
|
530
536
|
try { onExited(); } catch { /* never let the callback break reaping */ }
|
|
531
537
|
};
|
|
532
538
|
const timer = setTimeout(() => {
|
|
533
539
|
if (reaped) return;
|
|
534
540
|
console.error(`[Maxpool] Old worker outlived ${Math.ceil(cap / 1000)}s reload-drain cap; SIGKILL.`);
|
|
535
541
|
try { worker.child.kill('SIGKILL'); } catch { /* ignore */ }
|
|
542
|
+
killed = true;
|
|
536
543
|
finish();
|
|
537
544
|
}, cap);
|
|
538
545
|
timer.unref?.();
|
|
@@ -829,7 +836,10 @@ async function serverWorkerCommand() {
|
|
|
829
836
|
// by spawned integration tests so a startup probe can't race their refresh/rotation
|
|
830
837
|
// assertions (mirrors MAXPOOL_DISABLE_SLEEP_GUARD).
|
|
831
838
|
const probeSeconds = process.env.MAXPOOL_DISABLE_QUOTA_PROBE === '1' ? 0 : (config.quotaProbeSeconds || 0);
|
|
832
|
-
|
|
839
|
+
// Late-bound: routes probe lines through the TUI once it owns the console (a TUI
|
|
840
|
+
// render must be the only writer to the terminal, 2026-11-12).
|
|
841
|
+
const uiLog = (...a) => console.log(...a);
|
|
842
|
+
const prober = new Prober(accountManager, { intervalMs: probeSeconds * 1000, log: uiLog });
|
|
833
843
|
// Tell the AM the probe cadence so the TUI can flag a scoped/provider tag whose
|
|
834
844
|
// background probe has gone stale (> 3× interval since last success).
|
|
835
845
|
accountManager.quotaProbeIntervalMs = probeSeconds * 1000;
|
|
@@ -979,26 +989,39 @@ async function serverWorkerCommand() {
|
|
|
979
989
|
// Best-effort terminal restore on ANY abnormal exit path (uncaughtException,
|
|
980
990
|
// a bare process.exit, a crash) so the user's shell is never left in raw mode
|
|
981
991
|
// or the alt-screen. Idempotent and safe even when no TUI was running.
|
|
982
|
-
|
|
992
|
+
// `exiting`: only the process-exit path may turn blocking writes back on (r2 B1,
|
|
993
|
+
// 2026-11-12) — a TUI-owned stdout is NON-BLOCKING and its queued bytes are dropped
|
|
994
|
+
// at exit, so there the exit sequence goes out synchronously or the shell stays in
|
|
995
|
+
// the dashboard. A process that keeps running (a crash that does not exit) must
|
|
996
|
+
// stay non-blocking, or a terminal that stops reading freezes it again (red-team S-1).
|
|
997
|
+
const restoreTerminal = ({ exiting }) => {
|
|
983
998
|
// Handed the TTY to a new worker (seamless TUI reload) → the new worker owns
|
|
984
999
|
// it now; restoring here would clobber its live TUI. No-op.
|
|
985
1000
|
if (terminalHandedOff) return;
|
|
986
1001
|
try {
|
|
987
|
-
if (tui?.running) { tui.stop(); return; }
|
|
988
|
-
if (process.stdout.isTTY)
|
|
1002
|
+
if (tui?.running) { tui.stop({ exiting }); return; }
|
|
1003
|
+
if (process.stdout.isTTY) {
|
|
1004
|
+
if (exiting) writeTerminalSync('\x1b[?25h\x1b[?1049l');
|
|
1005
|
+
else process.stdout.write('\x1b[?25h\x1b[?1049l');
|
|
1006
|
+
}
|
|
989
1007
|
if (process.stdin.isTTY) { try { process.stdin.setRawMode(false); } catch { /* ignore */ } }
|
|
990
1008
|
} catch { /* never throw from a restore */ }
|
|
991
1009
|
};
|
|
992
|
-
process.on('exit', restoreTerminal);
|
|
1010
|
+
process.on('exit', () => restoreTerminal({ exiting: true }));
|
|
1011
|
+
// Test seam: SIGUSR2 throws from a timer, so a pty test can crash a worker for real.
|
|
1012
|
+
if (process.env.MAXPOOL_TEST_CRASH_SIGNAL === '1') {
|
|
1013
|
+
process.on('SIGUSR2', () => setImmediate(() => { throw new Error('maxpool test crash (MAXPOOL_TEST_CRASH_SIGNAL)'); }));
|
|
1014
|
+
}
|
|
993
1015
|
process.on('uncaughtException', err => {
|
|
994
1016
|
console.error(`[Maxpool] Worker uncaughtException: ${err?.stack || err}`);
|
|
995
|
-
restoreTerminal();
|
|
996
1017
|
// A reload worker must NEVER exit(1) (escapes the supervisor exit-75 loop).
|
|
997
1018
|
// Stay alive so the supervisor's baton timeouts roll us back cleanly. Also
|
|
998
1019
|
// skip the exit-75 restart while DRAINING: a shutdown-time write to a dead
|
|
999
1020
|
// terminal can throw here, and restarting then would respawn the very orphan
|
|
1000
1021
|
// a terminal-close shutdown is trying to avoid — let the drain finish instead.
|
|
1022
|
+
// Exiting → the 'exit' listener restores the terminal (synchronously, once).
|
|
1001
1023
|
if (!isReloadWorker && !draining) process.exit(SERVER_RESTART_EXIT_CODE);
|
|
1024
|
+
restoreTerminal({ exiting: false });
|
|
1002
1025
|
});
|
|
1003
1026
|
// Swallow EPIPE/EIO from writes to a hung-up terminal so they can't crash us.
|
|
1004
1027
|
process.stdout.on('error', () => {});
|
|
@@ -1077,7 +1100,7 @@ async function serverWorkerCommand() {
|
|
|
1077
1100
|
draining = true;
|
|
1078
1101
|
if (syncTimer) clearInterval(syncTimer);
|
|
1079
1102
|
if (updateTimer) clearInterval(updateTimer);
|
|
1080
|
-
if (tui?.running) { tui.stop(); }
|
|
1103
|
+
if (tui?.running) { tui.stop(); } // plain-log mode resumes for the drain messages
|
|
1081
1104
|
console.log('\n[Maxpool] Restarting server now; queued requests will reconnect automatically.');
|
|
1082
1105
|
server.closeAllConnections?.();
|
|
1083
1106
|
// Best-effort: flush quota + settle in-flight refreshes/prober before exit so
|
|
@@ -1162,6 +1185,7 @@ async function serverWorkerCommand() {
|
|
|
1162
1185
|
pauseAdmission: () => accountManager.setAdmissionPaused(true),
|
|
1163
1186
|
resumeAdmission: () => accountManager.setAdmissionPaused(false),
|
|
1164
1187
|
restartNow: requestReload,
|
|
1188
|
+
log: uiLog, // via the TUI's console, not straight to the terminal (T6)
|
|
1165
1189
|
// The pre-drain only earns its cost on the COLD path (the socket closes, so an
|
|
1166
1190
|
// in-flight request would be severed). Seamless keeps serving them post-baton.
|
|
1167
1191
|
isSeamless: () => {
|
|
@@ -1502,6 +1526,10 @@ async function serverWorkerCommand() {
|
|
|
1502
1526
|
const releaseBatonAndDrain = async () => {
|
|
1503
1527
|
if (draining) { try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ } return; }
|
|
1504
1528
|
draining = true;
|
|
1529
|
+
// Terminal hand-off, part 1 (red-team S-2): stop drawing and taking keys NOW and
|
|
1530
|
+
// let a slow terminal catch up WHILE steps 1-4 run — they can take many seconds,
|
|
1531
|
+
// and the supervisor waits only so long for MSG_RELEASED.
|
|
1532
|
+
const terminalCatchUp = tui?.running ? tui.beginHandOff({ timeoutMs: 1500 }) : null;
|
|
1505
1533
|
// MSG_RELEASE arrived → the new worker booted OK and this reload is really
|
|
1506
1534
|
// proceeding (not a rollback). Disarm the rollback self-heal.
|
|
1507
1535
|
if (reloadWatchdog) { clearTimeout(reloadWatchdog); reloadWatchdog = null; }
|
|
@@ -1533,14 +1561,27 @@ async function serverWorkerCommand() {
|
|
|
1533
1561
|
await flushConfigWrites();
|
|
1534
1562
|
await flushStateWrites();
|
|
1535
1563
|
|
|
1536
|
-
|
|
1564
|
+
// Muzzle our own remaining logs (in-flight requests completing) BEFORE the
|
|
1565
|
+
// hand-off's stop() gives console.log back to plain stdout, so none of them
|
|
1566
|
+
// print on the user's screen or paint the new worker's alt-screen.
|
|
1567
|
+
setConsoleStdoutSuppressed(true);
|
|
1568
|
+
// Terminal hand-off, part 2 (r2 B2, 2026-11-12): finish the catch-up started at
|
|
1569
|
+
// the top, then the exit sequence, each bounded (1.5s) — BEFORE releasing: the
|
|
1570
|
+
// new worker paints right after MSG_RELEASED, and its first frame must not
|
|
1571
|
+
// interleave with ours. Never turns blocking back on; keeps serving in-flight.
|
|
1572
|
+
if (terminalCatchUp) {
|
|
1573
|
+
await terminalCatchUp;
|
|
1574
|
+
appendEventLog(`[Maxpool] Reload: handing the terminal to the new worker (${process.stdout.writableLength || 0} bytes not yet shown)`);
|
|
1575
|
+
// keepRaw: the incoming worker puts the terminal in raw mode itself, and
|
|
1576
|
+
// switching it here would freeze this worker — still streaming in-flight
|
|
1577
|
+
// responses — for as long as the terminal is behind (TCSADRAIN; see TUI.stop).
|
|
1578
|
+
await tui.handOff({ timeoutMs: 1500, flushing: terminalCatchUp, keepRaw: true });
|
|
1579
|
+
}
|
|
1537
1580
|
// The terminal is now HANDED OFF: the incoming worker will take the TTY at
|
|
1538
1581
|
// MSG_TAKEOVER. Arm the guard UNCONDITIONALLY (even if the TUI was already
|
|
1539
1582
|
// stopped) so this worker's later exit(0) restoreTerminal can't clobber the
|
|
1540
|
-
// new worker's TUI.
|
|
1541
|
-
// requests completing) so they don't paint the new worker's alt-screen.
|
|
1583
|
+
// new worker's TUI.
|
|
1542
1584
|
terminalHandedOff = true;
|
|
1543
|
-
setConsoleStdoutSuppressed(true);
|
|
1544
1585
|
try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ }
|
|
1545
1586
|
|
|
1546
1587
|
// (macOS) releaseLease() above dropped the wake-lock, but we still stream our
|
|
@@ -1612,25 +1653,15 @@ async function serverWorkerCommand() {
|
|
|
1612
1653
|
await becomePrimary({ viaTakeover: true });
|
|
1613
1654
|
} else if (msg?.type === MSG_TTY_REASSERT) {
|
|
1614
1655
|
// The old worker has now EXITED, and its exit reset the shared terminal out
|
|
1615
|
-
// of raw mode (libuv's uv_tty_reset_mode on process exit). Reclaim it
|
|
1616
|
-
//
|
|
1617
|
-
|
|
1618
|
-
// when it believes isRaw is already true — and it does believe that, because
|
|
1619
|
-
// the clobber happened in ANOTHER process and left our flag untouched. A bare
|
|
1620
|
-
// setRawMode(true) here is a silent no-op (verified on a real pty 2026-08-07).
|
|
1621
|
-
if (process.stdin.isTTY) {
|
|
1622
|
-
try {
|
|
1623
|
-
process.stdin.setRawMode(false);
|
|
1624
|
-
process.stdin.setRawMode(true);
|
|
1625
|
-
process.stdin.resume();
|
|
1626
|
-
} catch { /* terminal gone — nothing to reclaim */ }
|
|
1627
|
-
}
|
|
1656
|
+
// of raw mode (libuv's uv_tty_reset_mode on process exit). Reclaim it —
|
|
1657
|
+
// without blocking on a terminal that is behind (reclaimRawMode).
|
|
1658
|
+
await reclaimRawModeLogged({ flush: tui?._nb ? o => tui.flushOutput(o) : null });
|
|
1628
1659
|
// Re-assert the screen state too: the dying worker's alt-screen exit and
|
|
1629
1660
|
// cursor-show can land after ours.
|
|
1630
1661
|
if (tui?.running && process.stdout.isTTY) {
|
|
1631
1662
|
try {
|
|
1632
1663
|
process.stdout.write('\x1b[?1049h\x1b[?25l');
|
|
1633
|
-
tui.
|
|
1664
|
+
tui.forceRender();
|
|
1634
1665
|
} catch { /* ignore */ }
|
|
1635
1666
|
}
|
|
1636
1667
|
} else if (msg?.type === MSG_ROLLED_BACK) {
|
|
@@ -2550,5 +2581,6 @@ function formatDurationMs(ms) {
|
|
|
2550
2581
|
function formatLoadWindow(load = {}) {
|
|
2551
2582
|
const avg = load.avgMs != null ? ` avg ${formatDurationMs(load.avgMs)}` : '';
|
|
2552
2583
|
const failed = load.failed ? `, ${load.failed} failed` : '';
|
|
2553
|
-
|
|
2584
|
+
const linkFailed = load.linkFailed ? `, ${load.linkFailed} link drop${load.linkFailed === 1 ? '' : 's'}` : '';
|
|
2585
|
+
return `${load.requests || 0} req${avg}${failed}${linkFailed}`;
|
|
2554
2586
|
}
|
package/src/reload-protocol.js
CHANGED
|
@@ -135,3 +135,9 @@ export async function runReloadBaton({
|
|
|
135
135
|
// in-process can detect it; the TUI keeps rendering while the kernel line-buffers, so
|
|
136
136
|
// every keystroke (r / q / p) is swallowed. Measured on a real pty 2026-08-07.
|
|
137
137
|
export const MSG_TTY_REASSERT = 'tty-reassert';
|
|
138
|
+
|
|
139
|
+
/** One log line for the update-window record (release B, B11): when the old worker exited and how. */
|
|
140
|
+
export function describeOldWorkerExit({ swappedAt, exitedAt, killed = false }) {
|
|
141
|
+
const after = ((exitedAt - swappedAt) / 1000).toFixed(1);
|
|
142
|
+
return `reload: old worker exited at ${new Date(exitedAt).toISOString()} (${after}s after swap, ${killed ? 'SIGKILL after the drain cap' : 'clean exit'})`;
|
|
143
|
+
}
|