maxpool 1.24.92 → 1.24.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +155 -4
- package/src/claude-reset-grants.js +8 -0
- package/src/event-log.js +79 -1
- package/src/index.js +325 -149
- package/src/prober.js +51 -9
- package/src/reload-protocol.js +86 -2
- package/src/reset-policy.js +94 -9
- package/src/terminal-mode.js +124 -0
- package/src/tui.js +106 -15
package/src/index.js
CHANGED
|
@@ -12,10 +12,11 @@ import net from 'node:net';
|
|
|
12
12
|
// connection is opened, hence here at module top.
|
|
13
13
|
net.setDefaultAutoSelectFamilyAttemptTimeout(Number(process.env.MAXPOOL_FAMILY_ATTEMPT_MS) || 2000);
|
|
14
14
|
import { stat } from 'node:fs/promises';
|
|
15
|
+
import { unlinkSync } from 'node:fs';
|
|
15
16
|
import { spawn, spawnSync } from 'node:child_process';
|
|
16
17
|
import { createInterface } from 'node:readline';
|
|
17
18
|
import { loadOrCreateConfig, loadConfig, saveConfig, atomicConfigUpdate, getConfigPath, loadState, saveState, getStatePath, getLogPath, readGeneration, flushConfigWrites, flushStateWrites, saveStateUnbumped } from './config.js';
|
|
18
|
-
import { setEventLogPath, installConsoleMirror, setConsoleStdoutSuppressed, appendEventLog } from './event-log.js';
|
|
19
|
+
import { setEventLogPath, installConsoleMirror, setConsoleStdoutSuppressed, setConsoleNonBlocking, reassertConsoleNonBlocking, appendEventLog, printFatal } from './event-log.js';
|
|
19
20
|
import { sendLoginToPool, planCliLogin, resolveUsablePools } from './cloud-login.js';
|
|
20
21
|
import { SleepGuard } from './sleep-guard.js';
|
|
21
22
|
import { controllingTerminalAlive } from './terminal-alive.js';
|
|
@@ -42,7 +43,8 @@ net.setDefaultAutoSelectFamilyAttemptTimeout(
|
|
|
42
43
|
import { Prober } from './prober.js';
|
|
43
44
|
import { CapacityLedger } from './capacity-ledger.js';
|
|
44
45
|
import { loginOAuth, fetchProfile, refreshAccessToken, isTokenExpiringSoon, tokenFingerprint, isLoginCancelled } from './oauth.js';
|
|
45
|
-
import { TUI, writeTerminalSync, reclaimRawModeLogged } from './tui.js';
|
|
46
|
+
import { TUI, writeTerminalSync, reclaimRawModeLogged, whenTerminalCaughtUp } from './tui.js';
|
|
47
|
+
import { createTerminalModeSwitcher } from './terminal-mode.js';
|
|
46
48
|
import { RestartController } from './restart-controller.js';
|
|
47
49
|
import { resolveAccounts } from './account-config.js';
|
|
48
50
|
import { maybeCheckForUpdate, getCurrentVersion, markApplied, clearQuarantine, captureBootVersion } from './updater.js';
|
|
@@ -51,7 +53,7 @@ import {
|
|
|
51
53
|
RELOAD_SWAPPED, RELOAD_ROLLED_BACK,
|
|
52
54
|
MSG_LISTEN, MSG_RELEASE, MSG_TAKEOVER, MSG_PROBE_READY,
|
|
53
55
|
MSG_RELOAD_REQUEST, MSG_READY, MSG_FAILED, MSG_RELEASED, MSG_PRIMARY, MSG_ROLLED_BACK,
|
|
54
|
-
MSG_TTY_REASSERT, MSG_CONN, describeOldWorkerExit,
|
|
56
|
+
MSG_TTY_REASSERT, MSG_CONN, describeOldWorkerExit, makeWorkerChannel, rollbackNoticeAction,
|
|
55
57
|
} from './reload-protocol.js';
|
|
56
58
|
|
|
57
59
|
const args = process.argv.slice(2);
|
|
@@ -87,6 +89,9 @@ const SERVER_WORKER_ENV = 'MAXPOOL_SERVER_WORKER';
|
|
|
87
89
|
// Set by the supervisor when it spawns a worker for a seamless reload: that
|
|
88
90
|
// worker boots HEADLESS (plain logs, no writer lease) and waits for the baton.
|
|
89
91
|
const SERVER_RELOAD_WORKER_ENV = 'MAXPOOL_RELOAD_WORKER';
|
|
92
|
+
// Set by the supervisor on every cold worker after its first (a restart / crash
|
|
93
|
+
// respawn): that worker switches raw mode off its serving thread (takeTerminal).
|
|
94
|
+
const SERVER_RESPAWNED_WORKER_ENV = 'MAXPOOL_RESPAWNED_WORKER';
|
|
90
95
|
|
|
91
96
|
// Is maxpool attached to an interactive terminal? Governs SIGNAL semantics: in a
|
|
92
97
|
// terminal a SIGHUP means "the window hung up → shut down" (never reload into a
|
|
@@ -278,6 +283,16 @@ async function supervisorCommand() {
|
|
|
278
283
|
return;
|
|
279
284
|
}
|
|
280
285
|
|
|
286
|
+
// The supervisor shares the terminal with the workers but owns no dashboard, so its
|
|
287
|
+
// lines went out as Node's default BLOCKING tty writes. With the terminal not reading,
|
|
288
|
+
// one reload log line froze it for 7+ minutes (2026-11-13): no rollback notice, no
|
|
289
|
+
// reap, no fallback. From here on a full terminal costs a line's terminal copy, never
|
|
290
|
+
// the supervisor (the event log keeps every line). The mirror is installed even with
|
|
291
|
+
// the event log off, since it is what drops the line. Bind errors above still print
|
|
292
|
+
// blocking — the process exits right after them.
|
|
293
|
+
installConsoleMirror();
|
|
294
|
+
setConsoleNonBlocking();
|
|
295
|
+
|
|
281
296
|
// Keep the supervisor attached to the shell. The worker shares the supervisor's
|
|
282
297
|
// process group, so a terminal Ctrl-C (SIGINT/SIGTERM) is delivered by the TTY
|
|
283
298
|
// to BOTH already — the supervisor must NOT forward those or the worker gets a
|
|
@@ -312,18 +327,29 @@ async function supervisorCommand() {
|
|
|
312
327
|
// After SIGKILLing a worker that may have owned the TUI, the worker had no
|
|
313
328
|
// chance to restore the terminal — the supervisor emits the restore itself
|
|
314
329
|
// (exit alt-screen + show cursor + raw off) so the user's shell is clean.
|
|
315
|
-
|
|
330
|
+
// `exiting` alone may block (writeTerminalSync: a non-blocking queue is dropped at
|
|
331
|
+
// exit). On a reload fallback the supervisor keeps running and is about to respawn
|
|
332
|
+
// the worker that serves the port, so there the restore is a non-blocking write
|
|
333
|
+
// (2026-11-13: the blocking one froze the supervisor behind a terminal that was not
|
|
334
|
+
// reading). Raw mode is left alone there: the supervisor never set it, so its
|
|
335
|
+
// setRawMode(false) is a libuv no-op, and the respawned worker sets its own.
|
|
336
|
+
const restoreTerminalFromSupervisor = ({ exiting = true } = {}) => {
|
|
316
337
|
try {
|
|
338
|
+
if (!exiting) {
|
|
339
|
+
if (process.stdout.isTTY) process.stdout.write('\x1b[?25h\x1b[?1049l');
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
317
342
|
if (process.stdout.isTTY) writeTerminalSync('\x1b[?25h\x1b[?1049l');
|
|
318
343
|
if (process.stdin.isTTY) { try { process.stdin.setRawMode(false); } catch { /* ignore */ } }
|
|
319
344
|
} catch { /* never throw */ }
|
|
320
345
|
};
|
|
321
|
-
process.on('exit', restoreTerminalFromSupervisor);
|
|
346
|
+
process.on('exit', () => restoreTerminalFromSupervisor({ exiting: true }));
|
|
322
347
|
|
|
323
348
|
// Crash-loop guard: count rapid consecutive non-restart exits and back off so
|
|
324
349
|
// a worker that crashes on boot doesn't spin the CPU forking. A clean run for
|
|
325
350
|
// a while resets the counter.
|
|
326
351
|
let crashCount = 0;
|
|
352
|
+
let coldSpawns = 0;
|
|
327
353
|
const CRASH_WINDOW_MS = 10_000;
|
|
328
354
|
const MAX_BACKOFF_MS = 8_000;
|
|
329
355
|
|
|
@@ -337,14 +363,17 @@ async function supervisorCommand() {
|
|
|
337
363
|
// group, so BOTH already receive it — the supervisor therefore does NOT
|
|
338
364
|
// forward those (that would double-deliver). Same-group also means a group
|
|
339
365
|
// SIGKILL of the supervisor reaps the worker (no orphan holding the port).
|
|
340
|
-
const spawnWorker = ({ reload = false } = {}) => {
|
|
366
|
+
const spawnWorker = ({ reload = false, respawn = false } = {}) => {
|
|
341
367
|
const env = { ...process.env, [SERVER_WORKER_ENV]: '1' };
|
|
342
368
|
if (reload) env[SERVER_RELOAD_WORKER_ENV] = '1';
|
|
369
|
+
if (respawn) env[SERVER_RESPAWNED_WORKER_ENV] = '1';
|
|
343
370
|
const child = spawn(process.execPath, process.argv.slice(1), {
|
|
344
371
|
cwd: process.cwd(),
|
|
345
372
|
env,
|
|
346
373
|
stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
|
|
347
374
|
});
|
|
375
|
+
// The child's libuv setup just made the shared terminal blocking again — undo it.
|
|
376
|
+
reassertConsoleNonBlocking();
|
|
348
377
|
const worker = makeWorkerChannel(child);
|
|
349
378
|
if (!reload) {
|
|
350
379
|
// Cold start: hand the socket and tell it to go primary immediately.
|
|
@@ -360,7 +389,7 @@ async function supervisorCommand() {
|
|
|
360
389
|
activeWorker = worker;
|
|
361
390
|
worker.child.removeAllListeners('message');
|
|
362
391
|
worker.child.on('message', msg => {
|
|
363
|
-
if (msg?.type === MSG_RELOAD_REQUEST) orchestrateReload().catch(() => {});
|
|
392
|
+
if (msg?.type === MSG_RELOAD_REQUEST) orchestrateReload({ reloadId: msg.reloadId }).catch(() => {});
|
|
364
393
|
else if (msg?.type === MSG_PRIMARY && activeWorker === worker) {
|
|
365
394
|
// The worker is now sole acceptor on the handle → stop the supervisor
|
|
366
395
|
// racing it for accepts (the steady-state ~78%-hang bug). The supervisor
|
|
@@ -381,11 +410,27 @@ async function supervisorCommand() {
|
|
|
381
410
|
};
|
|
382
411
|
|
|
383
412
|
// Reload orchestration: spawn a fresh headless worker, run the baton, swap.
|
|
384
|
-
const orchestrateReload = async () => {
|
|
413
|
+
const orchestrateReload = async ({ reloadId } = {}) => {
|
|
385
414
|
if (reloadInFlight) return;
|
|
386
415
|
reloadInFlight = true;
|
|
387
416
|
const oldWorker = activeWorker;
|
|
417
|
+
const turnEnd = endTurn;
|
|
388
418
|
let newWorker = null;
|
|
419
|
+
// The abrupt-restart fallback ends THIS reload's turn. If the old worker already
|
|
420
|
+
// exited mid-reload, that turn is over and the respawn loop has a fresh primary
|
|
421
|
+
// serving: killing the reload worker is all that is left. Ending the turn again
|
|
422
|
+
// ended the NEW worker's turn and spawned a second primary (red-team 2026-11-13).
|
|
423
|
+
const fallBack = () => {
|
|
424
|
+
try { newWorker?.child.kill('SIGKILL'); } catch { /* ignore */ }
|
|
425
|
+
if (activeWorker !== oldWorker || endTurn !== turnEnd) {
|
|
426
|
+
console.log('[Maxpool] reload: the old worker exited mid-reload; its replacement keeps serving');
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
try { oldWorker.child.kill('SIGKILL'); } catch { /* ignore */ }
|
|
430
|
+
restoreTerminalFromSupervisor({ exiting: false }); // SIGKILLed workers can't restore the TUI
|
|
431
|
+
activeWorker = null;
|
|
432
|
+
endTurn?.({ code: null, signal: 'SIGKILL', fallback: true });
|
|
433
|
+
};
|
|
389
434
|
try {
|
|
390
435
|
newWorker = spawnWorker({ reload: true });
|
|
391
436
|
// Hold the new worker's spawn-error so a fork failure mid-baton becomes a
|
|
@@ -406,6 +451,7 @@ async function supervisorCommand() {
|
|
|
406
451
|
readyTimeoutMs: RELOAD_READY_MS,
|
|
407
452
|
takeoverTimeoutMs: RELOAD_READY_MS,
|
|
408
453
|
log: msg => console.log(`[Maxpool] ${msg}`),
|
|
454
|
+
reloadId,
|
|
409
455
|
});
|
|
410
456
|
|
|
411
457
|
if (outcome === RELOAD_SWAPPED) {
|
|
@@ -437,18 +483,10 @@ async function supervisorCommand() {
|
|
|
437
483
|
// FALLBACK: the old worker may have released; neither is reliably primary.
|
|
438
484
|
// Kill both and let the respawn loop bring a fresh primary up on the
|
|
439
485
|
// supervisor-owned socket. Queued conns sit in the OS backlog meanwhile.
|
|
440
|
-
|
|
441
|
-
try { oldWorker.child.kill('SIGKILL'); } catch { /* ignore */ }
|
|
442
|
-
restoreTerminalFromSupervisor(); // SIGKILLed workers can't restore the TUI
|
|
443
|
-
activeWorker = null;
|
|
444
|
-
endTurn?.({ code: null, signal: 'SIGKILL', fallback: true });
|
|
486
|
+
fallBack();
|
|
445
487
|
} catch (err) {
|
|
446
488
|
console.error(`[Maxpool] Reload error: ${err.message}; falling back to abrupt restart`);
|
|
447
|
-
|
|
448
|
-
try { oldWorker.child.kill('SIGKILL'); } catch { /* ignore */ }
|
|
449
|
-
restoreTerminalFromSupervisor();
|
|
450
|
-
activeWorker = null;
|
|
451
|
-
endTurn?.({ code: null, signal: 'SIGKILL', fallback: true });
|
|
489
|
+
fallBack();
|
|
452
490
|
} finally {
|
|
453
491
|
acceptTarget = null;
|
|
454
492
|
reloadInFlight = false;
|
|
@@ -477,7 +515,8 @@ async function supervisorCommand() {
|
|
|
477
515
|
handleServerListenError(err, host, port);
|
|
478
516
|
return;
|
|
479
517
|
}
|
|
480
|
-
|
|
518
|
+
// Every cold worker after the first replaces one that may have left the terminal behind.
|
|
519
|
+
const worker = spawnWorker({ reload: false, respawn: coldSpawns++ > 0 });
|
|
481
520
|
const startedAt = Date.now();
|
|
482
521
|
const result = await superviseTurn(worker);
|
|
483
522
|
|
|
@@ -546,50 +585,14 @@ function reapOldWorker(worker, onExited = () => {}) {
|
|
|
546
585
|
worker.child.once('exit', finish);
|
|
547
586
|
}
|
|
548
587
|
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
return
|
|
556
|
-
|
|
557
|
-
send(msg, handle) {
|
|
558
|
-
try {
|
|
559
|
-
if (handle) child.send(msg, handle);
|
|
560
|
-
else child.send(msg);
|
|
561
|
-
} catch { /* IPC may be torn down mid-swap; baton timeouts cover it */ }
|
|
562
|
-
},
|
|
563
|
-
waitFor(types, timeoutMs) {
|
|
564
|
-
const want = Array.isArray(types) ? types : [types];
|
|
565
|
-
return new Promise((resolve, reject) => {
|
|
566
|
-
const cleanup = () => {
|
|
567
|
-
clearTimeout(timer);
|
|
568
|
-
child.removeListener('message', onMsg);
|
|
569
|
-
child.removeListener('exit', onExit);
|
|
570
|
-
child.removeListener('error', onError);
|
|
571
|
-
};
|
|
572
|
-
const onMsg = msg => {
|
|
573
|
-
if (msg && want.includes(msg.type)) { cleanup(); resolve(msg); }
|
|
574
|
-
};
|
|
575
|
-
const onExit = (code, signal) => {
|
|
576
|
-
cleanup();
|
|
577
|
-
reject(new Error(`worker exited (code ${code}, signal ${signal}) before ${want.join('/')}`));
|
|
578
|
-
};
|
|
579
|
-
// A spawn failure (EMFILE/EAGAIN) surfaces as 'error', not 'exit' — the
|
|
580
|
-
// baton must see it as a failed step (→ ROLLED_BACK/FALLBACK), not hang.
|
|
581
|
-
const onError = err => { cleanup(); reject(new Error(`worker spawn error before ${want.join('/')}: ${err.message}`)); };
|
|
582
|
-
const timer = setTimeout(() => {
|
|
583
|
-
cleanup();
|
|
584
|
-
reject(new Error(`timed out waiting for ${want.join('/')}`));
|
|
585
|
-
}, timeoutMs);
|
|
586
|
-
timer.unref?.();
|
|
587
|
-
child.on('message', onMsg);
|
|
588
|
-
child.once('exit', onExit);
|
|
589
|
-
child.once('error', onError);
|
|
590
|
-
});
|
|
591
|
-
},
|
|
592
|
-
};
|
|
588
|
+
// Test seam (gated: inert unless a MAXPOOL_TEST_* variable is set). '1' fires every
|
|
589
|
+
// time; any other value is a file path and fires ONCE across processes: the first
|
|
590
|
+
// process that deletes the file acts (a reload worker is a fresh process each time).
|
|
591
|
+
function testHookFires(name) {
|
|
592
|
+
const v = process.env[name];
|
|
593
|
+
if (!v) return false;
|
|
594
|
+
if (v === '1') return true;
|
|
595
|
+
try { unlinkSync(v); return true; } catch { return false; }
|
|
593
596
|
}
|
|
594
597
|
|
|
595
598
|
function delay(ms) {
|
|
@@ -597,6 +600,26 @@ function delay(ms) {
|
|
|
597
600
|
}
|
|
598
601
|
|
|
599
602
|
async function serverWorkerCommand() {
|
|
603
|
+
const isReloadWorker = process.env[SERVER_RELOAD_WORKER_ENV] === '1';
|
|
604
|
+
const isRespawnedWorker = process.env[SERVER_RESPAWNED_WORKER_ENV] === '1';
|
|
605
|
+
const useTUI = process.stdout.isTTY && process.stdin.isTTY;
|
|
606
|
+
// The TUI makes stdout non-blocking; stderr too, or a console.error on a quit path
|
|
607
|
+
// (drain timeout, force exit) blocks in write(2) while the terminal is not reading
|
|
608
|
+
// (red-team round 2). The console mirror then drops a line's terminal copy while the
|
|
609
|
+
// terminal is behind; the event log keeps it.
|
|
610
|
+
if (useTUI) setConsoleNonBlocking({ streams: [process.stderr] });
|
|
611
|
+
// A reload worker with a TUI must not write plain logs to the shared terminal while
|
|
612
|
+
// booting toward takeover: the old worker's dashboard still owns it, and Node's
|
|
613
|
+
// default tty write BLOCKS — with the terminal not reading, a boot line ("Duplicate
|
|
614
|
+
// guard", "Team pools", "Config providers") stalled the worker before its IPC handler
|
|
615
|
+
// existed, so it never answered the readiness probe (2026-11-13). So the muzzle goes
|
|
616
|
+
// on BEFORE the first line this process can print (event-log appends still run once
|
|
617
|
+
// the path is set; the mirror is installed even with the event log off, since it is
|
|
618
|
+
// what does the muzzling). Lifted in becomePrimary at takeover.
|
|
619
|
+
if (isReloadWorker && useTUI) {
|
|
620
|
+
installConsoleMirror();
|
|
621
|
+
setConsoleStdoutSuppressed(true);
|
|
622
|
+
}
|
|
600
623
|
const config = await loadOrCreateConfig();
|
|
601
624
|
// The lone unsupervised worker owns rotation; under a supervisor, the supervisor does.
|
|
602
625
|
initEventLog(config, { manageRotation: typeof process.send !== 'function' });
|
|
@@ -649,7 +672,6 @@ async function serverWorkerCommand() {
|
|
|
649
672
|
// baton). A reload worker boots HEADLESS without the writer lease and waits
|
|
650
673
|
// for the baton; a cold-start worker takes the lease on MSG_LISTEN.
|
|
651
674
|
const supervised = typeof process.send === 'function';
|
|
652
|
-
const isReloadWorker = process.env[SERVER_RELOAD_WORKER_ENV] === '1';
|
|
653
675
|
|
|
654
676
|
// M6: a reload worker boots WITHOUT the writer lease so the AM-level brick
|
|
655
677
|
// guard (ensureTokenFresh no-op) is CLOSED for the entire headless window —
|
|
@@ -959,13 +981,8 @@ async function serverWorkerCommand() {
|
|
|
959
981
|
// takeover (becomePrimary) — single-owner terminal, enforced by the baton order
|
|
960
982
|
// (old worker's tui.stop() precedes MSG_RELEASED precedes the new worker's
|
|
961
983
|
// MSG_TAKEOVER→tui.start()). It boots headless-silent until then (stdout muzzled
|
|
962
|
-
//
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
// A reload worker with a TUI must not write plain logs to the shared terminal
|
|
966
|
-
// (still owned by the old worker's TUI) while booting toward takeover; suppress
|
|
967
|
-
// stdout (event-log append still runs). Lifted in becomePrimary at takeover.
|
|
968
|
-
if (isReloadWorker && useTUI) setConsoleStdoutSuppressed(true);
|
|
984
|
+
// at the top of this function), so its boot logs can't paint over the old worker's
|
|
985
|
+
// live TUI or block on a terminal that is not reading.
|
|
969
986
|
|
|
970
987
|
let tui = null;
|
|
971
988
|
let server = null;
|
|
@@ -977,14 +994,26 @@ async function serverWorkerCommand() {
|
|
|
977
994
|
// reload; makes this (now non-owner) worker's exit-path restoreTerminal a NO-OP
|
|
978
995
|
// so it can't flip the shared TTY's raw-mode/alt-screen back and clobber the new
|
|
979
996
|
// worker's live TUI. Only the current terminal owner (or the supervisor) restores.
|
|
980
|
-
|
|
997
|
+
// A reload worker starts without the terminal (the old worker's dashboard owns it until
|
|
998
|
+
// takeTerminal): dying before that — a build that fails at the readiness probe — must
|
|
999
|
+
// not write an exit sequence over the live dashboard, or block on it.
|
|
1000
|
+
let terminalHandedOff = isReloadWorker;
|
|
1001
|
+
// Raw-mode switches that must not stall a serving worker (takeover, TTY re-assert).
|
|
1002
|
+
const terminalMode = createTerminalModeSwitcher();
|
|
1003
|
+
// An update's cold restart that waits for the terminal to catch up (cancel function).
|
|
1004
|
+
let cancelDeferredColdRestart = null;
|
|
981
1005
|
// Self-heal timer for a seamless reload that rolls back (new worker fails to boot,
|
|
982
1006
|
// we're never released). Cleared on MSG_RELEASE; fires cancelRestart otherwise.
|
|
983
1007
|
let reloadWatchdog = null;
|
|
984
1008
|
// One cold-restart fallback per reload request — see the rollback watchdog below.
|
|
985
1009
|
let coldFallbackUsed = false;
|
|
1010
|
+
// Set by the update paths just before they request a restart; the reload's settle
|
|
1011
|
+
// clears it, so it describes THAT reload and never a later one.
|
|
986
1012
|
let reloadIsForUpdate = false;
|
|
987
|
-
|
|
1013
|
+
// The reload this worker requested last: its id, echoed by the supervisor in
|
|
1014
|
+
// MSG_ROLLED_BACK.
|
|
1015
|
+
let reloadSeq = 0;
|
|
1016
|
+
let openReload = null;
|
|
988
1017
|
|
|
989
1018
|
// Best-effort terminal restore on ANY abnormal exit path (uncaughtException,
|
|
990
1019
|
// a bare process.exit, a crash) so the user's shell is never left in raw mode
|
|
@@ -1019,8 +1048,16 @@ async function serverWorkerCommand() {
|
|
|
1019
1048
|
// skip the exit-75 restart while DRAINING: a shutdown-time write to a dead
|
|
1020
1049
|
// terminal can throw here, and restarting then would respawn the very orphan
|
|
1021
1050
|
// a terminal-close shutdown is trying to avoid — let the drain finish instead.
|
|
1022
|
-
// Exiting → the 'exit' listener restores the terminal (synchronously, once)
|
|
1023
|
-
|
|
1051
|
+
// Exiting → the 'exit' listener restores the terminal (synchronously, once) —
|
|
1052
|
+
// unsupervised only. Supervised, the respawned worker takes the terminal over (as
|
|
1053
|
+
// after restartWorkerNow): a crashed primary that waited for the terminal on its way
|
|
1054
|
+
// out held the only accepting socket, and the port was dead until the terminal read
|
|
1055
|
+
// (red-team round 5). Same guard at every supervised exit-75 below.
|
|
1056
|
+
if (!isReloadWorker && !draining) {
|
|
1057
|
+
terminalMode.close();
|
|
1058
|
+
if (supervised) terminalHandedOff = true;
|
|
1059
|
+
process.exit(SERVER_RESTART_EXIT_CODE);
|
|
1060
|
+
}
|
|
1024
1061
|
restoreTerminal({ exiting: false });
|
|
1025
1062
|
});
|
|
1026
1063
|
// Swallow EPIPE/EIO from writes to a hung-up terminal so they can't crash us.
|
|
@@ -1098,9 +1135,21 @@ async function serverWorkerCommand() {
|
|
|
1098
1135
|
const restartWorkerNow = () => {
|
|
1099
1136
|
if (draining) return;
|
|
1100
1137
|
draining = true;
|
|
1138
|
+
// First: a raw-mode switch still waiting for the terminal must not hold this exit
|
|
1139
|
+
// (red-team round 2) — kill the helper; the next worker sets its own mode.
|
|
1140
|
+
terminalMode.close();
|
|
1101
1141
|
if (syncTimer) clearInterval(syncTimer);
|
|
1102
1142
|
if (updateTimer) clearInterval(updateTimer);
|
|
1103
|
-
|
|
1143
|
+
// Supervised, the respawned worker takes the terminal over (and the supervisor's
|
|
1144
|
+
// exit restores it if none does): hand it off like a seamless reload does, so
|
|
1145
|
+
// neither this stop() nor the exit handler waits for the terminal — a mode switch
|
|
1146
|
+
// (TCSADRAIN) or the exit handler's synchronous write. A worker stuck there still
|
|
1147
|
+
// holds the listening socket, and the port went dead while the terminal was not
|
|
1148
|
+
// reading (red-team round 1, 2026-11-13). Node restores the terminal mode at exit
|
|
1149
|
+
// by itself, without waiting (measured on a pty). Unsupervised, nothing else will
|
|
1150
|
+
// restore the user's shell, so the exit path keeps its synchronous restore.
|
|
1151
|
+
if (supervised) terminalHandedOff = true;
|
|
1152
|
+
if (tui?.running) { tui.stop(supervised ? { keepRaw: true } : {}); } // plain-log mode resumes for the drain messages
|
|
1104
1153
|
console.log('\n[Maxpool] Restarting server now; queued requests will reconnect automatically.');
|
|
1105
1154
|
server.closeAllConnections?.();
|
|
1106
1155
|
// Best-effort: flush quota + settle in-flight refreshes/prober before exit so
|
|
@@ -1122,6 +1171,74 @@ async function serverWorkerCommand() {
|
|
|
1122
1171
|
]).finally(() => process.exit(SERVER_RESTART_EXIT_CODE));
|
|
1123
1172
|
};
|
|
1124
1173
|
|
|
1174
|
+
// A seamless reload that did not hand over: resume serving NOW (2026-11-13: waiting
|
|
1175
|
+
// for the 50s self-heal kept the pool admitting nothing that long, although the
|
|
1176
|
+
// supervisor had already said why) and decide whether an update must still land.
|
|
1177
|
+
// `answered`: the supervisor sent MSG_ROLLED_BACK; `reason` is its 'timeout' (the new
|
|
1178
|
+
// build was merely slow) or 'failed' (it reported failure / died before ready).
|
|
1179
|
+
// `idless`: the notice named no reload (a supervisor that predates ids): it settles,
|
|
1180
|
+
// but its reason is not trusted for a cold restart (see rollbackNoticeAction).
|
|
1181
|
+
// Nothing about a notice outlives this call: a reason recorded for later was what a
|
|
1182
|
+
// stale notice used to feed into the NEXT reload's self-heal (red-team round 2).
|
|
1183
|
+
const settleRolledBackReload = ({ answered, reason = null, idless = false }) => {
|
|
1184
|
+
if (reloadWatchdog) { clearTimeout(reloadWatchdog); reloadWatchdog = null; }
|
|
1185
|
+
if (draining) return; // MSG_RELEASE already arrived → a real reload, not a rollback
|
|
1186
|
+
// An update that asked while this reload was already open was folded into it.
|
|
1187
|
+
const forUpdate = reloadIsForUpdate;
|
|
1188
|
+
openReload = null;
|
|
1189
|
+
reloadIsForUpdate = false;
|
|
1190
|
+
const why = !answered ? 'the supervisor did not answer'
|
|
1191
|
+
: reason === 'failed' ? 'the new worker failed to start'
|
|
1192
|
+
: idless ? 'the supervisor gave up on the new worker'
|
|
1193
|
+
: reason === 'timeout' ? 'the new worker did not report ready in time'
|
|
1194
|
+
: 'the supervisor gave no reason';
|
|
1195
|
+
if (restartController?.cancelRestart()) {
|
|
1196
|
+
console.log(`[Maxpool] Reload rolled back (${why}).`);
|
|
1197
|
+
}
|
|
1198
|
+
// FALL BACK TO A COLD RESTART — but ONLY when a newer build is actually on disk
|
|
1199
|
+
// waiting to be picked up. Resuming the old worker is what made "press u → c"
|
|
1200
|
+
// look like nothing happened: the graceful swap failed under machine load and
|
|
1201
|
+
// the update was abandoned, forever. Gating on a real pending update preserves
|
|
1202
|
+
// the safety property that a plain failed reload leaves a HEALTHY worker serving
|
|
1203
|
+
// (never kill a working process for nothing), while an update still lands.
|
|
1204
|
+
// One attempt per reload request, so a build that cannot boot can't crash-loop.
|
|
1205
|
+
// Cold-restart ONLY when the new build was merely SLOW to signal ready (a
|
|
1206
|
+
// loaded machine — the reported case). If it reported failure or died before
|
|
1207
|
+
// ready, the build cannot boot: restarting into it would leave the user with no
|
|
1208
|
+
// working proxy at all, strictly worse than the bug being fixed. No answer / no
|
|
1209
|
+
// reason is treated as unsafe too — and so is an id-less 'timeout': a supervisor
|
|
1210
|
+
// that predates ids sends it for a new build that EXITED before ready as well, and
|
|
1211
|
+
// cold-restarting into that build left no working proxy, respawned forever
|
|
1212
|
+
// (red-team round 2). Such an update waits for a full relaunch of Maxpool.
|
|
1213
|
+
const swapWasSlowNotBroken = answered && reason === 'timeout' && !idless;
|
|
1214
|
+
if (forUpdate && swapWasSlowNotBroken && !coldFallbackUsed) {
|
|
1215
|
+
coldFallbackUsed = true;
|
|
1216
|
+
// The respawned worker switches the terminal to raw mode before it serves, and
|
|
1217
|
+
// that waits for the terminal to read: while this terminal is behind, keep serving
|
|
1218
|
+
// here and restart once it has caught up (red-team 2026-11-13: the port froze).
|
|
1219
|
+
if (tui?.running && tui._nb && process.stdout.writableLength > 0) {
|
|
1220
|
+
console.log('[Maxpool] The update will finish with a full restart as soon as this terminal shows output again.');
|
|
1221
|
+
cancelDeferredColdRestart = whenTerminalCaughtUp(() => {
|
|
1222
|
+
cancelDeferredColdRestart = null;
|
|
1223
|
+
if (draining) return;
|
|
1224
|
+
console.log('[Maxpool] Applying the update with a full restart instead — one moment.');
|
|
1225
|
+
restartWorkerNow();
|
|
1226
|
+
});
|
|
1227
|
+
} else {
|
|
1228
|
+
console.log('[Maxpool] Applying the update with a full restart instead — one moment.');
|
|
1229
|
+
restartWorkerNow();
|
|
1230
|
+
}
|
|
1231
|
+
} else if (forUpdate && reason === 'failed') {
|
|
1232
|
+
console.log('[Maxpool] The new version failed to start — resumed serving on the current version. Update NOT applied.');
|
|
1233
|
+
} else if (forUpdate && idless) {
|
|
1234
|
+
console.log('[Maxpool] The update did not hand over — resumed serving on the current version. Update NOT applied. Quit and start Maxpool again to finish it.');
|
|
1235
|
+
} else if (forUpdate) {
|
|
1236
|
+
console.log(`[Maxpool] The update did not hand over (${why}) — resumed serving on the current version. Update NOT applied.`);
|
|
1237
|
+
} else {
|
|
1238
|
+
console.log('[Maxpool] Rollback complete — resumed serving on the current version.');
|
|
1239
|
+
}
|
|
1240
|
+
};
|
|
1241
|
+
|
|
1125
1242
|
// Seamless reload entry point. When supervised, ask the supervisor to run the
|
|
1126
1243
|
// baton (spawn new worker, hand off socket + lease). If anything is off
|
|
1127
1244
|
// (not supervised, no IPC), fall back to the abrupt restart.
|
|
@@ -1137,42 +1254,24 @@ async function serverWorkerCommand() {
|
|
|
1137
1254
|
if (!forceCold && reloadStrategy({ supervised }) === 'seamless') {
|
|
1138
1255
|
try {
|
|
1139
1256
|
coldFallbackUsed = false; // fresh reload → fresh cold-fallback budget
|
|
1140
|
-
|
|
1141
|
-
|
|
1257
|
+
cancelDeferredColdRestart?.(); // this reload supersedes an update still waiting to restart
|
|
1258
|
+
cancelDeferredColdRestart = null;
|
|
1259
|
+
openReload = { id: `${process.pid}-${++reloadSeq}` };
|
|
1260
|
+
process.send({ type: MSG_RELOAD_REQUEST, reloadId: openReload.id });
|
|
1142
1261
|
// Arm the rollback self-heal: restartController already latched
|
|
1143
1262
|
// pending/restarting + paused admission (in _restart, before we got here).
|
|
1144
1263
|
// On a SUCCESSFUL reload the supervisor sends MSG_RELEASE (→ releaseBaton
|
|
1145
1264
|
// AndDrain clears this) then we exit. On a ROLLBACK we're never released and
|
|
1146
1265
|
// never exit — so if this fires, resume serving instead of 503-ing forever.
|
|
1266
|
+
// Backstop only: the supervisor normally answers with MSG_ROLLED_BACK, which
|
|
1267
|
+
// settles the rollback at once (settleRolledBackReload). This fires when no
|
|
1268
|
+
// answer came — a supervisor that predates the notice, or one that is stuck.
|
|
1147
1269
|
if (reloadWatchdog) clearTimeout(reloadWatchdog);
|
|
1270
|
+
// Any notice while this reload is open settles it at once (MSG_ROLLED_BACK
|
|
1271
|
+
// handler), so firing here means no answer came at all.
|
|
1148
1272
|
reloadWatchdog = setTimeout(() => {
|
|
1149
1273
|
reloadWatchdog = null;
|
|
1150
|
-
|
|
1151
|
-
if (restartController?.cancelRestart()) {
|
|
1152
|
-
console.log('[Maxpool] Reload rolled back (new worker never took over).');
|
|
1153
|
-
}
|
|
1154
|
-
// FALL BACK TO A COLD RESTART — but ONLY when a newer build is actually on disk
|
|
1155
|
-
// waiting to be picked up. Resuming the old worker is what made "press u → c"
|
|
1156
|
-
// look like nothing happened: the graceful swap failed under machine load and
|
|
1157
|
-
// the update was abandoned, forever. Gating on a real pending update preserves
|
|
1158
|
-
// the safety property that a plain failed reload leaves a HEALTHY worker serving
|
|
1159
|
-
// (never kill a working process for nothing), while an update still lands.
|
|
1160
|
-
// One attempt per reload request, so a build that cannot boot can't crash-loop.
|
|
1161
|
-
// Cold-restart ONLY when the new build was merely SLOW to signal ready (a
|
|
1162
|
-
// loaded machine — the reported case). If it reported failure or died before
|
|
1163
|
-
// ready, the build cannot boot: restarting into it would leave the user with no
|
|
1164
|
-
// working proxy at all, strictly worse than the bug being fixed. Absent reason
|
|
1165
|
-
// (older supervisor) is treated as unsafe.
|
|
1166
|
-
const swapWasSlowNotBroken = lastRollbackReason === 'timeout';
|
|
1167
|
-
if (reloadIsForUpdate && swapWasSlowNotBroken && !coldFallbackUsed) {
|
|
1168
|
-
coldFallbackUsed = true;
|
|
1169
|
-
console.log('[Maxpool] Applying the update with a full restart instead — one moment.');
|
|
1170
|
-
restartWorkerNow();
|
|
1171
|
-
} else if (reloadIsForUpdate && !swapWasSlowNotBroken) {
|
|
1172
|
-
console.log('[Maxpool] The new version failed to start — resumed serving on the current version. Update NOT applied.');
|
|
1173
|
-
} else {
|
|
1174
|
-
console.log('[Maxpool] Rollback complete — resumed serving on the current version.');
|
|
1175
|
-
}
|
|
1274
|
+
settleRolledBackReload({ answered: false });
|
|
1176
1275
|
}, RELOAD_ROLLBACK_SELFHEAL_MS);
|
|
1177
1276
|
reloadWatchdog.unref?.();
|
|
1178
1277
|
return;
|
|
@@ -1203,7 +1302,11 @@ async function serverWorkerCommand() {
|
|
|
1203
1302
|
// the server comes back" is provable without allocating a pty. Gated — never
|
|
1204
1303
|
// active in normal runs.
|
|
1205
1304
|
if (process.env.MAXPOOL_TEST_RESTART_SIGNAL === '1') {
|
|
1206
|
-
process.on('SIGUSR2', () =>
|
|
1305
|
+
process.on('SIGUSR2', () => {
|
|
1306
|
+
if (testHookFires('MAXPOOL_TEST_SIGNAL_COLD_RESTART')) { restartWorkerNow(); return; } // test seam: the abrupt restart
|
|
1307
|
+
if (testHookFires('MAXPOOL_TEST_RELOAD_IS_UPDATE')) reloadIsForUpdate = true; // test seam: an update-driven reload
|
|
1308
|
+
restartController?.requestRestart();
|
|
1309
|
+
});
|
|
1207
1310
|
}
|
|
1208
1311
|
|
|
1209
1312
|
const shutdownGracefully = (reason, options = {}) => {
|
|
@@ -1221,7 +1324,14 @@ async function serverWorkerCommand() {
|
|
|
1221
1324
|
draining = true;
|
|
1222
1325
|
if (syncTimer) clearInterval(syncTimer);
|
|
1223
1326
|
if (updateTimer) clearInterval(updateTimer);
|
|
1224
|
-
|
|
1327
|
+
terminalMode.close();
|
|
1328
|
+
// Supervised, the supervisor's own exit restores the terminal (exit sequence + the
|
|
1329
|
+
// mode it started with) right after ours, so this worker never waits for the
|
|
1330
|
+
// terminal on its way out: no mode switch (TCSADRAIN) and no blocking exit write —
|
|
1331
|
+
// a quit (SIGTERM included) with the terminal not reading hung there (red-team
|
|
1332
|
+
// round 2). Unsupervised, nothing else restores the user's shell.
|
|
1333
|
+
if (supervised) terminalHandedOff = true;
|
|
1334
|
+
if (tui?.running) tui.stop(supervised ? { keepRaw: true } : {});
|
|
1225
1335
|
// Best-effort final flush + settle writes (fire-and-forget; the drain timeout
|
|
1226
1336
|
// below bounds total quit time, and write barriers protect the on-disk state).
|
|
1227
1337
|
(async () => {
|
|
@@ -1336,6 +1446,7 @@ async function serverWorkerCommand() {
|
|
|
1336
1446
|
shutdownGracefully('quit');
|
|
1337
1447
|
},
|
|
1338
1448
|
onRestart: () => {
|
|
1449
|
+
if (testHookFires('MAXPOOL_TEST_RELOAD_IS_UPDATE')) reloadIsForUpdate = true; // test seam: an update-driven reload
|
|
1339
1450
|
restartController.requestRestart();
|
|
1340
1451
|
},
|
|
1341
1452
|
});
|
|
@@ -1455,13 +1566,27 @@ async function serverWorkerCommand() {
|
|
|
1455
1566
|
// the update check. `viaTakeover` true means we acquired the socket through the
|
|
1456
1567
|
// baton (a reload) — freeze the update check so a reload doesn't re-probe npm
|
|
1457
1568
|
// (only a cold supervisor start self-updates).
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1569
|
+
// Take the terminal: lift the reload-worker boot muzzle so the TUI's first render +
|
|
1570
|
+
// any plain-mode fallback logs reach stdout (no-op for a worker that never muzzled),
|
|
1571
|
+
// then start the dashboard. On a TAKEOVER raw mode is switched on a helper thread
|
|
1572
|
+
// (terminal-mode.js): the old worker's frames may fill the terminal, this worker is
|
|
1573
|
+
// serving, and switching it blocks the switching thread while the terminal is behind
|
|
1574
|
+
// (2026-11-13: the takeover froze inside tui.start() and the port went dead; then a
|
|
1575
|
+
// frame small enough to look "taken" still stalled it ~1.7s on a bursty terminal). A cold
|
|
1576
|
+
// start keeps the synchronous switch, made before its first frame: with nothing of
|
|
1577
|
+
// ours queued yet it waits only for what the terminal already holds, and a deferred
|
|
1578
|
+
// switch there left a busy terminal's keyboard dead (tui-terminal-pty (b)/(c)).
|
|
1579
|
+
const takeTerminal = ({ viaTakeover }) => {
|
|
1462
1580
|
setConsoleStdoutSuppressed(false);
|
|
1581
|
+
terminalHandedOff = false; // ours now: our exit restores it
|
|
1582
|
+
// A RESPAWNED worker (the supervisor restarted one before it) also switches off its
|
|
1583
|
+
// serving thread: the worker before it may have left the terminal behind — an
|
|
1584
|
+
// update's deferred cold restart fires once our queue empties, while the kernel's
|
|
1585
|
+
// still holds a burst's worth (red-team round 2). Only the first launch keeps the
|
|
1586
|
+
// synchronous switch.
|
|
1587
|
+
const offThread = viaTakeover || isRespawnedWorker;
|
|
1463
1588
|
if (tui) {
|
|
1464
|
-
if (tui.start()) {
|
|
1589
|
+
if (tui.start(offThread ? { rawWhenDrained: true, modeSwitcher: terminalMode } : {})) {
|
|
1465
1590
|
console.log(`Listening on ${host}:${port} with ${accounts.length} account(s)`);
|
|
1466
1591
|
} else {
|
|
1467
1592
|
tui = null;
|
|
@@ -1470,6 +1595,13 @@ async function serverWorkerCommand() {
|
|
|
1470
1595
|
} else {
|
|
1471
1596
|
logPlainServerStart({ host, port, accounts, threshold, config });
|
|
1472
1597
|
}
|
|
1598
|
+
};
|
|
1599
|
+
|
|
1600
|
+
const becomePrimary = async ({ viaTakeover }) => {
|
|
1601
|
+
// Cold start: the terminal first, as always (nothing else to hand over). A reload
|
|
1602
|
+
// TAKEOVER serves and announces primacy FIRST and takes the terminal last (below):
|
|
1603
|
+
// whatever the terminal does, the baton completes and the port keeps answering.
|
|
1604
|
+
if (!viaTakeover) takeTerminal({ viaTakeover });
|
|
1473
1605
|
// On a reload TAKEOVER, adopt the latest tokens from disk. The headless
|
|
1474
1606
|
// reload worker loaded config at spawn time; the OLD worker may have rotated
|
|
1475
1607
|
// a single-use refresh token DURING the handoff (and flushed it before
|
|
@@ -1492,6 +1624,8 @@ async function serverWorkerCommand() {
|
|
|
1492
1624
|
// Tell the supervisor we are the sole acceptor now → it stops its own accept
|
|
1493
1625
|
// loop (the steady-state race fix). Cold start AND takeover both signal this.
|
|
1494
1626
|
if (supervised) { try { process.send({ type: MSG_PRIMARY }); } catch { /* ignore */ } }
|
|
1627
|
+
// Same tick as the send: no supervisor message (MSG_TTY_REASSERT) can land between.
|
|
1628
|
+
if (viaTakeover) takeTerminal({ viaTakeover });
|
|
1495
1629
|
|
|
1496
1630
|
// Reload-storm guard: only a cold (non-reload) lease holder probes npm.
|
|
1497
1631
|
// A reload-spawned/takeover worker must NEVER re-probe (1x not 2x traffic).
|
|
@@ -1526,15 +1660,37 @@ async function serverWorkerCommand() {
|
|
|
1526
1660
|
const releaseBatonAndDrain = async () => {
|
|
1527
1661
|
if (draining) { try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ } return; }
|
|
1528
1662
|
draining = true;
|
|
1529
|
-
// Terminal hand-off, part 1 (red-team S-2): stop drawing and taking keys NOW
|
|
1530
|
-
// let a slow terminal catch up WHILE steps 1-4 run — they can take many seconds,
|
|
1531
|
-
// and the supervisor waits only so long for MSG_RELEASED.
|
|
1663
|
+
// Terminal hand-off, part 1 (red-team S-2): stop drawing and taking keys NOW.
|
|
1532
1664
|
const terminalCatchUp = tui?.running ? tui.beginHandOff({ timeoutMs: 1500 }) : null;
|
|
1533
1665
|
// MSG_RELEASE arrived → the new worker booted OK and this reload is really
|
|
1534
1666
|
// proceeding (not a rollback). Disarm the rollback self-heal.
|
|
1535
1667
|
if (reloadWatchdog) { clearTimeout(reloadWatchdog); reloadWatchdog = null; }
|
|
1536
1668
|
if (syncTimer) clearInterval(syncTimer);
|
|
1537
1669
|
if (updateTimer) clearInterval(updateTimer);
|
|
1670
|
+
|
|
1671
|
+
// Terminal hand-off, part 2 (r2 B2, 2026-11-12): let a slow terminal catch up,
|
|
1672
|
+
// then the exit sequence, each bounded (1.5s) — BEFORE releasing: the new worker
|
|
1673
|
+
// paints right after MSG_RELEASED, and its first frame must not interleave with
|
|
1674
|
+
// ours. Done while this worker STILL ACCEPTS: once it stops, only the supervisor's
|
|
1675
|
+
// re-armed listener (after MSG_RELEASED) takes connections, and a terminal wait
|
|
1676
|
+
// in between refused every connection for ~3s (red-team round 1, 2026-11-13).
|
|
1677
|
+
// Muzzle our remaining logs first, so none print once stop() gives console.log
|
|
1678
|
+
// back to plain stdout. Never turns blocking back on.
|
|
1679
|
+
setConsoleStdoutSuppressed(true);
|
|
1680
|
+
if (terminalCatchUp) {
|
|
1681
|
+
await terminalCatchUp;
|
|
1682
|
+
appendEventLog(`[Maxpool] Reload: handing the terminal to the new worker (${process.stdout.writableLength || 0} bytes not yet shown)`);
|
|
1683
|
+
// keepRaw: the incoming worker puts the terminal in raw mode itself, and
|
|
1684
|
+
// switching it here would freeze this worker — still streaming in-flight
|
|
1685
|
+
// responses — for as long as the terminal is behind (TCSADRAIN; see TUI.stop).
|
|
1686
|
+
await tui.handOff({ timeoutMs: 1500, flushing: terminalCatchUp, keepRaw: true });
|
|
1687
|
+
}
|
|
1688
|
+
// The terminal is now HANDED OFF: the incoming worker will take the TTY at
|
|
1689
|
+
// MSG_TAKEOVER. Arm the guard UNCONDITIONALLY (even if the TUI was already
|
|
1690
|
+
// stopped) so this worker's later exit(0) restoreTerminal can't clobber the
|
|
1691
|
+
// new worker's TUI.
|
|
1692
|
+
terminalHandedOff = true;
|
|
1693
|
+
|
|
1538
1694
|
// Stop accepting NEW connections; KEEP in-flight requests alive.
|
|
1539
1695
|
server.maxpoolBeginDrain?.();
|
|
1540
1696
|
server.close(() => {});
|
|
@@ -1560,28 +1716,6 @@ async function serverWorkerCommand() {
|
|
|
1560
1716
|
// off (M3 — otherwise the new worker boots from the invalidated token).
|
|
1561
1717
|
await flushConfigWrites();
|
|
1562
1718
|
await flushStateWrites();
|
|
1563
|
-
|
|
1564
|
-
// Muzzle our own remaining logs (in-flight requests completing) BEFORE the
|
|
1565
|
-
// hand-off's stop() gives console.log back to plain stdout, so none of them
|
|
1566
|
-
// print on the user's screen or paint the new worker's alt-screen.
|
|
1567
|
-
setConsoleStdoutSuppressed(true);
|
|
1568
|
-
// Terminal hand-off, part 2 (r2 B2, 2026-11-12): finish the catch-up started at
|
|
1569
|
-
// the top, then the exit sequence, each bounded (1.5s) — BEFORE releasing: the
|
|
1570
|
-
// new worker paints right after MSG_RELEASED, and its first frame must not
|
|
1571
|
-
// interleave with ours. Never turns blocking back on; keeps serving in-flight.
|
|
1572
|
-
if (terminalCatchUp) {
|
|
1573
|
-
await terminalCatchUp;
|
|
1574
|
-
appendEventLog(`[Maxpool] Reload: handing the terminal to the new worker (${process.stdout.writableLength || 0} bytes not yet shown)`);
|
|
1575
|
-
// keepRaw: the incoming worker puts the terminal in raw mode itself, and
|
|
1576
|
-
// switching it here would freeze this worker — still streaming in-flight
|
|
1577
|
-
// responses — for as long as the terminal is behind (TCSADRAIN; see TUI.stop).
|
|
1578
|
-
await tui.handOff({ timeoutMs: 1500, flushing: terminalCatchUp, keepRaw: true });
|
|
1579
|
-
}
|
|
1580
|
-
// The terminal is now HANDED OFF: the incoming worker will take the TTY at
|
|
1581
|
-
// MSG_TAKEOVER. Arm the guard UNCONDITIONALLY (even if the TUI was already
|
|
1582
|
-
// stopped) so this worker's later exit(0) restoreTerminal can't clobber the
|
|
1583
|
-
// new worker's TUI.
|
|
1584
|
-
terminalHandedOff = true;
|
|
1585
1719
|
try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ }
|
|
1586
1720
|
|
|
1587
1721
|
// (macOS) releaseLease() above dropped the wake-lock, but we still stream our
|
|
@@ -1601,6 +1735,7 @@ async function serverWorkerCommand() {
|
|
|
1601
1735
|
// state file. Bounded (1s) and best-effort: losing the delta is a measurement
|
|
1602
1736
|
// gap, hanging the released worker is an operational fault, so the timeout wins.
|
|
1603
1737
|
const exitAfterMergeFlush = () => {
|
|
1738
|
+
terminalMode.close(); // a switch still waiting for the terminal never holds the exit
|
|
1604
1739
|
Promise.race([mergeFlushCapacityDelta(), delay(1000)])
|
|
1605
1740
|
.catch(() => {})
|
|
1606
1741
|
.finally(() => process.exit(0));
|
|
@@ -1629,6 +1764,7 @@ async function serverWorkerCommand() {
|
|
|
1629
1764
|
// a cold worker a dead/absent socket, fail LOUD and exit-75 so it respawns —
|
|
1630
1765
|
// never sit alive-but-not-primary (the blank-screen `r`-restart hang).
|
|
1631
1766
|
console.error('[Maxpool] Cold worker received MSG_LISTEN with no socket handle — exiting for respawn.');
|
|
1767
|
+
terminalHandedOff = true; // supervised (this branch is): never wait for the terminal on the way out
|
|
1632
1768
|
process.exit(SERVER_RESTART_EXIT_CODE);
|
|
1633
1769
|
} else if (msg?.type === MSG_CONN) {
|
|
1634
1770
|
// A connection the supervisor's own accept loop won the race for (cold start
|
|
@@ -1642,6 +1778,16 @@ async function serverWorkerCommand() {
|
|
|
1642
1778
|
// supervisor's rollback (old worker stays primary, zero disruption).
|
|
1643
1779
|
if (process.env.MAXPOOL_TEST_FAIL_RELOAD_WORKER === '1') {
|
|
1644
1780
|
process.send({ type: MSG_FAILED, reason: 'test-forced failure' });
|
|
1781
|
+
} else if (testHookFires('MAXPOOL_TEST_RELOAD_EXIT_BEFORE_READY')) {
|
|
1782
|
+
// Test seam: a new build that dies before it reports ready.
|
|
1783
|
+
process.exit(1);
|
|
1784
|
+
} else if (testHookFires('MAXPOOL_TEST_RELOAD_NEVER_READY')) {
|
|
1785
|
+
// Test seam: a new build too slow to ever report ready → the supervisor's
|
|
1786
|
+
// readiness timeout ('timeout' rollback).
|
|
1787
|
+
} else if (Number(process.env.MAXPOOL_TEST_RELOAD_READY_DELAY_MS) > 0) {
|
|
1788
|
+
// Test seam: a new build that reports ready only after a delay.
|
|
1789
|
+
setTimeout(() => { try { process.send({ type: MSG_READY }); } catch { /* gone */ } },
|
|
1790
|
+
Number(process.env.MAXPOOL_TEST_RELOAD_READY_DELAY_MS));
|
|
1645
1791
|
} else {
|
|
1646
1792
|
process.send({ type: MSG_READY });
|
|
1647
1793
|
}
|
|
@@ -1653,9 +1799,12 @@ async function serverWorkerCommand() {
|
|
|
1653
1799
|
await becomePrimary({ viaTakeover: true });
|
|
1654
1800
|
} else if (msg?.type === MSG_TTY_REASSERT) {
|
|
1655
1801
|
// The old worker has now EXITED, and its exit reset the shared terminal out
|
|
1656
|
-
// of raw mode (libuv's uv_tty_reset_mode on process exit). Reclaim it
|
|
1657
|
-
//
|
|
1658
|
-
|
|
1802
|
+
// of raw mode (libuv's uv_tty_reset_mode on process exit). Reclaim it on the
|
|
1803
|
+
// helper thread, so a terminal that is behind never stalls this worker;
|
|
1804
|
+
// without a helper, reclaimRawMode (waits for our queue to drain first).
|
|
1805
|
+
const r = await terminalMode.setRaw(true, { toggle: true });
|
|
1806
|
+
if (r.ok) appendEventLog(`[Maxpool] reload: terminal reclaimed in ${r.ms}ms (helper thread)`);
|
|
1807
|
+
else await reclaimRawModeLogged({ flush: tui?._nb ? o => tui.flushOutput(o) : null });
|
|
1659
1808
|
// Re-assert the screen state too: the dying worker's alt-screen exit and
|
|
1660
1809
|
// cursor-show can land after ours.
|
|
1661
1810
|
if (tui?.running && process.stdout.isTTY) {
|
|
@@ -1665,8 +1814,19 @@ async function serverWorkerCommand() {
|
|
|
1665
1814
|
} catch { /* ignore */ }
|
|
1666
1815
|
}
|
|
1667
1816
|
} else if (msg?.type === MSG_ROLLED_BACK) {
|
|
1668
|
-
//
|
|
1669
|
-
|
|
1817
|
+
// The supervisor gave up on the new worker; we were never released. Settle now
|
|
1818
|
+
// (resume admission, maybe cold-restart for an update) — not at the self-heal —
|
|
1819
|
+
// when the notice names the reload that is open: a late notice for an earlier
|
|
1820
|
+
// reload once settled the next one and cold-restarted mid-baton (red-team
|
|
1821
|
+
// 2026-11-13). An id-less notice comes from a supervisor that predates ids (it
|
|
1822
|
+
// survives seamless updates): it settles the open reload at once too — the
|
|
1823
|
+
// ~50s of 503s until the self-heal were the reported symptom — but never
|
|
1824
|
+
// authorizes a cold restart (red-team round 2).
|
|
1825
|
+
const action = rollbackNoticeAction({ noticeId: msg.reloadId, currentId: openReload?.id ?? null, open: Boolean(reloadWatchdog) });
|
|
1826
|
+
if (action === 'settle') settleRolledBackReload({ answered: true, reason: msg.reason || null });
|
|
1827
|
+
else if (action === 'settle-idless') settleRolledBackReload({ answered: true, reason: msg.reason || null, idless: true });
|
|
1828
|
+
else if (msg.reloadId != null) appendEventLog(`[Maxpool] Ignored a rollback notice for an earlier reload (${msg.reloadId}).`);
|
|
1829
|
+
else appendEventLog('[Maxpool] Ignored a rollback notice with no reload open.');
|
|
1670
1830
|
} else if (msg?.type === MSG_RELEASE) {
|
|
1671
1831
|
// Baton release: stop accepting NEW (keep in-flight), retire keep-alive
|
|
1672
1832
|
// sockets with Connection: close, stop writing, flush once, drop TUI.
|
|
@@ -1679,10 +1839,18 @@ async function serverWorkerCommand() {
|
|
|
1679
1839
|
// (closes the silent-non-primary hang class). A RELOAD worker mid-baton must
|
|
1680
1840
|
// NOT exit (that kills the supervisor loop) — report MSG_FAILED so the
|
|
1681
1841
|
// supervisor rolls back and the old worker stays primary (zero disruption).
|
|
1682
|
-
if (!isReloadWorker) process.exit(SERVER_RESTART_EXIT_CODE);
|
|
1842
|
+
if (!isReloadWorker) { terminalHandedOff = true; process.exit(SERVER_RESTART_EXIT_CODE); } // supervised: as above
|
|
1683
1843
|
try { process.send({ type: MSG_FAILED, reason: err.message }); } catch { /* ignore */ }
|
|
1684
1844
|
}
|
|
1685
1845
|
});
|
|
1846
|
+
// Test seam (inert unless set): SIGUSR2 feeds the handler above a message, so a pty
|
|
1847
|
+
// test can drive its exit-75 paths for real. 'listen-no-handle' = MSG_LISTEN with no
|
|
1848
|
+
// socket; 'listen' = MSG_LISTEN with a handle, which a serving worker refuses
|
|
1849
|
+
// (ERR_SERVER_ALREADY_LISTEN) → the handler's error path.
|
|
1850
|
+
const ipcSignal = process.env.MAXPOOL_TEST_IPC_SIGNAL;
|
|
1851
|
+
if (ipcSignal === 'listen-no-handle' || ipcSignal === 'listen') {
|
|
1852
|
+
process.on('SIGUSR2', () => process.emit('message', { type: MSG_LISTEN }, ipcSignal === 'listen' ? {} : undefined));
|
|
1853
|
+
}
|
|
1686
1854
|
} else {
|
|
1687
1855
|
// Direct (non-TTY service) path: bind the port ourselves, exactly as before.
|
|
1688
1856
|
const onListenError = err => handleServerListenError(err, host, port);
|
|
@@ -2556,17 +2724,25 @@ function argValue(flag) {
|
|
|
2556
2724
|
}
|
|
2557
2725
|
|
|
2558
2726
|
function handleServerListenError(err, host, port) {
|
|
2727
|
+
let lines;
|
|
2559
2728
|
if (err.code === 'EADDRINUSE') {
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2729
|
+
lines = [
|
|
2730
|
+
`[Maxpool] ${host}:${port} is already in use.`,
|
|
2731
|
+
'Another Maxpool proxy may already be running.',
|
|
2732
|
+
'Check the existing server with: maxpool status',
|
|
2733
|
+
`Find the listener with: lsof -nP -iTCP:${port} -sTCP:LISTEN`,
|
|
2734
|
+
];
|
|
2564
2735
|
} else if (err.code === 'EACCES') {
|
|
2565
|
-
|
|
2566
|
-
|
|
2736
|
+
lines = [
|
|
2737
|
+
`[Maxpool] Permission denied while listening on ${host}:${port}.`,
|
|
2738
|
+
'Choose a non-privileged port in the Maxpool config.',
|
|
2739
|
+
];
|
|
2567
2740
|
} else {
|
|
2568
|
-
|
|
2741
|
+
lines = [`[Maxpool] Failed to listen on ${host}:${port}: ${err.message}`];
|
|
2569
2742
|
}
|
|
2743
|
+
// The supervisor's terminal may be non-blocking by now (a respawn's bind): print
|
|
2744
|
+
// synchronously, or the reason is dropped at exit.
|
|
2745
|
+
printFatal(lines);
|
|
2570
2746
|
process.exit(1);
|
|
2571
2747
|
}
|
|
2572
2748
|
|