maxpool 1.24.92 → 1.24.93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/event-log.js +79 -1
- package/src/index.js +325 -149
- package/src/reload-protocol.js +86 -2
- package/src/terminal-mode.js +124 -0
- package/src/tui.js +106 -15
package/package.json
CHANGED
package/src/event-log.js
CHANGED
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
// write is fire-and-forget and all errors are swallowed.
|
|
7
7
|
|
|
8
8
|
import { appendFile, stat, rename, chmod } from 'node:fs/promises';
|
|
9
|
+
import { isatty } from 'node:tty';
|
|
10
|
+
import { writeSync } from 'node:fs';
|
|
9
11
|
|
|
10
12
|
const MAX_BYTES = 5 * 1024 * 1024; // rotate at ~5 MB
|
|
11
13
|
// Keep 5 generations (~30 MB total), not 1. With a single backup the log rotated away the
|
|
@@ -122,6 +124,78 @@ let stdoutSuppressed = false;
|
|
|
122
124
|
*/
|
|
123
125
|
export function setConsoleStdoutSuppressed(v) { stdoutSuppressed = !!v; }
|
|
124
126
|
|
|
127
|
+
// Streams made non-blocking by setConsoleNonBlocking (real terminals only).
|
|
128
|
+
const nonBlockingStreams = new Set();
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Never let the terminal stall this process (2026-11-13: the supervisor sat inside a
|
|
132
|
+
* console.log for 7+ minutes while the terminal was not reading, so a failed reload was
|
|
133
|
+
* neither rolled back nor reaped). Node writes a TTY synchronously; this turns the real
|
|
134
|
+
* terminal streams non-blocking — the same switch the TUI makes for its own frames
|
|
135
|
+
* (tui.js start()) — and the console mirror then drops a line's terminal copy while the
|
|
136
|
+
* terminal has not taken what it already got. The event log still gets every line.
|
|
137
|
+
* libuv reopens a TTY per process, so this never changes another process's terminal.
|
|
138
|
+
* Returns the number of streams switched.
|
|
139
|
+
*/
|
|
140
|
+
export function setConsoleNonBlocking({ streams = [process.stdout, process.stderr], isTTY = s => isatty(s.fd) } = {}) {
|
|
141
|
+
let n = 0;
|
|
142
|
+
for (const s of streams) {
|
|
143
|
+
if (!s || nonBlockingStreams.has(s) || !isTTY(s) || typeof s._handle?.setBlocking !== 'function') continue;
|
|
144
|
+
try { s._handle.setBlocking(false); nonBlockingStreams.add(s); n++; } catch { /* not a libuv tty */ }
|
|
145
|
+
}
|
|
146
|
+
return n;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Turn the switched streams non-blocking AGAIN. Call right after spawning a child with
|
|
151
|
+
* inherited stdio: libuv's child setup clears O_NONBLOCK on fds 0-2 (uv__process_child_init),
|
|
152
|
+
* and an inherited fd shares the parent's open file description, so every such spawn
|
|
153
|
+
* silently makes the parent's terminal blocking again (2026-11-13: the supervisor still
|
|
154
|
+
* froze in write() after spawning the reload worker). spawn() returns only after the
|
|
155
|
+
* child has exec'd, so the child's reset is already done when this runs.
|
|
156
|
+
*/
|
|
157
|
+
export function reassertConsoleNonBlocking() {
|
|
158
|
+
for (const s of nonBlockingStreams) {
|
|
159
|
+
try { s._handle?.setBlocking?.(false); } catch { /* stream gone */ }
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Undo setConsoleNonBlocking — for EXIT paths only. A non-blocking write still queued at
|
|
165
|
+
* process.exit is dropped, so a fatal message printed on the way out vanished
|
|
166
|
+
* (red-team 2026-11-13: a respawn's EADDRINUSE). Blocking is right there: the process
|
|
167
|
+
* is ending anyway, and the message waits for the terminal instead of being lost.
|
|
168
|
+
*/
|
|
169
|
+
export function setConsoleBlocking() {
|
|
170
|
+
for (const s of nonBlockingStreams) {
|
|
171
|
+
try { s._handle?.setBlocking?.(true); } catch { /* stream gone */ }
|
|
172
|
+
}
|
|
173
|
+
nonBlockingStreams.clear();
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Print the lines that explain why the process is about to exit, so that they reach the
|
|
178
|
+
* terminal even when it is not reading. A stderr we made non-blocking is made blocking
|
|
179
|
+
* again and written directly (fd 2), past anything still queued in front of it; every
|
|
180
|
+
* other stderr prints through console.error as usual.
|
|
181
|
+
*/
|
|
182
|
+
export function printFatal(lines, { stream = process.stderr, write = writeSync } = {}) {
|
|
183
|
+
if (!nonBlockingStreams.has(stream)) {
|
|
184
|
+
for (const line of lines) console.error(line);
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
setConsoleBlocking();
|
|
188
|
+
for (const line of lines) {
|
|
189
|
+
try { appendEventLog(line); } catch { /* never */ }
|
|
190
|
+
try { write(stream.fd, `${line}\n`); } catch { /* terminal gone */ }
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** True when `stream` is one we made non-blocking and the terminal has not drained it. */
|
|
195
|
+
function terminalBehind(stream) {
|
|
196
|
+
return nonBlockingStreams.has(stream) && Boolean(stream.writableNeedDrain);
|
|
197
|
+
}
|
|
198
|
+
|
|
125
199
|
export function installConsoleMirror() {
|
|
126
200
|
if (origConsole) return; // idempotent
|
|
127
201
|
origConsole = { log: console.log.bind(console), error: console.error.bind(console) };
|
|
@@ -129,7 +203,10 @@ export function installConsoleMirror() {
|
|
|
129
203
|
const orig = origConsole[method];
|
|
130
204
|
console[method] = (...args) => {
|
|
131
205
|
try { appendEventLog(args.map(a => (typeof a === 'string' ? a : String(a))).join(' ')); } catch { /* never */ }
|
|
132
|
-
if (
|
|
206
|
+
if (stdoutSuppressed) return;
|
|
207
|
+
// Read the stream live: a test (or the TUI) may swap process.stdout.
|
|
208
|
+
if (terminalBehind(method === 'log' ? process.stdout : process.stderr)) return;
|
|
209
|
+
orig(...args);
|
|
133
210
|
};
|
|
134
211
|
}
|
|
135
212
|
}
|
|
@@ -142,4 +219,5 @@ export function __resetEventLogForTest() {
|
|
|
142
219
|
if (origConsole) { console.log = origConsole.log; console.error = origConsole.error; origConsole = null; }
|
|
143
220
|
if (rotationTimer) { clearInterval(rotationTimer); rotationTimer = null; }
|
|
144
221
|
logPath = null; manageRotation = false; writeChain = Promise.resolve(); queued = 0; dropped = 0; stdoutSuppressed = false;
|
|
222
|
+
nonBlockingStreams.clear();
|
|
145
223
|
}
|
package/src/index.js
CHANGED
|
@@ -12,10 +12,11 @@ import net from 'node:net';
|
|
|
12
12
|
// connection is opened, hence here at module top.
|
|
13
13
|
net.setDefaultAutoSelectFamilyAttemptTimeout(Number(process.env.MAXPOOL_FAMILY_ATTEMPT_MS) || 2000);
|
|
14
14
|
import { stat } from 'node:fs/promises';
|
|
15
|
+
import { unlinkSync } from 'node:fs';
|
|
15
16
|
import { spawn, spawnSync } from 'node:child_process';
|
|
16
17
|
import { createInterface } from 'node:readline';
|
|
17
18
|
import { loadOrCreateConfig, loadConfig, saveConfig, atomicConfigUpdate, getConfigPath, loadState, saveState, getStatePath, getLogPath, readGeneration, flushConfigWrites, flushStateWrites, saveStateUnbumped } from './config.js';
|
|
18
|
-
import { setEventLogPath, installConsoleMirror, setConsoleStdoutSuppressed, appendEventLog } from './event-log.js';
|
|
19
|
+
import { setEventLogPath, installConsoleMirror, setConsoleStdoutSuppressed, setConsoleNonBlocking, reassertConsoleNonBlocking, appendEventLog, printFatal } from './event-log.js';
|
|
19
20
|
import { sendLoginToPool, planCliLogin, resolveUsablePools } from './cloud-login.js';
|
|
20
21
|
import { SleepGuard } from './sleep-guard.js';
|
|
21
22
|
import { controllingTerminalAlive } from './terminal-alive.js';
|
|
@@ -42,7 +43,8 @@ net.setDefaultAutoSelectFamilyAttemptTimeout(
|
|
|
42
43
|
import { Prober } from './prober.js';
|
|
43
44
|
import { CapacityLedger } from './capacity-ledger.js';
|
|
44
45
|
import { loginOAuth, fetchProfile, refreshAccessToken, isTokenExpiringSoon, tokenFingerprint, isLoginCancelled } from './oauth.js';
|
|
45
|
-
import { TUI, writeTerminalSync, reclaimRawModeLogged } from './tui.js';
|
|
46
|
+
import { TUI, writeTerminalSync, reclaimRawModeLogged, whenTerminalCaughtUp } from './tui.js';
|
|
47
|
+
import { createTerminalModeSwitcher } from './terminal-mode.js';
|
|
46
48
|
import { RestartController } from './restart-controller.js';
|
|
47
49
|
import { resolveAccounts } from './account-config.js';
|
|
48
50
|
import { maybeCheckForUpdate, getCurrentVersion, markApplied, clearQuarantine, captureBootVersion } from './updater.js';
|
|
@@ -51,7 +53,7 @@ import {
|
|
|
51
53
|
RELOAD_SWAPPED, RELOAD_ROLLED_BACK,
|
|
52
54
|
MSG_LISTEN, MSG_RELEASE, MSG_TAKEOVER, MSG_PROBE_READY,
|
|
53
55
|
MSG_RELOAD_REQUEST, MSG_READY, MSG_FAILED, MSG_RELEASED, MSG_PRIMARY, MSG_ROLLED_BACK,
|
|
54
|
-
MSG_TTY_REASSERT, MSG_CONN, describeOldWorkerExit,
|
|
56
|
+
MSG_TTY_REASSERT, MSG_CONN, describeOldWorkerExit, makeWorkerChannel, rollbackNoticeAction,
|
|
55
57
|
} from './reload-protocol.js';
|
|
56
58
|
|
|
57
59
|
const args = process.argv.slice(2);
|
|
@@ -87,6 +89,9 @@ const SERVER_WORKER_ENV = 'MAXPOOL_SERVER_WORKER';
|
|
|
87
89
|
// Set by the supervisor when it spawns a worker for a seamless reload: that
|
|
88
90
|
// worker boots HEADLESS (plain logs, no writer lease) and waits for the baton.
|
|
89
91
|
const SERVER_RELOAD_WORKER_ENV = 'MAXPOOL_RELOAD_WORKER';
|
|
92
|
+
// Set by the supervisor on every cold worker after its first (a restart / crash
|
|
93
|
+
// respawn): that worker switches raw mode off its serving thread (takeTerminal).
|
|
94
|
+
const SERVER_RESPAWNED_WORKER_ENV = 'MAXPOOL_RESPAWNED_WORKER';
|
|
90
95
|
|
|
91
96
|
// Is maxpool attached to an interactive terminal? Governs SIGNAL semantics: in a
|
|
92
97
|
// terminal a SIGHUP means "the window hung up → shut down" (never reload into a
|
|
@@ -278,6 +283,16 @@ async function supervisorCommand() {
|
|
|
278
283
|
return;
|
|
279
284
|
}
|
|
280
285
|
|
|
286
|
+
// The supervisor shares the terminal with the workers but owns no dashboard, so its
|
|
287
|
+
// lines went out as Node's default BLOCKING tty writes. With the terminal not reading,
|
|
288
|
+
// one reload log line froze it for 7+ minutes (2026-11-13): no rollback notice, no
|
|
289
|
+
// reap, no fallback. From here on a full terminal costs a line's terminal copy, never
|
|
290
|
+
// the supervisor (the event log keeps every line). The mirror is installed even with
|
|
291
|
+
// the event log off, since it is what drops the line. Bind errors above still print
|
|
292
|
+
// blocking — the process exits right after them.
|
|
293
|
+
installConsoleMirror();
|
|
294
|
+
setConsoleNonBlocking();
|
|
295
|
+
|
|
281
296
|
// Keep the supervisor attached to the shell. The worker shares the supervisor's
|
|
282
297
|
// process group, so a terminal Ctrl-C (SIGINT/SIGTERM) is delivered by the TTY
|
|
283
298
|
// to BOTH already — the supervisor must NOT forward those or the worker gets a
|
|
@@ -312,18 +327,29 @@ async function supervisorCommand() {
|
|
|
312
327
|
// After SIGKILLing a worker that may have owned the TUI, the worker had no
|
|
313
328
|
// chance to restore the terminal — the supervisor emits the restore itself
|
|
314
329
|
// (exit alt-screen + show cursor + raw off) so the user's shell is clean.
|
|
315
|
-
|
|
330
|
+
// `exiting` alone may block (writeTerminalSync: a non-blocking queue is dropped at
|
|
331
|
+
// exit). On a reload fallback the supervisor keeps running and is about to respawn
|
|
332
|
+
// the worker that serves the port, so there the restore is a non-blocking write
|
|
333
|
+
// (2026-11-13: the blocking one froze the supervisor behind a terminal that was not
|
|
334
|
+
// reading). Raw mode is left alone there: the supervisor never set it, so its
|
|
335
|
+
// setRawMode(false) is a libuv no-op, and the respawned worker sets its own.
|
|
336
|
+
const restoreTerminalFromSupervisor = ({ exiting = true } = {}) => {
|
|
316
337
|
try {
|
|
338
|
+
if (!exiting) {
|
|
339
|
+
if (process.stdout.isTTY) process.stdout.write('\x1b[?25h\x1b[?1049l');
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
317
342
|
if (process.stdout.isTTY) writeTerminalSync('\x1b[?25h\x1b[?1049l');
|
|
318
343
|
if (process.stdin.isTTY) { try { process.stdin.setRawMode(false); } catch { /* ignore */ } }
|
|
319
344
|
} catch { /* never throw */ }
|
|
320
345
|
};
|
|
321
|
-
process.on('exit', restoreTerminalFromSupervisor);
|
|
346
|
+
process.on('exit', () => restoreTerminalFromSupervisor({ exiting: true }));
|
|
322
347
|
|
|
323
348
|
// Crash-loop guard: count rapid consecutive non-restart exits and back off so
|
|
324
349
|
// a worker that crashes on boot doesn't spin the CPU forking. A clean run for
|
|
325
350
|
// a while resets the counter.
|
|
326
351
|
let crashCount = 0;
|
|
352
|
+
let coldSpawns = 0;
|
|
327
353
|
const CRASH_WINDOW_MS = 10_000;
|
|
328
354
|
const MAX_BACKOFF_MS = 8_000;
|
|
329
355
|
|
|
@@ -337,14 +363,17 @@ async function supervisorCommand() {
|
|
|
337
363
|
// group, so BOTH already receive it — the supervisor therefore does NOT
|
|
338
364
|
// forward those (that would double-deliver). Same-group also means a group
|
|
339
365
|
// SIGKILL of the supervisor reaps the worker (no orphan holding the port).
|
|
340
|
-
const spawnWorker = ({ reload = false } = {}) => {
|
|
366
|
+
const spawnWorker = ({ reload = false, respawn = false } = {}) => {
|
|
341
367
|
const env = { ...process.env, [SERVER_WORKER_ENV]: '1' };
|
|
342
368
|
if (reload) env[SERVER_RELOAD_WORKER_ENV] = '1';
|
|
369
|
+
if (respawn) env[SERVER_RESPAWNED_WORKER_ENV] = '1';
|
|
343
370
|
const child = spawn(process.execPath, process.argv.slice(1), {
|
|
344
371
|
cwd: process.cwd(),
|
|
345
372
|
env,
|
|
346
373
|
stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
|
|
347
374
|
});
|
|
375
|
+
// The child's libuv setup just made the shared terminal blocking again — undo it.
|
|
376
|
+
reassertConsoleNonBlocking();
|
|
348
377
|
const worker = makeWorkerChannel(child);
|
|
349
378
|
if (!reload) {
|
|
350
379
|
// Cold start: hand the socket and tell it to go primary immediately.
|
|
@@ -360,7 +389,7 @@ async function supervisorCommand() {
|
|
|
360
389
|
activeWorker = worker;
|
|
361
390
|
worker.child.removeAllListeners('message');
|
|
362
391
|
worker.child.on('message', msg => {
|
|
363
|
-
if (msg?.type === MSG_RELOAD_REQUEST) orchestrateReload().catch(() => {});
|
|
392
|
+
if (msg?.type === MSG_RELOAD_REQUEST) orchestrateReload({ reloadId: msg.reloadId }).catch(() => {});
|
|
364
393
|
else if (msg?.type === MSG_PRIMARY && activeWorker === worker) {
|
|
365
394
|
// The worker is now sole acceptor on the handle → stop the supervisor
|
|
366
395
|
// racing it for accepts (the steady-state ~78%-hang bug). The supervisor
|
|
@@ -381,11 +410,27 @@ async function supervisorCommand() {
|
|
|
381
410
|
};
|
|
382
411
|
|
|
383
412
|
// Reload orchestration: spawn a fresh headless worker, run the baton, swap.
|
|
384
|
-
const orchestrateReload = async () => {
|
|
413
|
+
const orchestrateReload = async ({ reloadId } = {}) => {
|
|
385
414
|
if (reloadInFlight) return;
|
|
386
415
|
reloadInFlight = true;
|
|
387
416
|
const oldWorker = activeWorker;
|
|
417
|
+
const turnEnd = endTurn;
|
|
388
418
|
let newWorker = null;
|
|
419
|
+
// The abrupt-restart fallback ends THIS reload's turn. If the old worker already
|
|
420
|
+
// exited mid-reload, that turn is over and the respawn loop has a fresh primary
|
|
421
|
+
// serving: killing the reload worker is all that is left. Ending the turn again
|
|
422
|
+
// ended the NEW worker's turn and spawned a second primary (red-team 2026-11-13).
|
|
423
|
+
const fallBack = () => {
|
|
424
|
+
try { newWorker?.child.kill('SIGKILL'); } catch { /* ignore */ }
|
|
425
|
+
if (activeWorker !== oldWorker || endTurn !== turnEnd) {
|
|
426
|
+
console.log('[Maxpool] reload: the old worker exited mid-reload; its replacement keeps serving');
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
try { oldWorker.child.kill('SIGKILL'); } catch { /* ignore */ }
|
|
430
|
+
restoreTerminalFromSupervisor({ exiting: false }); // SIGKILLed workers can't restore the TUI
|
|
431
|
+
activeWorker = null;
|
|
432
|
+
endTurn?.({ code: null, signal: 'SIGKILL', fallback: true });
|
|
433
|
+
};
|
|
389
434
|
try {
|
|
390
435
|
newWorker = spawnWorker({ reload: true });
|
|
391
436
|
// Hold the new worker's spawn-error so a fork failure mid-baton becomes a
|
|
@@ -406,6 +451,7 @@ async function supervisorCommand() {
|
|
|
406
451
|
readyTimeoutMs: RELOAD_READY_MS,
|
|
407
452
|
takeoverTimeoutMs: RELOAD_READY_MS,
|
|
408
453
|
log: msg => console.log(`[Maxpool] ${msg}`),
|
|
454
|
+
reloadId,
|
|
409
455
|
});
|
|
410
456
|
|
|
411
457
|
if (outcome === RELOAD_SWAPPED) {
|
|
@@ -437,18 +483,10 @@ async function supervisorCommand() {
|
|
|
437
483
|
// FALLBACK: the old worker may have released; neither is reliably primary.
|
|
438
484
|
// Kill both and let the respawn loop bring a fresh primary up on the
|
|
439
485
|
// supervisor-owned socket. Queued conns sit in the OS backlog meanwhile.
|
|
440
|
-
|
|
441
|
-
try { oldWorker.child.kill('SIGKILL'); } catch { /* ignore */ }
|
|
442
|
-
restoreTerminalFromSupervisor(); // SIGKILLed workers can't restore the TUI
|
|
443
|
-
activeWorker = null;
|
|
444
|
-
endTurn?.({ code: null, signal: 'SIGKILL', fallback: true });
|
|
486
|
+
fallBack();
|
|
445
487
|
} catch (err) {
|
|
446
488
|
console.error(`[Maxpool] Reload error: ${err.message}; falling back to abrupt restart`);
|
|
447
|
-
|
|
448
|
-
try { oldWorker.child.kill('SIGKILL'); } catch { /* ignore */ }
|
|
449
|
-
restoreTerminalFromSupervisor();
|
|
450
|
-
activeWorker = null;
|
|
451
|
-
endTurn?.({ code: null, signal: 'SIGKILL', fallback: true });
|
|
489
|
+
fallBack();
|
|
452
490
|
} finally {
|
|
453
491
|
acceptTarget = null;
|
|
454
492
|
reloadInFlight = false;
|
|
@@ -477,7 +515,8 @@ async function supervisorCommand() {
|
|
|
477
515
|
handleServerListenError(err, host, port);
|
|
478
516
|
return;
|
|
479
517
|
}
|
|
480
|
-
|
|
518
|
+
// Every cold worker after the first replaces one that may have left the terminal behind.
|
|
519
|
+
const worker = spawnWorker({ reload: false, respawn: coldSpawns++ > 0 });
|
|
481
520
|
const startedAt = Date.now();
|
|
482
521
|
const result = await superviseTurn(worker);
|
|
483
522
|
|
|
@@ -546,50 +585,14 @@ function reapOldWorker(worker, onExited = () => {}) {
|
|
|
546
585
|
worker.child.once('exit', finish);
|
|
547
586
|
}
|
|
548
587
|
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
return
|
|
556
|
-
|
|
557
|
-
send(msg, handle) {
|
|
558
|
-
try {
|
|
559
|
-
if (handle) child.send(msg, handle);
|
|
560
|
-
else child.send(msg);
|
|
561
|
-
} catch { /* IPC may be torn down mid-swap; baton timeouts cover it */ }
|
|
562
|
-
},
|
|
563
|
-
waitFor(types, timeoutMs) {
|
|
564
|
-
const want = Array.isArray(types) ? types : [types];
|
|
565
|
-
return new Promise((resolve, reject) => {
|
|
566
|
-
const cleanup = () => {
|
|
567
|
-
clearTimeout(timer);
|
|
568
|
-
child.removeListener('message', onMsg);
|
|
569
|
-
child.removeListener('exit', onExit);
|
|
570
|
-
child.removeListener('error', onError);
|
|
571
|
-
};
|
|
572
|
-
const onMsg = msg => {
|
|
573
|
-
if (msg && want.includes(msg.type)) { cleanup(); resolve(msg); }
|
|
574
|
-
};
|
|
575
|
-
const onExit = (code, signal) => {
|
|
576
|
-
cleanup();
|
|
577
|
-
reject(new Error(`worker exited (code ${code}, signal ${signal}) before ${want.join('/')}`));
|
|
578
|
-
};
|
|
579
|
-
// A spawn failure (EMFILE/EAGAIN) surfaces as 'error', not 'exit' — the
|
|
580
|
-
// baton must see it as a failed step (→ ROLLED_BACK/FALLBACK), not hang.
|
|
581
|
-
const onError = err => { cleanup(); reject(new Error(`worker spawn error before ${want.join('/')}: ${err.message}`)); };
|
|
582
|
-
const timer = setTimeout(() => {
|
|
583
|
-
cleanup();
|
|
584
|
-
reject(new Error(`timed out waiting for ${want.join('/')}`));
|
|
585
|
-
}, timeoutMs);
|
|
586
|
-
timer.unref?.();
|
|
587
|
-
child.on('message', onMsg);
|
|
588
|
-
child.once('exit', onExit);
|
|
589
|
-
child.once('error', onError);
|
|
590
|
-
});
|
|
591
|
-
},
|
|
592
|
-
};
|
|
588
|
+
// Test seam (gated: inert unless a MAXPOOL_TEST_* variable is set). '1' fires every
|
|
589
|
+
// time; any other value is a file path and fires ONCE across processes: the first
|
|
590
|
+
// process that deletes the file acts (a reload worker is a fresh process each time).
|
|
591
|
+
function testHookFires(name) {
|
|
592
|
+
const v = process.env[name];
|
|
593
|
+
if (!v) return false;
|
|
594
|
+
if (v === '1') return true;
|
|
595
|
+
try { unlinkSync(v); return true; } catch { return false; }
|
|
593
596
|
}
|
|
594
597
|
|
|
595
598
|
function delay(ms) {
|
|
@@ -597,6 +600,26 @@ function delay(ms) {
|
|
|
597
600
|
}
|
|
598
601
|
|
|
599
602
|
async function serverWorkerCommand() {
|
|
603
|
+
const isReloadWorker = process.env[SERVER_RELOAD_WORKER_ENV] === '1';
|
|
604
|
+
const isRespawnedWorker = process.env[SERVER_RESPAWNED_WORKER_ENV] === '1';
|
|
605
|
+
const useTUI = process.stdout.isTTY && process.stdin.isTTY;
|
|
606
|
+
// The TUI makes stdout non-blocking; stderr too, or a console.error on a quit path
|
|
607
|
+
// (drain timeout, force exit) blocks in write(2) while the terminal is not reading
|
|
608
|
+
// (red-team round 2). The console mirror then drops a line's terminal copy while the
|
|
609
|
+
// terminal is behind; the event log keeps it.
|
|
610
|
+
if (useTUI) setConsoleNonBlocking({ streams: [process.stderr] });
|
|
611
|
+
// A reload worker with a TUI must not write plain logs to the shared terminal while
|
|
612
|
+
// booting toward takeover: the old worker's dashboard still owns it, and Node's
|
|
613
|
+
// default tty write BLOCKS — with the terminal not reading, a boot line ("Duplicate
|
|
614
|
+
// guard", "Team pools", "Config providers") stalled the worker before its IPC handler
|
|
615
|
+
// existed, so it never answered the readiness probe (2026-11-13). So the muzzle goes
|
|
616
|
+
// on BEFORE the first line this process can print (event-log appends still run once
|
|
617
|
+
// the path is set; the mirror is installed even with the event log off, since it is
|
|
618
|
+
// what does the muzzling). Lifted in becomePrimary at takeover.
|
|
619
|
+
if (isReloadWorker && useTUI) {
|
|
620
|
+
installConsoleMirror();
|
|
621
|
+
setConsoleStdoutSuppressed(true);
|
|
622
|
+
}
|
|
600
623
|
const config = await loadOrCreateConfig();
|
|
601
624
|
// The lone unsupervised worker owns rotation; under a supervisor, the supervisor does.
|
|
602
625
|
initEventLog(config, { manageRotation: typeof process.send !== 'function' });
|
|
@@ -649,7 +672,6 @@ async function serverWorkerCommand() {
|
|
|
649
672
|
// baton). A reload worker boots HEADLESS without the writer lease and waits
|
|
650
673
|
// for the baton; a cold-start worker takes the lease on MSG_LISTEN.
|
|
651
674
|
const supervised = typeof process.send === 'function';
|
|
652
|
-
const isReloadWorker = process.env[SERVER_RELOAD_WORKER_ENV] === '1';
|
|
653
675
|
|
|
654
676
|
// M6: a reload worker boots WITHOUT the writer lease so the AM-level brick
|
|
655
677
|
// guard (ensureTokenFresh no-op) is CLOSED for the entire headless window —
|
|
@@ -959,13 +981,8 @@ async function serverWorkerCommand() {
|
|
|
959
981
|
// takeover (becomePrimary) — single-owner terminal, enforced by the baton order
|
|
960
982
|
// (old worker's tui.stop() precedes MSG_RELEASED precedes the new worker's
|
|
961
983
|
// MSG_TAKEOVER→tui.start()). It boots headless-silent until then (stdout muzzled
|
|
962
|
-
//
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
// A reload worker with a TUI must not write plain logs to the shared terminal
|
|
966
|
-
// (still owned by the old worker's TUI) while booting toward takeover; suppress
|
|
967
|
-
// stdout (event-log append still runs). Lifted in becomePrimary at takeover.
|
|
968
|
-
if (isReloadWorker && useTUI) setConsoleStdoutSuppressed(true);
|
|
984
|
+
// at the top of this function), so its boot logs can't paint over the old worker's
|
|
985
|
+
// live TUI or block on a terminal that is not reading.
|
|
969
986
|
|
|
970
987
|
let tui = null;
|
|
971
988
|
let server = null;
|
|
@@ -977,14 +994,26 @@ async function serverWorkerCommand() {
|
|
|
977
994
|
// reload; makes this (now non-owner) worker's exit-path restoreTerminal a NO-OP
|
|
978
995
|
// so it can't flip the shared TTY's raw-mode/alt-screen back and clobber the new
|
|
979
996
|
// worker's live TUI. Only the current terminal owner (or the supervisor) restores.
|
|
980
|
-
|
|
997
|
+
// A reload worker starts without the terminal (the old worker's dashboard owns it until
|
|
998
|
+
// takeTerminal): dying before that — a build that fails at the readiness probe — must
|
|
999
|
+
// not write an exit sequence over the live dashboard, or block on it.
|
|
1000
|
+
let terminalHandedOff = isReloadWorker;
|
|
1001
|
+
// Raw-mode switches that must not stall a serving worker (takeover, TTY re-assert).
|
|
1002
|
+
const terminalMode = createTerminalModeSwitcher();
|
|
1003
|
+
// An update's cold restart that waits for the terminal to catch up (cancel function).
|
|
1004
|
+
let cancelDeferredColdRestart = null;
|
|
981
1005
|
// Self-heal timer for a seamless reload that rolls back (new worker fails to boot,
|
|
982
1006
|
// we're never released). Cleared on MSG_RELEASE; fires cancelRestart otherwise.
|
|
983
1007
|
let reloadWatchdog = null;
|
|
984
1008
|
// One cold-restart fallback per reload request — see the rollback watchdog below.
|
|
985
1009
|
let coldFallbackUsed = false;
|
|
1010
|
+
// Set by the update paths just before they request a restart; the reload's settle
|
|
1011
|
+
// clears it, so it describes THAT reload and never a later one.
|
|
986
1012
|
let reloadIsForUpdate = false;
|
|
987
|
-
|
|
1013
|
+
// The reload this worker requested last: its id, echoed by the supervisor in
|
|
1014
|
+
// MSG_ROLLED_BACK.
|
|
1015
|
+
let reloadSeq = 0;
|
|
1016
|
+
let openReload = null;
|
|
988
1017
|
|
|
989
1018
|
// Best-effort terminal restore on ANY abnormal exit path (uncaughtException,
|
|
990
1019
|
// a bare process.exit, a crash) so the user's shell is never left in raw mode
|
|
@@ -1019,8 +1048,16 @@ async function serverWorkerCommand() {
|
|
|
1019
1048
|
// skip the exit-75 restart while DRAINING: a shutdown-time write to a dead
|
|
1020
1049
|
// terminal can throw here, and restarting then would respawn the very orphan
|
|
1021
1050
|
// a terminal-close shutdown is trying to avoid — let the drain finish instead.
|
|
1022
|
-
// Exiting → the 'exit' listener restores the terminal (synchronously, once)
|
|
1023
|
-
|
|
1051
|
+
// Exiting → the 'exit' listener restores the terminal (synchronously, once) —
|
|
1052
|
+
// unsupervised only. Supervised, the respawned worker takes the terminal over (as
|
|
1053
|
+
// after restartWorkerNow): a crashed primary that waited for the terminal on its way
|
|
1054
|
+
// out held the only accepting socket, and the port was dead until the terminal read
|
|
1055
|
+
// (red-team round 5). Same guard at every supervised exit-75 below.
|
|
1056
|
+
if (!isReloadWorker && !draining) {
|
|
1057
|
+
terminalMode.close();
|
|
1058
|
+
if (supervised) terminalHandedOff = true;
|
|
1059
|
+
process.exit(SERVER_RESTART_EXIT_CODE);
|
|
1060
|
+
}
|
|
1024
1061
|
restoreTerminal({ exiting: false });
|
|
1025
1062
|
});
|
|
1026
1063
|
// Swallow EPIPE/EIO from writes to a hung-up terminal so they can't crash us.
|
|
@@ -1098,9 +1135,21 @@ async function serverWorkerCommand() {
|
|
|
1098
1135
|
const restartWorkerNow = () => {
|
|
1099
1136
|
if (draining) return;
|
|
1100
1137
|
draining = true;
|
|
1138
|
+
// First: a raw-mode switch still waiting for the terminal must not hold this exit
|
|
1139
|
+
// (red-team round 2) — kill the helper; the next worker sets its own mode.
|
|
1140
|
+
terminalMode.close();
|
|
1101
1141
|
if (syncTimer) clearInterval(syncTimer);
|
|
1102
1142
|
if (updateTimer) clearInterval(updateTimer);
|
|
1103
|
-
|
|
1143
|
+
// Supervised, the respawned worker takes the terminal over (and the supervisor's
|
|
1144
|
+
// exit restores it if none does): hand it off like a seamless reload does, so
|
|
1145
|
+
// neither this stop() nor the exit handler waits for the terminal — a mode switch
|
|
1146
|
+
// (TCSADRAIN) or the exit handler's synchronous write. A worker stuck there still
|
|
1147
|
+
// holds the listening socket, and the port went dead while the terminal was not
|
|
1148
|
+
// reading (red-team round 1, 2026-11-13). Node restores the terminal mode at exit
|
|
1149
|
+
// by itself, without waiting (measured on a pty). Unsupervised, nothing else will
|
|
1150
|
+
// restore the user's shell, so the exit path keeps its synchronous restore.
|
|
1151
|
+
if (supervised) terminalHandedOff = true;
|
|
1152
|
+
if (tui?.running) { tui.stop(supervised ? { keepRaw: true } : {}); } // plain-log mode resumes for the drain messages
|
|
1104
1153
|
console.log('\n[Maxpool] Restarting server now; queued requests will reconnect automatically.');
|
|
1105
1154
|
server.closeAllConnections?.();
|
|
1106
1155
|
// Best-effort: flush quota + settle in-flight refreshes/prober before exit so
|
|
@@ -1122,6 +1171,74 @@ async function serverWorkerCommand() {
|
|
|
1122
1171
|
]).finally(() => process.exit(SERVER_RESTART_EXIT_CODE));
|
|
1123
1172
|
};
|
|
1124
1173
|
|
|
1174
|
+
// A seamless reload that did not hand over: resume serving NOW (2026-11-13: waiting
|
|
1175
|
+
// for the 50s self-heal kept the pool admitting nothing that long, although the
|
|
1176
|
+
// supervisor had already said why) and decide whether an update must still land.
|
|
1177
|
+
// `answered`: the supervisor sent MSG_ROLLED_BACK; `reason` is its 'timeout' (the new
|
|
1178
|
+
// build was merely slow) or 'failed' (it reported failure / died before ready).
|
|
1179
|
+
// `idless`: the notice named no reload (a supervisor that predates ids): it settles,
|
|
1180
|
+
// but its reason is not trusted for a cold restart (see rollbackNoticeAction).
|
|
1181
|
+
// Nothing about a notice outlives this call: a reason recorded for later was what a
|
|
1182
|
+
// stale notice used to feed into the NEXT reload's self-heal (red-team round 2).
|
|
1183
|
+
const settleRolledBackReload = ({ answered, reason = null, idless = false }) => {
|
|
1184
|
+
if (reloadWatchdog) { clearTimeout(reloadWatchdog); reloadWatchdog = null; }
|
|
1185
|
+
if (draining) return; // MSG_RELEASE already arrived → a real reload, not a rollback
|
|
1186
|
+
// An update that asked while this reload was already open was folded into it.
|
|
1187
|
+
const forUpdate = reloadIsForUpdate;
|
|
1188
|
+
openReload = null;
|
|
1189
|
+
reloadIsForUpdate = false;
|
|
1190
|
+
const why = !answered ? 'the supervisor did not answer'
|
|
1191
|
+
: reason === 'failed' ? 'the new worker failed to start'
|
|
1192
|
+
: idless ? 'the supervisor gave up on the new worker'
|
|
1193
|
+
: reason === 'timeout' ? 'the new worker did not report ready in time'
|
|
1194
|
+
: 'the supervisor gave no reason';
|
|
1195
|
+
if (restartController?.cancelRestart()) {
|
|
1196
|
+
console.log(`[Maxpool] Reload rolled back (${why}).`);
|
|
1197
|
+
}
|
|
1198
|
+
// FALL BACK TO A COLD RESTART — but ONLY when a newer build is actually on disk
|
|
1199
|
+
// waiting to be picked up. Resuming the old worker is what made "press u → c"
|
|
1200
|
+
// look like nothing happened: the graceful swap failed under machine load and
|
|
1201
|
+
// the update was abandoned, forever. Gating on a real pending update preserves
|
|
1202
|
+
// the safety property that a plain failed reload leaves a HEALTHY worker serving
|
|
1203
|
+
// (never kill a working process for nothing), while an update still lands.
|
|
1204
|
+
// One attempt per reload request, so a build that cannot boot can't crash-loop.
|
|
1205
|
+
// Cold-restart ONLY when the new build was merely SLOW to signal ready (a
|
|
1206
|
+
// loaded machine — the reported case). If it reported failure or died before
|
|
1207
|
+
// ready, the build cannot boot: restarting into it would leave the user with no
|
|
1208
|
+
// working proxy at all, strictly worse than the bug being fixed. No answer / no
|
|
1209
|
+
// reason is treated as unsafe too — and so is an id-less 'timeout': a supervisor
|
|
1210
|
+
// that predates ids sends it for a new build that EXITED before ready as well, and
|
|
1211
|
+
// cold-restarting into that build left no working proxy, respawned forever
|
|
1212
|
+
// (red-team round 2). Such an update waits for a full relaunch of Maxpool.
|
|
1213
|
+
const swapWasSlowNotBroken = answered && reason === 'timeout' && !idless;
|
|
1214
|
+
if (forUpdate && swapWasSlowNotBroken && !coldFallbackUsed) {
|
|
1215
|
+
coldFallbackUsed = true;
|
|
1216
|
+
// The respawned worker switches the terminal to raw mode before it serves, and
|
|
1217
|
+
// that waits for the terminal to read: while this terminal is behind, keep serving
|
|
1218
|
+
// here and restart once it has caught up (red-team 2026-11-13: the port froze).
|
|
1219
|
+
if (tui?.running && tui._nb && process.stdout.writableLength > 0) {
|
|
1220
|
+
console.log('[Maxpool] The update will finish with a full restart as soon as this terminal shows output again.');
|
|
1221
|
+
cancelDeferredColdRestart = whenTerminalCaughtUp(() => {
|
|
1222
|
+
cancelDeferredColdRestart = null;
|
|
1223
|
+
if (draining) return;
|
|
1224
|
+
console.log('[Maxpool] Applying the update with a full restart instead — one moment.');
|
|
1225
|
+
restartWorkerNow();
|
|
1226
|
+
});
|
|
1227
|
+
} else {
|
|
1228
|
+
console.log('[Maxpool] Applying the update with a full restart instead — one moment.');
|
|
1229
|
+
restartWorkerNow();
|
|
1230
|
+
}
|
|
1231
|
+
} else if (forUpdate && reason === 'failed') {
|
|
1232
|
+
console.log('[Maxpool] The new version failed to start — resumed serving on the current version. Update NOT applied.');
|
|
1233
|
+
} else if (forUpdate && idless) {
|
|
1234
|
+
console.log('[Maxpool] The update did not hand over — resumed serving on the current version. Update NOT applied. Quit and start Maxpool again to finish it.');
|
|
1235
|
+
} else if (forUpdate) {
|
|
1236
|
+
console.log(`[Maxpool] The update did not hand over (${why}) — resumed serving on the current version. Update NOT applied.`);
|
|
1237
|
+
} else {
|
|
1238
|
+
console.log('[Maxpool] Rollback complete — resumed serving on the current version.');
|
|
1239
|
+
}
|
|
1240
|
+
};
|
|
1241
|
+
|
|
1125
1242
|
// Seamless reload entry point. When supervised, ask the supervisor to run the
|
|
1126
1243
|
// baton (spawn new worker, hand off socket + lease). If anything is off
|
|
1127
1244
|
// (not supervised, no IPC), fall back to the abrupt restart.
|
|
@@ -1137,42 +1254,24 @@ async function serverWorkerCommand() {
|
|
|
1137
1254
|
if (!forceCold && reloadStrategy({ supervised }) === 'seamless') {
|
|
1138
1255
|
try {
|
|
1139
1256
|
coldFallbackUsed = false; // fresh reload → fresh cold-fallback budget
|
|
1140
|
-
|
|
1141
|
-
|
|
1257
|
+
cancelDeferredColdRestart?.(); // this reload supersedes an update still waiting to restart
|
|
1258
|
+
cancelDeferredColdRestart = null;
|
|
1259
|
+
openReload = { id: `${process.pid}-${++reloadSeq}` };
|
|
1260
|
+
process.send({ type: MSG_RELOAD_REQUEST, reloadId: openReload.id });
|
|
1142
1261
|
// Arm the rollback self-heal: restartController already latched
|
|
1143
1262
|
// pending/restarting + paused admission (in _restart, before we got here).
|
|
1144
1263
|
// On a SUCCESSFUL reload the supervisor sends MSG_RELEASE (→ releaseBaton
|
|
1145
1264
|
// AndDrain clears this) then we exit. On a ROLLBACK we're never released and
|
|
1146
1265
|
// never exit — so if this fires, resume serving instead of 503-ing forever.
|
|
1266
|
+
// Backstop only: the supervisor normally answers with MSG_ROLLED_BACK, which
|
|
1267
|
+
// settles the rollback at once (settleRolledBackReload). This fires when no
|
|
1268
|
+
// answer came — a supervisor that predates the notice, or one that is stuck.
|
|
1147
1269
|
if (reloadWatchdog) clearTimeout(reloadWatchdog);
|
|
1270
|
+
// Any notice while this reload is open settles it at once (MSG_ROLLED_BACK
|
|
1271
|
+
// handler), so firing here means no answer came at all.
|
|
1148
1272
|
reloadWatchdog = setTimeout(() => {
|
|
1149
1273
|
reloadWatchdog = null;
|
|
1150
|
-
|
|
1151
|
-
if (restartController?.cancelRestart()) {
|
|
1152
|
-
console.log('[Maxpool] Reload rolled back (new worker never took over).');
|
|
1153
|
-
}
|
|
1154
|
-
// FALL BACK TO A COLD RESTART — but ONLY when a newer build is actually on disk
|
|
1155
|
-
// waiting to be picked up. Resuming the old worker is what made "press u → c"
|
|
1156
|
-
// look like nothing happened: the graceful swap failed under machine load and
|
|
1157
|
-
// the update was abandoned, forever. Gating on a real pending update preserves
|
|
1158
|
-
// the safety property that a plain failed reload leaves a HEALTHY worker serving
|
|
1159
|
-
// (never kill a working process for nothing), while an update still lands.
|
|
1160
|
-
// One attempt per reload request, so a build that cannot boot can't crash-loop.
|
|
1161
|
-
// Cold-restart ONLY when the new build was merely SLOW to signal ready (a
|
|
1162
|
-
// loaded machine — the reported case). If it reported failure or died before
|
|
1163
|
-
// ready, the build cannot boot: restarting into it would leave the user with no
|
|
1164
|
-
// working proxy at all, strictly worse than the bug being fixed. Absent reason
|
|
1165
|
-
// (older supervisor) is treated as unsafe.
|
|
1166
|
-
const swapWasSlowNotBroken = lastRollbackReason === 'timeout';
|
|
1167
|
-
if (reloadIsForUpdate && swapWasSlowNotBroken && !coldFallbackUsed) {
|
|
1168
|
-
coldFallbackUsed = true;
|
|
1169
|
-
console.log('[Maxpool] Applying the update with a full restart instead — one moment.');
|
|
1170
|
-
restartWorkerNow();
|
|
1171
|
-
} else if (reloadIsForUpdate && !swapWasSlowNotBroken) {
|
|
1172
|
-
console.log('[Maxpool] The new version failed to start — resumed serving on the current version. Update NOT applied.');
|
|
1173
|
-
} else {
|
|
1174
|
-
console.log('[Maxpool] Rollback complete — resumed serving on the current version.');
|
|
1175
|
-
}
|
|
1274
|
+
settleRolledBackReload({ answered: false });
|
|
1176
1275
|
}, RELOAD_ROLLBACK_SELFHEAL_MS);
|
|
1177
1276
|
reloadWatchdog.unref?.();
|
|
1178
1277
|
return;
|
|
@@ -1203,7 +1302,11 @@ async function serverWorkerCommand() {
|
|
|
1203
1302
|
// the server comes back" is provable without allocating a pty. Gated — never
|
|
1204
1303
|
// active in normal runs.
|
|
1205
1304
|
if (process.env.MAXPOOL_TEST_RESTART_SIGNAL === '1') {
|
|
1206
|
-
process.on('SIGUSR2', () =>
|
|
1305
|
+
process.on('SIGUSR2', () => {
|
|
1306
|
+
if (testHookFires('MAXPOOL_TEST_SIGNAL_COLD_RESTART')) { restartWorkerNow(); return; } // test seam: the abrupt restart
|
|
1307
|
+
if (testHookFires('MAXPOOL_TEST_RELOAD_IS_UPDATE')) reloadIsForUpdate = true; // test seam: an update-driven reload
|
|
1308
|
+
restartController?.requestRestart();
|
|
1309
|
+
});
|
|
1207
1310
|
}
|
|
1208
1311
|
|
|
1209
1312
|
const shutdownGracefully = (reason, options = {}) => {
|
|
@@ -1221,7 +1324,14 @@ async function serverWorkerCommand() {
|
|
|
1221
1324
|
draining = true;
|
|
1222
1325
|
if (syncTimer) clearInterval(syncTimer);
|
|
1223
1326
|
if (updateTimer) clearInterval(updateTimer);
|
|
1224
|
-
|
|
1327
|
+
terminalMode.close();
|
|
1328
|
+
// Supervised, the supervisor's own exit restores the terminal (exit sequence + the
|
|
1329
|
+
// mode it started with) right after ours, so this worker never waits for the
|
|
1330
|
+
// terminal on its way out: no mode switch (TCSADRAIN) and no blocking exit write —
|
|
1331
|
+
// a quit (SIGTERM included) with the terminal not reading hung there (red-team
|
|
1332
|
+
// round 2). Unsupervised, nothing else restores the user's shell.
|
|
1333
|
+
if (supervised) terminalHandedOff = true;
|
|
1334
|
+
if (tui?.running) tui.stop(supervised ? { keepRaw: true } : {});
|
|
1225
1335
|
// Best-effort final flush + settle writes (fire-and-forget; the drain timeout
|
|
1226
1336
|
// below bounds total quit time, and write barriers protect the on-disk state).
|
|
1227
1337
|
(async () => {
|
|
@@ -1336,6 +1446,7 @@ async function serverWorkerCommand() {
|
|
|
1336
1446
|
shutdownGracefully('quit');
|
|
1337
1447
|
},
|
|
1338
1448
|
onRestart: () => {
|
|
1449
|
+
if (testHookFires('MAXPOOL_TEST_RELOAD_IS_UPDATE')) reloadIsForUpdate = true; // test seam: an update-driven reload
|
|
1339
1450
|
restartController.requestRestart();
|
|
1340
1451
|
},
|
|
1341
1452
|
});
|
|
@@ -1455,13 +1566,27 @@ async function serverWorkerCommand() {
|
|
|
1455
1566
|
// the update check. `viaTakeover` true means we acquired the socket through the
|
|
1456
1567
|
// baton (a reload) — freeze the update check so a reload doesn't re-probe npm
|
|
1457
1568
|
// (only a cold supervisor start self-updates).
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1569
|
+
// Take the terminal: lift the reload-worker boot muzzle so the TUI's first render +
|
|
1570
|
+
// any plain-mode fallback logs reach stdout (no-op for a worker that never muzzled),
|
|
1571
|
+
// then start the dashboard. On a TAKEOVER raw mode is switched on a helper thread
|
|
1572
|
+
// (terminal-mode.js): the old worker's frames may fill the terminal, this worker is
|
|
1573
|
+
// serving, and switching it blocks the switching thread while the terminal is behind
|
|
1574
|
+
// (2026-11-13: the takeover froze inside tui.start() and the port went dead; then a
|
|
1575
|
+
// frame small enough to look "taken" still stalled it ~1.7s on a bursty terminal). A cold
|
|
1576
|
+
// start keeps the synchronous switch, made before its first frame: with nothing of
|
|
1577
|
+
// ours queued yet it waits only for what the terminal already holds, and a deferred
|
|
1578
|
+
// switch there left a busy terminal's keyboard dead (tui-terminal-pty (b)/(c)).
|
|
1579
|
+
const takeTerminal = ({ viaTakeover }) => {
|
|
1462
1580
|
setConsoleStdoutSuppressed(false);
|
|
1581
|
+
terminalHandedOff = false; // ours now: our exit restores it
|
|
1582
|
+
// A RESPAWNED worker (the supervisor restarted one before it) also switches off its
|
|
1583
|
+
// serving thread: the worker before it may have left the terminal behind — an
|
|
1584
|
+
// update's deferred cold restart fires once our queue empties, while the kernel's
|
|
1585
|
+
// still holds a burst's worth (red-team round 2). Only the first launch keeps the
|
|
1586
|
+
// synchronous switch.
|
|
1587
|
+
const offThread = viaTakeover || isRespawnedWorker;
|
|
1463
1588
|
if (tui) {
|
|
1464
|
-
if (tui.start()) {
|
|
1589
|
+
if (tui.start(offThread ? { rawWhenDrained: true, modeSwitcher: terminalMode } : {})) {
|
|
1465
1590
|
console.log(`Listening on ${host}:${port} with ${accounts.length} account(s)`);
|
|
1466
1591
|
} else {
|
|
1467
1592
|
tui = null;
|
|
@@ -1470,6 +1595,13 @@ async function serverWorkerCommand() {
|
|
|
1470
1595
|
} else {
|
|
1471
1596
|
logPlainServerStart({ host, port, accounts, threshold, config });
|
|
1472
1597
|
}
|
|
1598
|
+
};
|
|
1599
|
+
|
|
1600
|
+
const becomePrimary = async ({ viaTakeover }) => {
|
|
1601
|
+
// Cold start: the terminal first, as always (nothing else to hand over). A reload
|
|
1602
|
+
// TAKEOVER serves and announces primacy FIRST and takes the terminal last (below):
|
|
1603
|
+
// whatever the terminal does, the baton completes and the port keeps answering.
|
|
1604
|
+
if (!viaTakeover) takeTerminal({ viaTakeover });
|
|
1473
1605
|
// On a reload TAKEOVER, adopt the latest tokens from disk. The headless
|
|
1474
1606
|
// reload worker loaded config at spawn time; the OLD worker may have rotated
|
|
1475
1607
|
// a single-use refresh token DURING the handoff (and flushed it before
|
|
@@ -1492,6 +1624,8 @@ async function serverWorkerCommand() {
|
|
|
1492
1624
|
// Tell the supervisor we are the sole acceptor now → it stops its own accept
|
|
1493
1625
|
// loop (the steady-state race fix). Cold start AND takeover both signal this.
|
|
1494
1626
|
if (supervised) { try { process.send({ type: MSG_PRIMARY }); } catch { /* ignore */ } }
|
|
1627
|
+
// Same tick as the send: no supervisor message (MSG_TTY_REASSERT) can land between.
|
|
1628
|
+
if (viaTakeover) takeTerminal({ viaTakeover });
|
|
1495
1629
|
|
|
1496
1630
|
// Reload-storm guard: only a cold (non-reload) lease holder probes npm.
|
|
1497
1631
|
// A reload-spawned/takeover worker must NEVER re-probe (1x not 2x traffic).
|
|
@@ -1526,15 +1660,37 @@ async function serverWorkerCommand() {
|
|
|
1526
1660
|
const releaseBatonAndDrain = async () => {
|
|
1527
1661
|
if (draining) { try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ } return; }
|
|
1528
1662
|
draining = true;
|
|
1529
|
-
// Terminal hand-off, part 1 (red-team S-2): stop drawing and taking keys NOW
|
|
1530
|
-
// let a slow terminal catch up WHILE steps 1-4 run — they can take many seconds,
|
|
1531
|
-
// and the supervisor waits only so long for MSG_RELEASED.
|
|
1663
|
+
// Terminal hand-off, part 1 (red-team S-2): stop drawing and taking keys NOW.
|
|
1532
1664
|
const terminalCatchUp = tui?.running ? tui.beginHandOff({ timeoutMs: 1500 }) : null;
|
|
1533
1665
|
// MSG_RELEASE arrived → the new worker booted OK and this reload is really
|
|
1534
1666
|
// proceeding (not a rollback). Disarm the rollback self-heal.
|
|
1535
1667
|
if (reloadWatchdog) { clearTimeout(reloadWatchdog); reloadWatchdog = null; }
|
|
1536
1668
|
if (syncTimer) clearInterval(syncTimer);
|
|
1537
1669
|
if (updateTimer) clearInterval(updateTimer);
|
|
1670
|
+
|
|
1671
|
+
// Terminal hand-off, part 2 (r2 B2, 2026-11-12): let a slow terminal catch up,
|
|
1672
|
+
// then the exit sequence, each bounded (1.5s) — BEFORE releasing: the new worker
|
|
1673
|
+
// paints right after MSG_RELEASED, and its first frame must not interleave with
|
|
1674
|
+
// ours. Done while this worker STILL ACCEPTS: once it stops, only the supervisor's
|
|
1675
|
+
// re-armed listener (after MSG_RELEASED) takes connections, and a terminal wait
|
|
1676
|
+
// in between refused every connection for ~3s (red-team round 1, 2026-11-13).
|
|
1677
|
+
// Muzzle our remaining logs first, so none print once stop() gives console.log
|
|
1678
|
+
// back to plain stdout. Never turns blocking back on.
|
|
1679
|
+
setConsoleStdoutSuppressed(true);
|
|
1680
|
+
if (terminalCatchUp) {
|
|
1681
|
+
await terminalCatchUp;
|
|
1682
|
+
appendEventLog(`[Maxpool] Reload: handing the terminal to the new worker (${process.stdout.writableLength || 0} bytes not yet shown)`);
|
|
1683
|
+
// keepRaw: the incoming worker puts the terminal in raw mode itself, and
|
|
1684
|
+
// switching it here would freeze this worker — still streaming in-flight
|
|
1685
|
+
// responses — for as long as the terminal is behind (TCSADRAIN; see TUI.stop).
|
|
1686
|
+
await tui.handOff({ timeoutMs: 1500, flushing: terminalCatchUp, keepRaw: true });
|
|
1687
|
+
}
|
|
1688
|
+
// The terminal is now HANDED OFF: the incoming worker will take the TTY at
|
|
1689
|
+
// MSG_TAKEOVER. Arm the guard UNCONDITIONALLY (even if the TUI was already
|
|
1690
|
+
// stopped) so this worker's later exit(0) restoreTerminal can't clobber the
|
|
1691
|
+
// new worker's TUI.
|
|
1692
|
+
terminalHandedOff = true;
|
|
1693
|
+
|
|
1538
1694
|
// Stop accepting NEW connections; KEEP in-flight requests alive.
|
|
1539
1695
|
server.maxpoolBeginDrain?.();
|
|
1540
1696
|
server.close(() => {});
|
|
@@ -1560,28 +1716,6 @@ async function serverWorkerCommand() {
|
|
|
1560
1716
|
// off (M3 — otherwise the new worker boots from the invalidated token).
|
|
1561
1717
|
await flushConfigWrites();
|
|
1562
1718
|
await flushStateWrites();
|
|
1563
|
-
|
|
1564
|
-
// Muzzle our own remaining logs (in-flight requests completing) BEFORE the
|
|
1565
|
-
// hand-off's stop() gives console.log back to plain stdout, so none of them
|
|
1566
|
-
// print on the user's screen or paint the new worker's alt-screen.
|
|
1567
|
-
setConsoleStdoutSuppressed(true);
|
|
1568
|
-
// Terminal hand-off, part 2 (r2 B2, 2026-11-12): finish the catch-up started at
|
|
1569
|
-
// the top, then the exit sequence, each bounded (1.5s) — BEFORE releasing: the
|
|
1570
|
-
// new worker paints right after MSG_RELEASED, and its first frame must not
|
|
1571
|
-
// interleave with ours. Never turns blocking back on; keeps serving in-flight.
|
|
1572
|
-
if (terminalCatchUp) {
|
|
1573
|
-
await terminalCatchUp;
|
|
1574
|
-
appendEventLog(`[Maxpool] Reload: handing the terminal to the new worker (${process.stdout.writableLength || 0} bytes not yet shown)`);
|
|
1575
|
-
// keepRaw: the incoming worker puts the terminal in raw mode itself, and
|
|
1576
|
-
// switching it here would freeze this worker — still streaming in-flight
|
|
1577
|
-
// responses — for as long as the terminal is behind (TCSADRAIN; see TUI.stop).
|
|
1578
|
-
await tui.handOff({ timeoutMs: 1500, flushing: terminalCatchUp, keepRaw: true });
|
|
1579
|
-
}
|
|
1580
|
-
// The terminal is now HANDED OFF: the incoming worker will take the TTY at
|
|
1581
|
-
// MSG_TAKEOVER. Arm the guard UNCONDITIONALLY (even if the TUI was already
|
|
1582
|
-
// stopped) so this worker's later exit(0) restoreTerminal can't clobber the
|
|
1583
|
-
// new worker's TUI.
|
|
1584
|
-
terminalHandedOff = true;
|
|
1585
1719
|
try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ }
|
|
1586
1720
|
|
|
1587
1721
|
// (macOS) releaseLease() above dropped the wake-lock, but we still stream our
|
|
@@ -1601,6 +1735,7 @@ async function serverWorkerCommand() {
|
|
|
1601
1735
|
// state file. Bounded (1s) and best-effort: losing the delta is a measurement
|
|
1602
1736
|
// gap, hanging the released worker is an operational fault, so the timeout wins.
|
|
1603
1737
|
const exitAfterMergeFlush = () => {
|
|
1738
|
+
terminalMode.close(); // a switch still waiting for the terminal never holds the exit
|
|
1604
1739
|
Promise.race([mergeFlushCapacityDelta(), delay(1000)])
|
|
1605
1740
|
.catch(() => {})
|
|
1606
1741
|
.finally(() => process.exit(0));
|
|
@@ -1629,6 +1764,7 @@ async function serverWorkerCommand() {
|
|
|
1629
1764
|
// a cold worker a dead/absent socket, fail LOUD and exit-75 so it respawns —
|
|
1630
1765
|
// never sit alive-but-not-primary (the blank-screen `r`-restart hang).
|
|
1631
1766
|
console.error('[Maxpool] Cold worker received MSG_LISTEN with no socket handle — exiting for respawn.');
|
|
1767
|
+
terminalHandedOff = true; // supervised (this branch is): never wait for the terminal on the way out
|
|
1632
1768
|
process.exit(SERVER_RESTART_EXIT_CODE);
|
|
1633
1769
|
} else if (msg?.type === MSG_CONN) {
|
|
1634
1770
|
// A connection the supervisor's own accept loop won the race for (cold start
|
|
@@ -1642,6 +1778,16 @@ async function serverWorkerCommand() {
|
|
|
1642
1778
|
// supervisor's rollback (old worker stays primary, zero disruption).
|
|
1643
1779
|
if (process.env.MAXPOOL_TEST_FAIL_RELOAD_WORKER === '1') {
|
|
1644
1780
|
process.send({ type: MSG_FAILED, reason: 'test-forced failure' });
|
|
1781
|
+
} else if (testHookFires('MAXPOOL_TEST_RELOAD_EXIT_BEFORE_READY')) {
|
|
1782
|
+
// Test seam: a new build that dies before it reports ready.
|
|
1783
|
+
process.exit(1);
|
|
1784
|
+
} else if (testHookFires('MAXPOOL_TEST_RELOAD_NEVER_READY')) {
|
|
1785
|
+
// Test seam: a new build too slow to ever report ready → the supervisor's
|
|
1786
|
+
// readiness timeout ('timeout' rollback).
|
|
1787
|
+
} else if (Number(process.env.MAXPOOL_TEST_RELOAD_READY_DELAY_MS) > 0) {
|
|
1788
|
+
// Test seam: a new build that reports ready only after a delay.
|
|
1789
|
+
setTimeout(() => { try { process.send({ type: MSG_READY }); } catch { /* gone */ } },
|
|
1790
|
+
Number(process.env.MAXPOOL_TEST_RELOAD_READY_DELAY_MS));
|
|
1645
1791
|
} else {
|
|
1646
1792
|
process.send({ type: MSG_READY });
|
|
1647
1793
|
}
|
|
@@ -1653,9 +1799,12 @@ async function serverWorkerCommand() {
|
|
|
1653
1799
|
await becomePrimary({ viaTakeover: true });
|
|
1654
1800
|
} else if (msg?.type === MSG_TTY_REASSERT) {
|
|
1655
1801
|
// The old worker has now EXITED, and its exit reset the shared terminal out
|
|
1656
|
-
// of raw mode (libuv's uv_tty_reset_mode on process exit). Reclaim it
|
|
1657
|
-
//
|
|
1658
|
-
|
|
1802
|
+
// of raw mode (libuv's uv_tty_reset_mode on process exit). Reclaim it on the
|
|
1803
|
+
// helper thread, so a terminal that is behind never stalls this worker;
|
|
1804
|
+
// without a helper, reclaimRawMode (waits for our queue to drain first).
|
|
1805
|
+
const r = await terminalMode.setRaw(true, { toggle: true });
|
|
1806
|
+
if (r.ok) appendEventLog(`[Maxpool] reload: terminal reclaimed in ${r.ms}ms (helper thread)`);
|
|
1807
|
+
else await reclaimRawModeLogged({ flush: tui?._nb ? o => tui.flushOutput(o) : null });
|
|
1659
1808
|
// Re-assert the screen state too: the dying worker's alt-screen exit and
|
|
1660
1809
|
// cursor-show can land after ours.
|
|
1661
1810
|
if (tui?.running && process.stdout.isTTY) {
|
|
@@ -1665,8 +1814,19 @@ async function serverWorkerCommand() {
|
|
|
1665
1814
|
} catch { /* ignore */ }
|
|
1666
1815
|
}
|
|
1667
1816
|
} else if (msg?.type === MSG_ROLLED_BACK) {
|
|
1668
|
-
//
|
|
1669
|
-
|
|
1817
|
+
// The supervisor gave up on the new worker; we were never released. Settle now
|
|
1818
|
+
// (resume admission, maybe cold-restart for an update) — not at the self-heal —
|
|
1819
|
+
// when the notice names the reload that is open: a late notice for an earlier
|
|
1820
|
+
// reload once settled the next one and cold-restarted mid-baton (red-team
|
|
1821
|
+
// 2026-11-13). An id-less notice comes from a supervisor that predates ids (it
|
|
1822
|
+
// survives seamless updates): it settles the open reload at once too — the
|
|
1823
|
+
// ~50s of 503s until the self-heal were the reported symptom — but never
|
|
1824
|
+
// authorizes a cold restart (red-team round 2).
|
|
1825
|
+
const action = rollbackNoticeAction({ noticeId: msg.reloadId, currentId: openReload?.id ?? null, open: Boolean(reloadWatchdog) });
|
|
1826
|
+
if (action === 'settle') settleRolledBackReload({ answered: true, reason: msg.reason || null });
|
|
1827
|
+
else if (action === 'settle-idless') settleRolledBackReload({ answered: true, reason: msg.reason || null, idless: true });
|
|
1828
|
+
else if (msg.reloadId != null) appendEventLog(`[Maxpool] Ignored a rollback notice for an earlier reload (${msg.reloadId}).`);
|
|
1829
|
+
else appendEventLog('[Maxpool] Ignored a rollback notice with no reload open.');
|
|
1670
1830
|
} else if (msg?.type === MSG_RELEASE) {
|
|
1671
1831
|
// Baton release: stop accepting NEW (keep in-flight), retire keep-alive
|
|
1672
1832
|
// sockets with Connection: close, stop writing, flush once, drop TUI.
|
|
@@ -1679,10 +1839,18 @@ async function serverWorkerCommand() {
|
|
|
1679
1839
|
// (closes the silent-non-primary hang class). A RELOAD worker mid-baton must
|
|
1680
1840
|
// NOT exit (that kills the supervisor loop) — report MSG_FAILED so the
|
|
1681
1841
|
// supervisor rolls back and the old worker stays primary (zero disruption).
|
|
1682
|
-
if (!isReloadWorker) process.exit(SERVER_RESTART_EXIT_CODE);
|
|
1842
|
+
if (!isReloadWorker) { terminalHandedOff = true; process.exit(SERVER_RESTART_EXIT_CODE); } // supervised: as above
|
|
1683
1843
|
try { process.send({ type: MSG_FAILED, reason: err.message }); } catch { /* ignore */ }
|
|
1684
1844
|
}
|
|
1685
1845
|
});
|
|
1846
|
+
// Test seam (inert unless set): SIGUSR2 feeds the handler above a message, so a pty
|
|
1847
|
+
// test can drive its exit-75 paths for real. 'listen-no-handle' = MSG_LISTEN with no
|
|
1848
|
+
// socket; 'listen' = MSG_LISTEN with a handle, which a serving worker refuses
|
|
1849
|
+
// (ERR_SERVER_ALREADY_LISTEN) → the handler's error path.
|
|
1850
|
+
const ipcSignal = process.env.MAXPOOL_TEST_IPC_SIGNAL;
|
|
1851
|
+
if (ipcSignal === 'listen-no-handle' || ipcSignal === 'listen') {
|
|
1852
|
+
process.on('SIGUSR2', () => process.emit('message', { type: MSG_LISTEN }, ipcSignal === 'listen' ? {} : undefined));
|
|
1853
|
+
}
|
|
1686
1854
|
} else {
|
|
1687
1855
|
// Direct (non-TTY service) path: bind the port ourselves, exactly as before.
|
|
1688
1856
|
const onListenError = err => handleServerListenError(err, host, port);
|
|
@@ -2556,17 +2724,25 @@ function argValue(flag) {
|
|
|
2556
2724
|
}
|
|
2557
2725
|
|
|
2558
2726
|
function handleServerListenError(err, host, port) {
|
|
2727
|
+
let lines;
|
|
2559
2728
|
if (err.code === 'EADDRINUSE') {
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2729
|
+
lines = [
|
|
2730
|
+
`[Maxpool] ${host}:${port} is already in use.`,
|
|
2731
|
+
'Another Maxpool proxy may already be running.',
|
|
2732
|
+
'Check the existing server with: maxpool status',
|
|
2733
|
+
`Find the listener with: lsof -nP -iTCP:${port} -sTCP:LISTEN`,
|
|
2734
|
+
];
|
|
2564
2735
|
} else if (err.code === 'EACCES') {
|
|
2565
|
-
|
|
2566
|
-
|
|
2736
|
+
lines = [
|
|
2737
|
+
`[Maxpool] Permission denied while listening on ${host}:${port}.`,
|
|
2738
|
+
'Choose a non-privileged port in the Maxpool config.',
|
|
2739
|
+
];
|
|
2567
2740
|
} else {
|
|
2568
|
-
|
|
2741
|
+
lines = [`[Maxpool] Failed to listen on ${host}:${port}: ${err.message}`];
|
|
2569
2742
|
}
|
|
2743
|
+
// The supervisor's terminal may be non-blocking by now (a respawn's bind): print
|
|
2744
|
+
// synchronously, or the reason is dropped at exit.
|
|
2745
|
+
printFatal(lines);
|
|
2570
2746
|
process.exit(1);
|
|
2571
2747
|
}
|
|
2572
2748
|
|
package/src/reload-protocol.js
CHANGED
|
@@ -68,21 +68,33 @@ export async function runReloadBaton({
|
|
|
68
68
|
releaseTimeoutMs = 20_000,
|
|
69
69
|
takeoverTimeoutMs = 10_000,
|
|
70
70
|
log = () => {},
|
|
71
|
+
reloadId,
|
|
71
72
|
}) {
|
|
73
|
+
// Every rollback notice names the reload it is about, so the old worker can tell a
|
|
74
|
+
// late notice for an EARLIER reload from one for the reload it has open.
|
|
75
|
+
const rolledBack = reason => {
|
|
76
|
+
try { oldWorker.send({ type: MSG_ROLLED_BACK, reason, reloadId }); } catch { /* old worker gone */ }
|
|
77
|
+
};
|
|
72
78
|
// (a) Confirm the freshly-spawned headless worker booted the new code.
|
|
73
79
|
let ready;
|
|
74
80
|
try {
|
|
75
81
|
newWorker.send({ type: MSG_PROBE_READY });
|
|
76
82
|
ready = await newWorker.waitFor([MSG_READY, MSG_FAILED], readyTimeoutMs);
|
|
77
83
|
} catch (err) {
|
|
84
|
+
// Tell the old worker BEFORE logging: it resumes serving on this notice, and a log
|
|
85
|
+
// line must never stand between the two (2026-11-13: a blocked terminal write did).
|
|
86
|
+
// Only an expired timer is 'timeout' (merely slow: a cold restart may retry it). A
|
|
87
|
+
// new worker that exited or could not be spawned is 'failed' — as is any rejection
|
|
88
|
+
// whose cause is unknown, since a cold restart into a build that cannot boot leaves
|
|
89
|
+
// no working proxy.
|
|
90
|
+
rolledBack(err?.reason === 'timeout' ? 'timeout' : 'failed');
|
|
78
91
|
log(`reload: readiness wait failed (${err.message}); rolling back`);
|
|
79
|
-
try { oldWorker.send({ type: MSG_ROLLED_BACK, reason: 'timeout' }); } catch { /* old worker gone */ }
|
|
80
92
|
return RELOAD_ROLLED_BACK;
|
|
81
93
|
}
|
|
82
94
|
// (b) New worker did not come up cleanly → roll back fully, old stays primary.
|
|
83
95
|
if (!ready || ready.type === MSG_FAILED) {
|
|
96
|
+
rolledBack('failed');
|
|
84
97
|
log(`reload: new worker reported failed (${ready?.reason || 'no ready'}); rolling back`);
|
|
85
|
-
try { oldWorker.send({ type: MSG_ROLLED_BACK, reason: 'failed' }); } catch { /* old worker gone */ }
|
|
86
98
|
return RELOAD_ROLLED_BACK;
|
|
87
99
|
}
|
|
88
100
|
|
|
@@ -128,6 +140,58 @@ export async function runReloadBaton({
|
|
|
128
140
|
return RELOAD_SWAPPED;
|
|
129
141
|
}
|
|
130
142
|
|
|
143
|
+
/**
|
|
144
|
+
* Wrap a child process in a small IPC channel: `.send(msg[,handle])` and an
|
|
145
|
+
* async `.waitFor([types], timeoutMs)` that resolves with the first matching
|
|
146
|
+
* message, or rejects on timeout / premature child exit.
|
|
147
|
+
*/
|
|
148
|
+
export function makeWorkerChannel(child) {
|
|
149
|
+
return {
|
|
150
|
+
child,
|
|
151
|
+
send(msg, handle) {
|
|
152
|
+
try {
|
|
153
|
+
if (handle) child.send(msg, handle);
|
|
154
|
+
else child.send(msg);
|
|
155
|
+
} catch { /* IPC may be torn down mid-swap; baton timeouts cover it */ }
|
|
156
|
+
},
|
|
157
|
+
// A rejection carries `.reason`: 'timeout' (the timer expired), 'exited' (the child
|
|
158
|
+
// exited first — or had already exited) or 'spawn-error'.
|
|
159
|
+
waitFor(types, timeoutMs) {
|
|
160
|
+
const want = Array.isArray(types) ? types : [types];
|
|
161
|
+
const fail = (reason, message) => Object.assign(new Error(message), { reason });
|
|
162
|
+
if (child.exitCode != null || child.signalCode != null) {
|
|
163
|
+
return Promise.reject(fail('exited', `worker already exited (code ${child.exitCode}, signal ${child.signalCode}) before ${want.join('/')}`));
|
|
164
|
+
}
|
|
165
|
+
return new Promise((resolve, reject) => {
|
|
166
|
+
const cleanup = () => {
|
|
167
|
+
clearTimeout(timer);
|
|
168
|
+
child.removeListener('message', onMsg);
|
|
169
|
+
child.removeListener('exit', onExit);
|
|
170
|
+
child.removeListener('error', onError);
|
|
171
|
+
};
|
|
172
|
+
const onMsg = msg => {
|
|
173
|
+
if (msg && want.includes(msg.type)) { cleanup(); resolve(msg); }
|
|
174
|
+
};
|
|
175
|
+
const onExit = (code, signal) => {
|
|
176
|
+
cleanup();
|
|
177
|
+
reject(fail('exited', `worker exited (code ${code}, signal ${signal}) before ${want.join('/')}`));
|
|
178
|
+
};
|
|
179
|
+
// A spawn failure (EMFILE/EAGAIN) surfaces as 'error', not 'exit' — the
|
|
180
|
+
// baton must see it as a failed step (→ ROLLED_BACK/FALLBACK), not hang.
|
|
181
|
+
const onError = err => { cleanup(); reject(fail('spawn-error', `worker spawn error before ${want.join('/')}: ${err.message}`)); };
|
|
182
|
+
const timer = setTimeout(() => {
|
|
183
|
+
cleanup();
|
|
184
|
+
reject(fail('timeout', `timed out waiting for ${want.join('/')}`));
|
|
185
|
+
}, timeoutMs);
|
|
186
|
+
timer.unref?.();
|
|
187
|
+
child.on('message', onMsg);
|
|
188
|
+
child.once('exit', onExit);
|
|
189
|
+
child.once('error', onError);
|
|
190
|
+
});
|
|
191
|
+
},
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
131
195
|
// Sent to the NEW primary worker once the OLD worker has actually exited. Node calls
|
|
132
196
|
// libuv's uv_tty_reset_mode() on process exit, which restores the termios the dying
|
|
133
197
|
// process saved when IT first enabled raw mode — clobbering the live worker's terminal
|
|
@@ -141,3 +205,23 @@ export function describeOldWorkerExit({ swappedAt, exitedAt, killed = false }) {
|
|
|
141
205
|
const after = ((exitedAt - swappedAt) / 1000).toFixed(1);
|
|
142
206
|
return `reload: old worker exited at ${new Date(exitedAt).toISOString()} (${after}s after swap, ${killed ? 'SIGKILL after the drain cap' : 'clean exit'})`;
|
|
143
207
|
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* What the old worker does with a rollback notice. `noticeId` is the reload id the
|
|
211
|
+
* supervisor echoed (absent from a supervisor that predates ids), `currentId` the id
|
|
212
|
+
* of the reload this worker requested last, `open` whether that reload is still
|
|
213
|
+
* waiting for an outcome.
|
|
214
|
+
* 'settle' — the notice names the open reload: act on it now
|
|
215
|
+
* 'settle-idless' — an id-less notice (a supervisor that predates ids — it survives
|
|
216
|
+
* seamless updates) while a reload is open: resume serving now, but
|
|
217
|
+
* its reason never authorizes a cold restart. That supervisor tags
|
|
218
|
+
* every readiness rejection 'timeout', a new build that died before
|
|
219
|
+
* ready included, and cannot say which reload it means (red-team
|
|
220
|
+
* round 2, 2026-11-13)
|
|
221
|
+
* 'ignore' — a notice about an earlier reload, or nothing is open
|
|
222
|
+
*/
|
|
223
|
+
export function rollbackNoticeAction({ noticeId, currentId, open }) {
|
|
224
|
+
if (!open) return 'ignore';
|
|
225
|
+
if (noticeId == null) return 'settle-idless';
|
|
226
|
+
return noticeId === currentId ? 'settle' : 'ignore';
|
|
227
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// Switch the terminal's raw mode WITHOUT stalling the thread that serves requests.
|
|
2
|
+
//
|
|
3
|
+
// Node switches it with tcsetattr(TCSADRAIN), which waits until the terminal has read
|
|
4
|
+
// everything already queued for it — up to the pty's ~1 KB kernel queue, which no Node
|
|
5
|
+
// API can see. A worker that has just taken over the pool cannot know whether its
|
|
6
|
+
// terminal is reading (red-team 2026-11-13: a dashboard frame under 1 KB, or a terminal
|
|
7
|
+
// that reads in bursts, stalled the new sole acceptor for ~1.7s inside setRawMode).
|
|
8
|
+
// So that switch runs in a helper PROCESS with its own handle on the terminal (our
|
|
9
|
+
// stdin, else /dev/tty): the helper waits for the terminal, the pool keeps serving.
|
|
10
|
+
// termios belongs to the terminal device, so a mode set from the helper is the mode
|
|
11
|
+
// every reader of the terminal sees (verified on a real pty, 2026-11-13).
|
|
12
|
+
//
|
|
13
|
+
// A process, not a worker thread (red-team round 2, 2026-11-13): process.exit JOINS
|
|
14
|
+
// worker threads, a thread inside tcsetattr cannot be interrupted (Worker.terminate()
|
|
15
|
+
// included — libuv retries the call on EINTR), so a worker with a switch pending lived
|
|
16
|
+
// until the terminal read: on the abrupt-restart path it kept the port. A child process
|
|
17
|
+
// is never joined, and SIGKILL ends it mid-wait without applying the pending mode.
|
|
18
|
+
// The helper only ever dies by SIGKILL: a normal exit would run Node's terminal reset
|
|
19
|
+
// and put the terminal back in the mode it found before its first switch.
|
|
20
|
+
import { spawn } from 'node:child_process';
|
|
21
|
+
|
|
22
|
+
const HELPER_SRC = `
|
|
23
|
+
const fs = require('node:fs');
|
|
24
|
+
const tty = require('node:tty');
|
|
25
|
+
const die = () => process.kill(process.pid, 'SIGKILL');
|
|
26
|
+
process.on('disconnect', die); // our parent is gone: never apply a stale mode
|
|
27
|
+
process.on('uncaughtException', die);
|
|
28
|
+
let term = null;
|
|
29
|
+
// fd 3 is the parent's stdin (the terminal the dashboard takes keys from — it need not
|
|
30
|
+
// be the controlling terminal), else the controlling terminal.
|
|
31
|
+
function openTerminal() {
|
|
32
|
+
if (tty.isatty(3)) return new tty.ReadStream(3);
|
|
33
|
+
let fd = null;
|
|
34
|
+
try {
|
|
35
|
+
fd = fs.openSync('/dev/tty', fs.constants.O_RDWR | fs.constants.O_NOCTTY);
|
|
36
|
+
if (tty.isatty(fd)) return new tty.ReadStream(fd);
|
|
37
|
+
} catch (err) {
|
|
38
|
+
if (fd != null) { try { fs.closeSync(fd); } catch {} }
|
|
39
|
+
throw err;
|
|
40
|
+
}
|
|
41
|
+
try { fs.closeSync(fd); } catch {}
|
|
42
|
+
throw Object.assign(new Error('/dev/tty is not a terminal'), { code: 'ENOTTY' });
|
|
43
|
+
}
|
|
44
|
+
process.on('message', ({ id, raw, toggle }) => {
|
|
45
|
+
try {
|
|
46
|
+
// Never read from it (no resume): the parent owns the keyboard.
|
|
47
|
+
if (!term) term = openTerminal();
|
|
48
|
+
// libuv skips a switch to the mode it believes is already set; another process
|
|
49
|
+
// may have changed the terminal since, so 'toggle' goes through the other mode.
|
|
50
|
+
if (toggle) term.setRawMode(!raw);
|
|
51
|
+
term.setRawMode(raw);
|
|
52
|
+
process.send({ id, ok: true });
|
|
53
|
+
} catch (err) {
|
|
54
|
+
process.send({ id, ok: false, error: err.code || err.message });
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
`;
|
|
58
|
+
|
|
59
|
+
const spawnHelper = () => spawn(process.execPath, ['-e', HELPER_SRC], {
|
|
60
|
+
// fd 3 = our stdin; nothing on 0-2, so the helper's own exit never touches them.
|
|
61
|
+
stdio: ['ignore', 'ignore', 'ignore', 0, 'ipc'],
|
|
62
|
+
env: { PATH: process.env.PATH || '' },
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* `setRaw(raw, { toggle })` resolves {ok, error?, ms} once the terminal is in that mode
|
|
67
|
+
* — which may be long after the call while the terminal is not reading; the caller's
|
|
68
|
+
* thread is never blocked. ok:false (no terminal, helper failed) means the caller must
|
|
69
|
+
* fall back to its own switch. The helper starts on demand and is killed once it has
|
|
70
|
+
* nothing pending; close() kills it at once — a pending switch then never lands — and
|
|
71
|
+
* so does this process's exit. `makeHelper` is injectable for tests.
|
|
72
|
+
*/
|
|
73
|
+
export function createTerminalModeSwitcher({ makeHelper = spawnHelper, now = Date.now } = {}) {
|
|
74
|
+
let helper = null;
|
|
75
|
+
let seq = 0;
|
|
76
|
+
const pending = new Map();
|
|
77
|
+
const failAll = error => {
|
|
78
|
+
for (const done of pending.values()) done({ ok: false, error });
|
|
79
|
+
pending.clear();
|
|
80
|
+
};
|
|
81
|
+
const kill = h => { try { h?.kill?.('SIGKILL'); } catch { /* already gone */ } };
|
|
82
|
+
const onProcessExit = () => kill(helper);
|
|
83
|
+
let exitHooked = false;
|
|
84
|
+
const ensure = () => {
|
|
85
|
+
if (helper) return helper;
|
|
86
|
+
const h = makeHelper();
|
|
87
|
+
h.unref?.(); // never keeps this process alive
|
|
88
|
+
h.channel?.unref?.();
|
|
89
|
+
h.on('message', ({ id, ok, error }) => {
|
|
90
|
+
const done = pending.get(id);
|
|
91
|
+
if (!done) return;
|
|
92
|
+
pending.delete(id);
|
|
93
|
+
done({ ok, error });
|
|
94
|
+
// Idle: nothing left for it to switch. No helper process lingers between switches.
|
|
95
|
+
if (!pending.size && helper === h) { helper = null; kill(h); }
|
|
96
|
+
});
|
|
97
|
+
h.on('error', err => { if (helper === h) { helper = null; failAll(err.message); } kill(h); });
|
|
98
|
+
h.on('exit', () => { if (helper === h) { helper = null; failAll('terminal helper exited'); } });
|
|
99
|
+
if (!exitHooked) { process.on('exit', onProcessExit); exitHooked = true; }
|
|
100
|
+
helper = h;
|
|
101
|
+
return h;
|
|
102
|
+
};
|
|
103
|
+
return {
|
|
104
|
+
setRaw(raw, { toggle = false } = {}) {
|
|
105
|
+
const t0 = now();
|
|
106
|
+
return new Promise(resolve => {
|
|
107
|
+
const done = r => resolve({ ...r, ms: now() - t0 });
|
|
108
|
+
let h;
|
|
109
|
+
try { h = ensure(); } catch (err) { done({ ok: false, error: err.message }); return; }
|
|
110
|
+
const id = ++seq;
|
|
111
|
+
pending.set(id, done);
|
|
112
|
+
try { h.send({ id, raw: Boolean(raw), toggle: Boolean(toggle) }); } catch (err) { pending.delete(id); done({ ok: false, error: err.message }); }
|
|
113
|
+
});
|
|
114
|
+
},
|
|
115
|
+
/** Kill the helper NOW (exit paths call this first): a pending switch never lands. */
|
|
116
|
+
close() {
|
|
117
|
+
const h = helper;
|
|
118
|
+
helper = null;
|
|
119
|
+
failAll('closed');
|
|
120
|
+
kill(h);
|
|
121
|
+
if (exitHooked) { process.removeListener('exit', onProcessExit); exitHooked = false; }
|
|
122
|
+
},
|
|
123
|
+
};
|
|
124
|
+
}
|
package/src/tui.js
CHANGED
|
@@ -56,19 +56,67 @@ export function writeTerminalSync(s) {
|
|
|
56
56
|
* toggle waits until our own queued output has drained — the remaining wait is the
|
|
57
57
|
* kernel's ~1 KB — and a terminal that is not reading at all gets it on 'drain'.
|
|
58
58
|
* Resolves true when reclaimed now, false when deferred to 'drain'.
|
|
59
|
+
* 'drain' is emitted only after a write crossed the stream's high-water mark, so a
|
|
60
|
+
* deferred reclaim also polls the queue length and goes ahead once it is empty
|
|
61
|
+
* (2026-11-13: a frame below 16 KB never produced 'drain'). Fallback only: a worker
|
|
62
|
+
* with a terminal helper (terminal-mode.js) never switches on its own thread. `guard` is re-checked
|
|
63
|
+
* at reclaim time (a TUI stopped meanwhile must not be put back in raw mode);
|
|
64
|
+
* `onDeferredReclaim` runs once the deferred reclaim actually happened.
|
|
59
65
|
*/
|
|
60
|
-
export async function reclaimRawMode({
|
|
66
|
+
export async function reclaimRawMode({
|
|
67
|
+
stdin = process.stdin, stdout = process.stdout, flush, timeoutMs = 1500,
|
|
68
|
+
guard = () => true, onDeferredReclaim = () => {}, pollMs = 100,
|
|
69
|
+
} = {}) {
|
|
61
70
|
if (!stdin.isTTY) return true;
|
|
62
71
|
// OFF then ON: Node short-circuits setRawMode(true) while it believes isRaw is already
|
|
63
72
|
// true, and the reset happened in ANOTHER process (verified on a real pty 2026-08-07).
|
|
64
73
|
const reclaim = () => {
|
|
74
|
+
if (!guard()) return;
|
|
65
75
|
try { stdin.setRawMode(false); stdin.setRawMode(true); stdin.resume(); } catch { /* terminal gone */ }
|
|
66
76
|
};
|
|
67
77
|
if (!flush || await flush({ timeoutMs })) { reclaim(); return true; }
|
|
68
|
-
|
|
78
|
+
let settled = false;
|
|
79
|
+
const later = () => {
|
|
80
|
+
if (settled) return;
|
|
81
|
+
settled = true;
|
|
82
|
+
clearInterval(poll);
|
|
83
|
+
stdout.removeListener?.('drain', later);
|
|
84
|
+
reclaim();
|
|
85
|
+
try { onDeferredReclaim(); } catch { /* never */ }
|
|
86
|
+
};
|
|
87
|
+
// Only once our queue is EMPTY (polled, or 'drain'): a mere shrink — one whole write
|
|
88
|
+
// landed — fired the switch on the serving thread while a bursty terminal still held
|
|
89
|
+
// several bursts of ours, stalling the pool for each (red-team round 2). Our queue
|
|
90
|
+
// cannot grow without bound meanwhile: the dashboard stops writing at the high-water
|
|
91
|
+
// mark and owes one frame on 'drain', so a busy dashboard still reaches 'drain'.
|
|
92
|
+
const poll = setInterval(() => { if (stdout.writableLength === 0) later(); }, pollMs);
|
|
93
|
+
poll.unref?.();
|
|
94
|
+
stdout.once('drain', later);
|
|
69
95
|
return false;
|
|
70
96
|
}
|
|
71
97
|
|
|
98
|
+
/**
|
|
99
|
+
* Call `cb` once, as soon as the terminal has taken everything queued for it (our
|
|
100
|
+
* write queue is empty) — polled, since 'drain' fires only after a write crossed the
|
|
101
|
+
* high-water mark. Returns a cancel function.
|
|
102
|
+
*/
|
|
103
|
+
export function whenTerminalCaughtUp(cb, { stdout = process.stdout, pollMs = 100 } = {}) {
|
|
104
|
+
let done = false;
|
|
105
|
+
const finish = () => {
|
|
106
|
+
if (done) return;
|
|
107
|
+
done = true;
|
|
108
|
+
clearInterval(poll);
|
|
109
|
+
stdout.removeListener?.('drain', onDrain);
|
|
110
|
+
cb();
|
|
111
|
+
};
|
|
112
|
+
const onDrain = () => { if (!stdout.writableLength) finish(); };
|
|
113
|
+
const poll = setInterval(onDrain, pollMs);
|
|
114
|
+
poll.unref?.();
|
|
115
|
+
stdout.on?.('drain', onDrain);
|
|
116
|
+
if (!stdout.writableLength) Promise.resolve().then(finish);
|
|
117
|
+
return () => { if (done) return; done = true; clearInterval(poll); stdout.removeListener?.('drain', onDrain); };
|
|
118
|
+
}
|
|
119
|
+
|
|
72
120
|
/**
|
|
73
121
|
* reclaimRawMode + the update-window measurement (release B, B11): says how long the new
|
|
74
122
|
* worker took to take the terminal back, or that it was deferred to the terminal's drain
|
|
@@ -76,13 +124,12 @@ export async function reclaimRawMode({ stdin = process.stdin, stdout = process.s
|
|
|
76
124
|
*/
|
|
77
125
|
export async function reclaimRawModeLogged({ stdout = process.stdout, log = m => console.log(`[Maxpool] ${m}`), now = Date.now, ...rest } = {}) {
|
|
78
126
|
const t0 = now();
|
|
79
|
-
const done = await reclaimRawMode({
|
|
80
|
-
|
|
81
|
-
log(`reload: terminal reclaimed in ${now() - t0}ms`)
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
}
|
|
127
|
+
const done = await reclaimRawMode({
|
|
128
|
+
stdout, ...rest,
|
|
129
|
+
onDeferredReclaim: () => log(`reload: terminal reclaimed after drain in ${now() - t0}ms`),
|
|
130
|
+
});
|
|
131
|
+
if (done) log(`reload: terminal reclaimed in ${now() - t0}ms`);
|
|
132
|
+
else log('reload: terminal reclaim deferred to drain');
|
|
86
133
|
return done;
|
|
87
134
|
}
|
|
88
135
|
|
|
@@ -596,12 +643,28 @@ export class TUI {
|
|
|
596
643
|
|
|
597
644
|
// ── lifecycle ──────────────────────────────────────
|
|
598
645
|
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
646
|
+
/** `rawWhenDrained` (a worker that is already serving — reload takeover, a respawn):
|
|
647
|
+
* enter raw mode only once the terminal has taken our first frame. setRawMode is
|
|
648
|
+
* tcsetattr(TCSADRAIN) and blocks the WHOLE process until the terminal has read
|
|
649
|
+
* everything queued; on a terminal that is not reading that is forever, and the port
|
|
650
|
+
* goes dead with it (2026-11-13). So the dashboard is drawn non-blocking first and the
|
|
651
|
+
* mode switch waits for the terminal (reclaimRawMode) — the keyboard, not the serving,
|
|
652
|
+
* is what a stalled terminal delays. A terminal that took the frame at once gets raw
|
|
653
|
+
* mode synchronously, so a broken one still falls back to plain logs (returns false). */
|
|
654
|
+
start({ rawWhenDrained = false, modeSwitcher = null } = {}) {
|
|
655
|
+
const rawError = err => `[Maxpool] TUI unavailable (${err.code || err.message}); continuing with plain logs.\n`;
|
|
656
|
+
if (rawWhenDrained) {
|
|
657
|
+
if (!process.stdin.isTTY || typeof process.stdin.setRawMode !== 'function') {
|
|
658
|
+
process.stderr.write(rawError({ message: 'stdin is not a terminal' }));
|
|
659
|
+
return false;
|
|
660
|
+
}
|
|
661
|
+
} else {
|
|
662
|
+
try {
|
|
663
|
+
process.stdin.setRawMode(true);
|
|
664
|
+
} catch (err) {
|
|
665
|
+
process.stderr.write(rawError(err));
|
|
666
|
+
return false;
|
|
667
|
+
}
|
|
605
668
|
}
|
|
606
669
|
|
|
607
670
|
this.running = true;
|
|
@@ -632,6 +695,34 @@ export class TUI {
|
|
|
632
695
|
this.frame = (this.frame + 1) % SPINNER.length;
|
|
633
696
|
this.requestRender();
|
|
634
697
|
}, 500);
|
|
698
|
+
if (rawWhenDrained && modeSwitcher) {
|
|
699
|
+
// Off this thread (terminal-mode.js): the pool keeps serving however long the
|
|
700
|
+
// terminal takes to read. Without a helper (no controlling terminal) fall back.
|
|
701
|
+
modeSwitcher.setRaw(true).then(r => {
|
|
702
|
+
if (r.ok) {
|
|
703
|
+
if (r.ms >= 100) appendEventLog(`[Maxpool] TUI: keyboard ready after the terminal caught up (${r.ms}ms)`);
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
if (!this.running) return;
|
|
707
|
+
appendEventLog(`[Maxpool] TUI: terminal helper unavailable (${r.error}); keyboard waits for the terminal to read`);
|
|
708
|
+
reclaimRawMode({ flush: o => this.flushOutput(o), guard: () => this.running }).catch(() => {});
|
|
709
|
+
}).catch(() => {});
|
|
710
|
+
} else if (rawWhenDrained) {
|
|
711
|
+
if (!process.stdout.writableLength) {
|
|
712
|
+
// The terminal took the whole frame: it is reading, so the drain wait is short.
|
|
713
|
+
try {
|
|
714
|
+
process.stdin.setRawMode(true);
|
|
715
|
+
} catch (err) {
|
|
716
|
+
this.stop({ keepRaw: true });
|
|
717
|
+
console.error(rawError(err).trim());
|
|
718
|
+
return false;
|
|
719
|
+
}
|
|
720
|
+
} else {
|
|
721
|
+
reclaimRawMode({ flush: o => this.flushOutput(o), guard: () => this.running })
|
|
722
|
+
.then(now => { if (!now) appendEventLog('[Maxpool] TUI: keyboard waits for the terminal to read (raw mode deferred to drain)'); })
|
|
723
|
+
.catch(() => {});
|
|
724
|
+
}
|
|
725
|
+
}
|
|
635
726
|
return true;
|
|
636
727
|
}
|
|
637
728
|
|