maxpool 1.24.90 → 1.24.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +24 -3
- package/src/index.js +53 -28
- package/src/server.js +13 -2
- package/src/tui.js +192 -29
package/package.json
CHANGED
package/src/account-manager.js
CHANGED
|
@@ -1118,6 +1118,11 @@ export class AccountManager {
|
|
|
1118
1118
|
* recorded (failedRequests, lastError) but never extends the
|
|
1119
1119
|
* consecutiveFailures streak, so it cannot trip the hold-release
|
|
1120
1120
|
* gate for the account's other conversations.
|
|
1121
|
+
* outcome.linkFault — a LINK-transport fault (link_ttfb / link_idle /
|
|
1122
|
+
* pool_link_unreachable): the Mac↔cloud link gave up, not the
|
|
1123
|
+
* account (2026-11-12: cloud rows showed 10-26 "failed" while the
|
|
1124
|
+
* cloud itself failed ≈6 of ~2,100). Records a link drop load
|
|
1125
|
+
* event and returns — no failedRequests, no streak, no error text.
|
|
1121
1126
|
*/
|
|
1122
1127
|
releaseAccount(lease, outcome = {}) {
|
|
1123
1128
|
if (!lease?.account) return;
|
|
@@ -1146,6 +1151,14 @@ export class AccountManager {
|
|
|
1146
1151
|
|
|
1147
1152
|
if (outcome.neutral) return;
|
|
1148
1153
|
|
|
1154
|
+
// A link fault is not the account failing: tally it as a link drop (kept out of
|
|
1155
|
+
// avgMs too — a 120s give-up is not the account's latency) and stop here, before
|
|
1156
|
+
// every failure counter / error field.
|
|
1157
|
+
if (outcome.linkFault) {
|
|
1158
|
+
this._recordLoadEvent(account, lease, { success: false, linkFault: true });
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1149
1162
|
// A per-model weekly cap is NOT an account failure — the account is healthy for
|
|
1150
1163
|
// every other model. Don't poison its failure counters / scoring penalty (mirror
|
|
1151
1164
|
// the network-blip carve-out); the scoped bench is already set in markRateLimited.
|
|
@@ -1198,6 +1211,7 @@ export class AccountManager {
|
|
|
1198
1211
|
durationMs: Math.max(0, now - lease.startedAt),
|
|
1199
1212
|
weight: Math.max(1, lease.weight || 1),
|
|
1200
1213
|
success: Boolean(outcome.success),
|
|
1214
|
+
linkFault: Boolean(outcome.linkFault),
|
|
1201
1215
|
status: outcome.status || null,
|
|
1202
1216
|
});
|
|
1203
1217
|
this._pruneLoadEvents(account, now);
|
|
@@ -1216,14 +1230,21 @@ export class AccountManager {
|
|
|
1216
1230
|
const since = now - windowMs;
|
|
1217
1231
|
const events = (account.loadEvents || []).filter(e => e.at >= since);
|
|
1218
1232
|
const requests = events.length;
|
|
1219
|
-
|
|
1233
|
+
// "failed" is what the operator reads as the ACCOUNT misbehaving; a link fault is
|
|
1234
|
+
// the pool's own tunnel to the team pool and gets its own field (2026-11-12).
|
|
1235
|
+
const failed = events.filter(e => !e.success && !e.linkFault).length;
|
|
1236
|
+
const linkFailed = events.filter(e => !e.success && e.linkFault).length;
|
|
1237
|
+
// avgMs excludes link faults (N1, pre-mortem r2): a 120s give-up on the link is not
|
|
1238
|
+
// this account's response time.
|
|
1239
|
+
const timed = events.filter(e => !e.linkFault);
|
|
1220
1240
|
const weight = events.reduce((sum, e) => sum + (e.weight || 1), 0);
|
|
1221
|
-
const durationMs =
|
|
1241
|
+
const durationMs = timed.reduce((sum, e) => sum + (e.durationMs || 0), 0);
|
|
1222
1242
|
return {
|
|
1223
1243
|
requests,
|
|
1224
1244
|
failed,
|
|
1245
|
+
linkFailed,
|
|
1225
1246
|
weight,
|
|
1226
|
-
avgMs:
|
|
1247
|
+
avgMs: timed.length ? Math.round(durationMs / timed.length) : null,
|
|
1227
1248
|
};
|
|
1228
1249
|
}
|
|
1229
1250
|
|
package/src/index.js
CHANGED
|
@@ -42,7 +42,7 @@ net.setDefaultAutoSelectFamilyAttemptTimeout(
|
|
|
42
42
|
import { Prober } from './prober.js';
|
|
43
43
|
import { CapacityLedger } from './capacity-ledger.js';
|
|
44
44
|
import { loginOAuth, fetchProfile, refreshAccessToken, isTokenExpiringSoon, tokenFingerprint, isLoginCancelled } from './oauth.js';
|
|
45
|
-
import { TUI } from './tui.js';
|
|
45
|
+
import { TUI, writeTerminalSync, reclaimRawMode } from './tui.js';
|
|
46
46
|
import { RestartController } from './restart-controller.js';
|
|
47
47
|
import { resolveAccounts } from './account-config.js';
|
|
48
48
|
import { maybeCheckForUpdate, getCurrentVersion, markApplied, clearQuarantine, captureBootVersion } from './updater.js';
|
|
@@ -314,7 +314,7 @@ async function supervisorCommand() {
|
|
|
314
314
|
// (exit alt-screen + show cursor + raw off) so the user's shell is clean.
|
|
315
315
|
const restoreTerminalFromSupervisor = () => {
|
|
316
316
|
try {
|
|
317
|
-
if (process.stdout.isTTY)
|
|
317
|
+
if (process.stdout.isTTY) writeTerminalSync('\x1b[?25h\x1b[?1049l');
|
|
318
318
|
if (process.stdin.isTTY) { try { process.stdin.setRawMode(false); } catch { /* ignore */ } }
|
|
319
319
|
} catch { /* never throw */ }
|
|
320
320
|
};
|
|
@@ -829,7 +829,10 @@ async function serverWorkerCommand() {
|
|
|
829
829
|
// by spawned integration tests so a startup probe can't race their refresh/rotation
|
|
830
830
|
// assertions (mirrors MAXPOOL_DISABLE_SLEEP_GUARD).
|
|
831
831
|
const probeSeconds = process.env.MAXPOOL_DISABLE_QUOTA_PROBE === '1' ? 0 : (config.quotaProbeSeconds || 0);
|
|
832
|
-
|
|
832
|
+
// Late-bound: routes probe lines through the TUI once it owns the console (a TUI
|
|
833
|
+
// render must be the only writer to the terminal, 2026-11-12).
|
|
834
|
+
const uiLog = (...a) => console.log(...a);
|
|
835
|
+
const prober = new Prober(accountManager, { intervalMs: probeSeconds * 1000, log: uiLog });
|
|
833
836
|
// Tell the AM the probe cadence so the TUI can flag a scoped/provider tag whose
|
|
834
837
|
// background probe has gone stale (> 3× interval since last success).
|
|
835
838
|
accountManager.quotaProbeIntervalMs = probeSeconds * 1000;
|
|
@@ -979,26 +982,39 @@ async function serverWorkerCommand() {
|
|
|
979
982
|
// Best-effort terminal restore on ANY abnormal exit path (uncaughtException,
|
|
980
983
|
// a bare process.exit, a crash) so the user's shell is never left in raw mode
|
|
981
984
|
// or the alt-screen. Idempotent and safe even when no TUI was running.
|
|
982
|
-
|
|
985
|
+
// `exiting`: only the process-exit path may turn blocking writes back on (r2 B1,
|
|
986
|
+
// 2026-11-12) — a TUI-owned stdout is NON-BLOCKING and its queued bytes are dropped
|
|
987
|
+
// at exit, so there the exit sequence goes out synchronously or the shell stays in
|
|
988
|
+
// the dashboard. A process that keeps running (a crash that does not exit) must
|
|
989
|
+
// stay non-blocking, or a terminal that stops reading freezes it again (red-team S-1).
|
|
990
|
+
const restoreTerminal = ({ exiting }) => {
|
|
983
991
|
// Handed the TTY to a new worker (seamless TUI reload) → the new worker owns
|
|
984
992
|
// it now; restoring here would clobber its live TUI. No-op.
|
|
985
993
|
if (terminalHandedOff) return;
|
|
986
994
|
try {
|
|
987
|
-
if (tui?.running) { tui.stop(); return; }
|
|
988
|
-
if (process.stdout.isTTY)
|
|
995
|
+
if (tui?.running) { tui.stop({ exiting }); return; }
|
|
996
|
+
if (process.stdout.isTTY) {
|
|
997
|
+
if (exiting) writeTerminalSync('\x1b[?25h\x1b[?1049l');
|
|
998
|
+
else process.stdout.write('\x1b[?25h\x1b[?1049l');
|
|
999
|
+
}
|
|
989
1000
|
if (process.stdin.isTTY) { try { process.stdin.setRawMode(false); } catch { /* ignore */ } }
|
|
990
1001
|
} catch { /* never throw from a restore */ }
|
|
991
1002
|
};
|
|
992
|
-
process.on('exit', restoreTerminal);
|
|
1003
|
+
process.on('exit', () => restoreTerminal({ exiting: true }));
|
|
1004
|
+
// Test seam: SIGUSR2 throws from a timer, so a pty test can crash a worker for real.
|
|
1005
|
+
if (process.env.MAXPOOL_TEST_CRASH_SIGNAL === '1') {
|
|
1006
|
+
process.on('SIGUSR2', () => setImmediate(() => { throw new Error('maxpool test crash (MAXPOOL_TEST_CRASH_SIGNAL)'); }));
|
|
1007
|
+
}
|
|
993
1008
|
process.on('uncaughtException', err => {
|
|
994
1009
|
console.error(`[Maxpool] Worker uncaughtException: ${err?.stack || err}`);
|
|
995
|
-
restoreTerminal();
|
|
996
1010
|
// A reload worker must NEVER exit(1) (escapes the supervisor exit-75 loop).
|
|
997
1011
|
// Stay alive so the supervisor's baton timeouts roll us back cleanly. Also
|
|
998
1012
|
// skip the exit-75 restart while DRAINING: a shutdown-time write to a dead
|
|
999
1013
|
// terminal can throw here, and restarting then would respawn the very orphan
|
|
1000
1014
|
// a terminal-close shutdown is trying to avoid — let the drain finish instead.
|
|
1015
|
+
// Exiting → the 'exit' listener restores the terminal (synchronously, once).
|
|
1001
1016
|
if (!isReloadWorker && !draining) process.exit(SERVER_RESTART_EXIT_CODE);
|
|
1017
|
+
restoreTerminal({ exiting: false });
|
|
1002
1018
|
});
|
|
1003
1019
|
// Swallow EPIPE/EIO from writes to a hung-up terminal so they can't crash us.
|
|
1004
1020
|
process.stdout.on('error', () => {});
|
|
@@ -1077,7 +1093,7 @@ async function serverWorkerCommand() {
|
|
|
1077
1093
|
draining = true;
|
|
1078
1094
|
if (syncTimer) clearInterval(syncTimer);
|
|
1079
1095
|
if (updateTimer) clearInterval(updateTimer);
|
|
1080
|
-
if (tui?.running) { tui.stop(); }
|
|
1096
|
+
if (tui?.running) { tui.stop(); } // plain-log mode resumes for the drain messages
|
|
1081
1097
|
console.log('\n[Maxpool] Restarting server now; queued requests will reconnect automatically.');
|
|
1082
1098
|
server.closeAllConnections?.();
|
|
1083
1099
|
// Best-effort: flush quota + settle in-flight refreshes/prober before exit so
|
|
@@ -1162,6 +1178,7 @@ async function serverWorkerCommand() {
|
|
|
1162
1178
|
pauseAdmission: () => accountManager.setAdmissionPaused(true),
|
|
1163
1179
|
resumeAdmission: () => accountManager.setAdmissionPaused(false),
|
|
1164
1180
|
restartNow: requestReload,
|
|
1181
|
+
log: uiLog, // via the TUI's console, not straight to the terminal (T6)
|
|
1165
1182
|
// The pre-drain only earns its cost on the COLD path (the socket closes, so an
|
|
1166
1183
|
// in-flight request would be severed). Seamless keeps serving them post-baton.
|
|
1167
1184
|
isSeamless: () => {
|
|
@@ -1502,6 +1519,10 @@ async function serverWorkerCommand() {
|
|
|
1502
1519
|
const releaseBatonAndDrain = async () => {
|
|
1503
1520
|
if (draining) { try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ } return; }
|
|
1504
1521
|
draining = true;
|
|
1522
|
+
// Terminal hand-off, part 1 (red-team S-2): stop drawing and taking keys NOW and
|
|
1523
|
+
// let a slow terminal catch up WHILE steps 1-4 run — they can take many seconds,
|
|
1524
|
+
// and the supervisor waits only so long for MSG_RELEASED.
|
|
1525
|
+
const terminalCatchUp = tui?.running ? tui.beginHandOff({ timeoutMs: 1500 }) : null;
|
|
1505
1526
|
// MSG_RELEASE arrived → the new worker booted OK and this reload is really
|
|
1506
1527
|
// proceeding (not a rollback). Disarm the rollback self-heal.
|
|
1507
1528
|
if (reloadWatchdog) { clearTimeout(reloadWatchdog); reloadWatchdog = null; }
|
|
@@ -1533,14 +1554,27 @@ async function serverWorkerCommand() {
|
|
|
1533
1554
|
await flushConfigWrites();
|
|
1534
1555
|
await flushStateWrites();
|
|
1535
1556
|
|
|
1536
|
-
|
|
1557
|
+
// Muzzle our own remaining logs (in-flight requests completing) BEFORE the
|
|
1558
|
+
// hand-off's stop() gives console.log back to plain stdout, so none of them
|
|
1559
|
+
// print on the user's screen or paint the new worker's alt-screen.
|
|
1560
|
+
setConsoleStdoutSuppressed(true);
|
|
1561
|
+
// Terminal hand-off, part 2 (r2 B2, 2026-11-12): finish the catch-up started at
|
|
1562
|
+
// the top, then the exit sequence, each bounded (1.5s) — BEFORE releasing: the
|
|
1563
|
+
// new worker paints right after MSG_RELEASED, and its first frame must not
|
|
1564
|
+
// interleave with ours. Never turns blocking back on; keeps serving in-flight.
|
|
1565
|
+
if (terminalCatchUp) {
|
|
1566
|
+
await terminalCatchUp;
|
|
1567
|
+
appendEventLog(`[Maxpool] Reload: handing the terminal to the new worker (${process.stdout.writableLength || 0} bytes not yet shown)`);
|
|
1568
|
+
// keepRaw: the incoming worker puts the terminal in raw mode itself, and
|
|
1569
|
+
// switching it here would freeze this worker — still streaming in-flight
|
|
1570
|
+
// responses — for as long as the terminal is behind (TCSADRAIN; see TUI.stop).
|
|
1571
|
+
await tui.handOff({ timeoutMs: 1500, flushing: terminalCatchUp, keepRaw: true });
|
|
1572
|
+
}
|
|
1537
1573
|
// The terminal is now HANDED OFF: the incoming worker will take the TTY at
|
|
1538
1574
|
// MSG_TAKEOVER. Arm the guard UNCONDITIONALLY (even if the TUI was already
|
|
1539
1575
|
// stopped) so this worker's later exit(0) restoreTerminal can't clobber the
|
|
1540
|
-
// new worker's TUI.
|
|
1541
|
-
// requests completing) so they don't paint the new worker's alt-screen.
|
|
1576
|
+
// new worker's TUI.
|
|
1542
1577
|
terminalHandedOff = true;
|
|
1543
|
-
setConsoleStdoutSuppressed(true);
|
|
1544
1578
|
try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ }
|
|
1545
1579
|
|
|
1546
1580
|
// (macOS) releaseLease() above dropped the wake-lock, but we still stream our
|
|
@@ -1612,25 +1646,15 @@ async function serverWorkerCommand() {
|
|
|
1612
1646
|
await becomePrimary({ viaTakeover: true });
|
|
1613
1647
|
} else if (msg?.type === MSG_TTY_REASSERT) {
|
|
1614
1648
|
// The old worker has now EXITED, and its exit reset the shared terminal out
|
|
1615
|
-
// of raw mode (libuv's uv_tty_reset_mode on process exit). Reclaim it
|
|
1616
|
-
//
|
|
1617
|
-
|
|
1618
|
-
// when it believes isRaw is already true — and it does believe that, because
|
|
1619
|
-
// the clobber happened in ANOTHER process and left our flag untouched. A bare
|
|
1620
|
-
// setRawMode(true) here is a silent no-op (verified on a real pty 2026-08-07).
|
|
1621
|
-
if (process.stdin.isTTY) {
|
|
1622
|
-
try {
|
|
1623
|
-
process.stdin.setRawMode(false);
|
|
1624
|
-
process.stdin.setRawMode(true);
|
|
1625
|
-
process.stdin.resume();
|
|
1626
|
-
} catch { /* terminal gone — nothing to reclaim */ }
|
|
1627
|
-
}
|
|
1649
|
+
// of raw mode (libuv's uv_tty_reset_mode on process exit). Reclaim it —
|
|
1650
|
+
// without blocking on a terminal that is behind (reclaimRawMode).
|
|
1651
|
+
await reclaimRawMode({ flush: tui?._nb ? o => tui.flushOutput(o) : null });
|
|
1628
1652
|
// Re-assert the screen state too: the dying worker's alt-screen exit and
|
|
1629
1653
|
// cursor-show can land after ours.
|
|
1630
1654
|
if (tui?.running && process.stdout.isTTY) {
|
|
1631
1655
|
try {
|
|
1632
1656
|
process.stdout.write('\x1b[?1049h\x1b[?25l');
|
|
1633
|
-
tui.
|
|
1657
|
+
tui.forceRender();
|
|
1634
1658
|
} catch { /* ignore */ }
|
|
1635
1659
|
}
|
|
1636
1660
|
} else if (msg?.type === MSG_ROLLED_BACK) {
|
|
@@ -2550,5 +2574,6 @@ function formatDurationMs(ms) {
|
|
|
2550
2574
|
function formatLoadWindow(load = {}) {
|
|
2551
2575
|
const avg = load.avgMs != null ? ` avg ${formatDurationMs(load.avgMs)}` : '';
|
|
2552
2576
|
const failed = load.failed ? `, ${load.failed} failed` : '';
|
|
2553
|
-
|
|
2577
|
+
const linkFailed = load.linkFailed ? `, ${load.linkFailed} link drop${load.linkFailed === 1 ? '' : 's'}` : '';
|
|
2578
|
+
return `${load.requests || 0} req${avg}${failed}${linkFailed}`;
|
|
2554
2579
|
}
|
package/src/server.js
CHANGED
|
@@ -1145,9 +1145,20 @@ async function forwardRequest(
|
|
|
1145
1145
|
: err?.linkBody ? 'link_idle'
|
|
1146
1146
|
: err?.name === 'AbortError' ? 'link_ttfb'
|
|
1147
1147
|
: 'pool_link_unreachable';
|
|
1148
|
-
|
|
1148
|
+
// D4 split (2026-11-12): a client leaving after output started says nothing about
|
|
1149
|
+
// the link either — neutral. A LINK-transport fault is booked as a link drop, not
|
|
1150
|
+
// a failure of the cloud row behind it.
|
|
1151
|
+
if (linkClientGone) {
|
|
1152
|
+
finishLink({ success: false, neutral: true, error: linkKind });
|
|
1153
|
+
} else {
|
|
1154
|
+
finishLink({ status: 502, error: linkKind, scope: 'request', linkFault: true });
|
|
1155
|
+
}
|
|
1149
1156
|
excludedIndexes.add(account.index); // P4b: never re-pick a row that just failed
|
|
1150
|
-
const
|
|
1157
|
+
const streamTag = requestInfo.stream ? 'streaming' : 'non-streaming';
|
|
1158
|
+
// A string code (UND_ERR_SOCKET, ECONNRESET) names the fault; DOMException
|
|
1159
|
+
// codes are numbers (AbortError = 20), so fall back to the cause, then the name.
|
|
1160
|
+
const errCode = (typeof err?.code === 'string' && err.code) || err?.cause?.code || err?.name || 'unknown';
|
|
1161
|
+
const linkDetail = `${linkStatus == null ? 'no answer' : `HTTP ${linkStatus}`} after ${heldS}s (${err?.message || err}) [${errCode}, ${streamTag}]`;
|
|
1151
1162
|
if (!clientHasRealBytes(res, requestInfo)) {
|
|
1152
1163
|
if (retryCount + 1 < maxAttempts && hasEligibleRoute(accountManager, requestInfo, excludedIndexes)) {
|
|
1153
1164
|
logFailedAttempt(account.name, linkKind, linkDetail, 'moved');
|
package/src/tui.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createInterface } from 'node:readline';
|
|
2
|
-
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { readFileSync, writeSync } from 'node:fs';
|
|
3
|
+
import { isatty } from 'node:tty';
|
|
3
4
|
import { homedir } from 'node:os';
|
|
4
5
|
import { fetchProfile, loginOAuth, tokenFingerprint, isLoginCancelled } from './oauth.js';
|
|
5
6
|
import { appendEventLog, setConsoleStdoutSuppressed } from './event-log.js';
|
|
@@ -27,6 +28,47 @@ const cyan = s => fg(36, s);
|
|
|
27
28
|
const gray = s => fg(90, s);
|
|
28
29
|
const dimUnderline = s => `${ESC}2;4m${s}${RESET}`;
|
|
29
30
|
|
|
31
|
+
// Terminal output budget (2026-11-12: the pool froze for up to 60s inside a synchronous
|
|
32
|
+
// TTY write() — a full-screen redraw per log line, 4.9 GB in 12h49m). Frames are
|
|
33
|
+
// coalesced to one per RENDER_MIN_INTERVAL_MS (≤10/s), an unchanged frame is re-sent
|
|
34
|
+
// only after REPAINT_MS, and nothing is written while the terminal has not drained.
|
|
35
|
+
const RENDER_MIN_INTERVAL_MS = 100;
|
|
36
|
+
const REPAINT_MS = 5000;
|
|
37
|
+
const EXIT_SEQ = `${ESC}?25h${ESC}?1049l`; // show cursor + leave the alt screen
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Write to the terminal synchronously at PROCESS EXIT. The TUI turns its output
|
|
41
|
+
* non-blocking, and Node drops a non-blocking stream's queued bytes on exit — the exit
|
|
42
|
+
* sequence written through the stream never arrives and the shell is left in the
|
|
43
|
+
* dashboard (pre-mortem r2 B1, real-pty probe). Blocking back on + a direct fd write
|
|
44
|
+
* delivers it. Exit paths only: a blocking write is exactly what froze the pool.
|
|
45
|
+
*/
|
|
46
|
+
export function writeTerminalSync(s) {
|
|
47
|
+
if (!isatty(1)) return;
|
|
48
|
+
try { process.stdout._handle?.setBlocking?.(true); } catch { /* not a libuv tty */ }
|
|
49
|
+
try { writeSync(1, s); } catch { /* terminal gone */ }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Re-enter raw mode after another process reset the terminal (post-update reassert).
|
|
54
|
+
* setRawMode is tcsetattr(TCSADRAIN): it blocks the WHOLE process until the terminal
|
|
55
|
+
* has read everything already sent (2026-11-12, measured 2.8s on a paused pty). So the
|
|
56
|
+
* toggle waits until our own queued output has drained — the remaining wait is the
|
|
57
|
+
* kernel's ~1 KB — and a terminal that is not reading at all gets it on 'drain'.
|
|
58
|
+
* Resolves true when reclaimed now, false when deferred to 'drain'.
|
|
59
|
+
*/
|
|
60
|
+
export async function reclaimRawMode({ stdin = process.stdin, stdout = process.stdout, flush, timeoutMs = 1500 } = {}) {
|
|
61
|
+
if (!stdin.isTTY) return true;
|
|
62
|
+
// OFF then ON: Node short-circuits setRawMode(true) while it believes isRaw is already
|
|
63
|
+
// true, and the reset happened in ANOTHER process (verified on a real pty 2026-08-07).
|
|
64
|
+
const reclaim = () => {
|
|
65
|
+
try { stdin.setRawMode(false); stdin.setRawMode(true); stdin.resume(); } catch { /* terminal gone */ }
|
|
66
|
+
};
|
|
67
|
+
if (!flush || await flush({ timeoutMs })) { reclaim(); return true; }
|
|
68
|
+
stdout.once('drain', reclaim);
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
|
|
30
72
|
const ANSI_RE = /\x1b\[[0-9;]*m/g;
|
|
31
73
|
const strip = s => s.replace(ANSI_RE, '');
|
|
32
74
|
const vw = s => strip(s).length;
|
|
@@ -204,19 +246,24 @@ function countdown(ts) {
|
|
|
204
246
|
* "how long each request takes" is not something the owner acts on. An idle
|
|
205
247
|
* account renders NOTHING.
|
|
206
248
|
*
|
|
207
|
-
* 2 live · 17 req/hr · 1 failed
|
|
249
|
+
* 2 live · 17 req/hr · 1 failed · 3 link drops
|
|
208
250
|
* 17 req/hr working, idle this second
|
|
209
251
|
* (blank) resting — nothing to say
|
|
252
|
+
*
|
|
253
|
+
* "link drops" (2026-11-12): the link to a team pool gave up, not the account —
|
|
254
|
+
* yellow, never folded into "failed".
|
|
210
255
|
*/
|
|
211
256
|
function loadText(load) {
|
|
212
257
|
const inflight = load?.current?.inFlight || 0;
|
|
213
258
|
const hourReq = load?.last1h?.requests || 0;
|
|
214
259
|
const failed = load?.last15m?.failed || 0;
|
|
215
|
-
|
|
260
|
+
const linkFailed = load?.last15m?.linkFailed || 0;
|
|
261
|
+
if (!inflight && !hourReq && !failed && !linkFailed) return '';
|
|
216
262
|
const parts = [];
|
|
217
263
|
if (inflight) parts.push(`${inflight} live`);
|
|
218
264
|
if (hourReq) parts.push(`${hourReq} req/hr`);
|
|
219
265
|
if (failed) parts.push(red(`${failed} failed`));
|
|
266
|
+
if (linkFailed) parts.push(yellow(`${linkFailed} link drop${linkFailed === 1 ? '' : 's'}`));
|
|
220
267
|
return parts.join(' · ');
|
|
221
268
|
}
|
|
222
269
|
|
|
@@ -516,6 +563,18 @@ export class TUI {
|
|
|
516
563
|
this.timer = null;
|
|
517
564
|
this._origLog = null;
|
|
518
565
|
this._origErr = null;
|
|
566
|
+
// Terminal output state (2026-11-12 freeze fix): _nb = stdout made non-blocking by
|
|
567
|
+
// start(); _renderTimer = the one scheduled coalesced render; _lastFrame /
|
|
568
|
+
// _lastFullWriteAt = what the terminal last got and when; _forceNext = the next
|
|
569
|
+
// frame must be sent even if unchanged; _drainArmed = one 'drain' listener pending.
|
|
570
|
+
this._nb = false;
|
|
571
|
+
this._renderTimer = null;
|
|
572
|
+
this._lastRenderAt = 0;
|
|
573
|
+
this._lastFrame = null;
|
|
574
|
+
this._lastFullWriteAt = 0;
|
|
575
|
+
this._forceNext = false;
|
|
576
|
+
this._drainArmed = false;
|
|
577
|
+
this._onDrain = () => { this._drainArmed = false; if (this.running) this.render(); };
|
|
519
578
|
}
|
|
520
579
|
|
|
521
580
|
// ── lifecycle ──────────────────────────────────────
|
|
@@ -529,11 +588,19 @@ export class TUI {
|
|
|
529
588
|
}
|
|
530
589
|
|
|
531
590
|
this.running = true;
|
|
591
|
+
// Never block the pool on its own terminal (2026-11-12): Node writes a TTY
|
|
592
|
+
// synchronously, so a terminal that fell behind stopped EVERY request for up to 60s.
|
|
593
|
+
// Real TTYs only (r2 S3); a later stop() leaves it non-blocking (the exit path alone
|
|
594
|
+
// turns blocking back on — writeTerminalSync).
|
|
595
|
+
if (!this._nb && isatty(process.stdout.fd) && typeof process.stdout._handle?.setBlocking === 'function') {
|
|
596
|
+
process.stdout._handle.setBlocking(false);
|
|
597
|
+
this._nb = true;
|
|
598
|
+
}
|
|
532
599
|
process.stdout.write(`${ESC}?1049h${ESC}?25l`);
|
|
533
600
|
process.stdin.resume();
|
|
534
601
|
process.stdin.setEncoding('utf8');
|
|
535
602
|
this._dataHandler = d => this._onData(d);
|
|
536
|
-
this._resizeHandler = () => this.
|
|
603
|
+
this._resizeHandler = () => this.forceRender();
|
|
537
604
|
process.stdin.on('data', this._dataHandler);
|
|
538
605
|
process.stdout.on('resize', this._resizeHandler);
|
|
539
606
|
|
|
@@ -543,30 +610,84 @@ export class TUI {
|
|
|
543
610
|
console.log = (...a) => this._addLog(a.join(' '));
|
|
544
611
|
console.error = (...a) => this._addLog(a.join(' '));
|
|
545
612
|
|
|
546
|
-
this.
|
|
613
|
+
this.forceRender();
|
|
547
614
|
this.timer = setInterval(() => {
|
|
548
615
|
this.frame = (this.frame + 1) % SPINNER.length;
|
|
549
|
-
this.
|
|
616
|
+
this.requestRender();
|
|
550
617
|
}, 500);
|
|
551
618
|
return true;
|
|
552
619
|
}
|
|
553
620
|
|
|
554
|
-
|
|
621
|
+
/** Stop drawing frames (no more renders, no drain wake-up) and stop taking keys, so
|
|
622
|
+
* nothing typed during a hand-off wait acts; the screen stays as is. */
|
|
623
|
+
_haltRendering() {
|
|
555
624
|
this.running = false;
|
|
625
|
+
if (this._dataHandler) process.stdin.removeListener('data', this._dataHandler);
|
|
556
626
|
if (this.timer) { clearInterval(this.timer); this.timer = null; }
|
|
627
|
+
if (this._renderTimer) { clearTimeout(this._renderTimer); this._renderTimer = null; }
|
|
628
|
+
if (this._drainArmed) { process.stdout.removeListener('drain', this._onDrain); this._drainArmed = false; }
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
/** `exiting`: called from the process-exit path — the exit sequence must be written
|
|
632
|
+
* synchronously or it is lost with the non-blocking stream's queue (r2 B1).
|
|
633
|
+
* `keepRaw`: leave the terminal's raw mode as is. Node switches it with
|
|
634
|
+
* tcsetattr(TCSADRAIN), which BLOCKS the whole process until the terminal has read
|
|
635
|
+
* everything already sent (measured: 2.8s for a terminal that read nothing for 3s) —
|
|
636
|
+
* so a process that keeps serving after stop() must not switch it while the
|
|
637
|
+
* terminal may be behind. */
|
|
638
|
+
stop({ exiting = false, keepRaw = false } = {}) {
|
|
639
|
+
this._haltRendering();
|
|
557
640
|
if (this._origLog) { console.log = this._origLog; console.error = this._origErr; }
|
|
558
641
|
process.stdin.removeListener('data', this._dataHandler);
|
|
559
642
|
process.stdout.removeListener('resize', this._resizeHandler);
|
|
560
|
-
|
|
561
|
-
|
|
643
|
+
if (exiting) writeTerminalSync(EXIT_SEQ);
|
|
644
|
+
else process.stdout.write(EXIT_SEQ);
|
|
645
|
+
if (!keepRaw) { try { process.stdin.setRawMode(false); } catch {} }
|
|
562
646
|
process.stdin.pause();
|
|
563
647
|
}
|
|
564
648
|
|
|
649
|
+
/** Give the terminal up to someone else (a self-update's new worker, a login prompt)
|
|
650
|
+
* while this process keeps running (r2 B2): let the frames a slow terminal is still
|
|
651
|
+
* taking land, then the exit sequence, each bounded by timeoutMs — so the next
|
|
652
|
+
* writer never interleaves with our queue, and blocking is never turned back on. */
|
|
653
|
+
async handOff({ timeoutMs = 1500, flushing = null, keepRaw = false } = {}) {
|
|
654
|
+
await (flushing || this.beginHandOff({ timeoutMs }));
|
|
655
|
+
this.stop({ keepRaw });
|
|
656
|
+
await this.flushOutput({ timeoutMs });
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
/** First half of handOff, started early so the terminal catches up while the caller
|
|
660
|
+
* does other work: stop drawing and taking keys, and return the flush (pass it to
|
|
661
|
+
* handOff as `flushing`). Gate on that promise — `running` is already false. */
|
|
662
|
+
beginHandOff({ timeoutMs = 1500 } = {}) {
|
|
663
|
+
this._haltRendering();
|
|
664
|
+
return this.flushOutput({ timeoutMs });
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
/** Resolve once the terminal has taken everything queued for it: true when drained,
|
|
668
|
+
* false at timeoutMs. 'drain' fires only after a write hit the high-water mark, so
|
|
669
|
+
* the queue length is polled too. */
|
|
670
|
+
flushOutput({ timeoutMs = 1500 } = {}) {
|
|
671
|
+
const out = process.stdout;
|
|
672
|
+
if (!out.writableLength) return Promise.resolve(true);
|
|
673
|
+
return new Promise(resolve => {
|
|
674
|
+
const finish = drained => {
|
|
675
|
+
clearTimeout(deadline); clearInterval(poll);
|
|
676
|
+
out.removeListener('drain', onDrain);
|
|
677
|
+
resolve(drained);
|
|
678
|
+
};
|
|
679
|
+
const onDrain = () => { if (!out.writableLength) finish(true); };
|
|
680
|
+
const deadline = setTimeout(() => finish(false), timeoutMs);
|
|
681
|
+
const poll = setInterval(onDrain, 25);
|
|
682
|
+
out.on('drain', onDrain);
|
|
683
|
+
});
|
|
684
|
+
}
|
|
685
|
+
|
|
565
686
|
// ── server hooks ───────────────────────────────────
|
|
566
687
|
|
|
567
688
|
onRequestStart(id, info) {
|
|
568
689
|
this.active.set(id, { ...info, t: timestamp(), started: Date.now(), account: null, caller: info.caller || null, conversation: info.conversation || null });
|
|
569
|
-
this.
|
|
690
|
+
this.requestRender();
|
|
570
691
|
}
|
|
571
692
|
|
|
572
693
|
onRequestRouted(id, info) {
|
|
@@ -624,7 +745,8 @@ export class TUI {
|
|
|
624
745
|
appendEventLog(msg);
|
|
625
746
|
this.log.unshift({ t: timestamp(), msg });
|
|
626
747
|
if (this.log.length > 200) this.log.length = 200;
|
|
627
|
-
|
|
748
|
+
// Scheduled, not drawn: a burst of N lines is ONE frame (was N full screens).
|
|
749
|
+
this.requestRender();
|
|
628
750
|
}
|
|
629
751
|
|
|
630
752
|
// ── input handling ─────────────────────────────────
|
|
@@ -660,7 +782,7 @@ export class TUI {
|
|
|
660
782
|
.split('')
|
|
661
783
|
.filter(c => c >= ' ' && c !== '\x7f')
|
|
662
784
|
.join('');
|
|
663
|
-
if (typed) { this.inputBuf += typed; this.
|
|
785
|
+
if (typed) { this.inputBuf += typed; this.forceRender(); }
|
|
664
786
|
if (nlIdx !== -1) { this._pendingPaste = ''; return this._key('enter'); }
|
|
665
787
|
return;
|
|
666
788
|
}
|
|
@@ -684,7 +806,7 @@ export class TUI {
|
|
|
684
806
|
case 'input': this._keyInput(k); break;
|
|
685
807
|
case 'confirm': this._keyConfirm(k); break;
|
|
686
808
|
}
|
|
687
|
-
this.
|
|
809
|
+
this.forceRender();
|
|
688
810
|
}
|
|
689
811
|
|
|
690
812
|
_keyNormal(k) {
|
|
@@ -756,14 +878,14 @@ export class TUI {
|
|
|
756
878
|
if (!this.checkNow) { this._addLog('Update check unavailable on this worker'); return; }
|
|
757
879
|
if (this.updateBusy) return; // already running
|
|
758
880
|
this.updateBusy = 'Checking npm…';
|
|
759
|
-
this.
|
|
881
|
+
this.forceRender();
|
|
760
882
|
Promise.resolve(this.checkNow())
|
|
761
883
|
.catch(() => {})
|
|
762
884
|
.finally(() => {
|
|
763
885
|
// If an update was found+applied, the seamless reload replaces this worker and
|
|
764
886
|
// this never renders. Otherwise report the outcome in place.
|
|
765
887
|
this.updateBusy = null;
|
|
766
|
-
if (this.mode === 'updates') this.
|
|
888
|
+
if (this.mode === 'updates') this.forceRender();
|
|
767
889
|
});
|
|
768
890
|
} else if (k === 't') {
|
|
769
891
|
this._toggleAutoUpdate();
|
|
@@ -1000,7 +1122,7 @@ export class TUI {
|
|
|
1000
1122
|
const looksLikeSecretName = /^[A-Z][A-Z0-9_-]{2,60}$/.test(input) && !input.includes('.');
|
|
1001
1123
|
if (!looksLikeSecretName) return input; // a pasted key
|
|
1002
1124
|
this._addLog(`Resolving GCP secret "${input}"…`);
|
|
1003
|
-
this.
|
|
1125
|
+
this.forceRender();
|
|
1004
1126
|
try {
|
|
1005
1127
|
const { resolveSecret } = await import('../secret-resolver.js');
|
|
1006
1128
|
const v = await resolveSecret(input);
|
|
@@ -1102,7 +1224,7 @@ export class TUI {
|
|
|
1102
1224
|
this.mode = 'accounts';
|
|
1103
1225
|
if (secretName) this._addLog(`Resolving secret "${secretName}" from GCP…`);
|
|
1104
1226
|
else this._addLog(`Adding "${name}" with direct API key…`);
|
|
1105
|
-
this.
|
|
1227
|
+
this.forceRender();
|
|
1106
1228
|
try {
|
|
1107
1229
|
let token = null;
|
|
1108
1230
|
if (secretName) {
|
|
@@ -1504,14 +1626,16 @@ export class TUI {
|
|
|
1504
1626
|
// it's a RE-AUTH in place (revives a dead-refresh/reauth account), not a duplicate.
|
|
1505
1627
|
async _doLogin() {
|
|
1506
1628
|
const wasRunning = this.running;
|
|
1507
|
-
if (wasRunning) this.stop();
|
|
1508
1629
|
// Mute the console-log MIRROR to stdout during the interactive prompt so routing
|
|
1509
1630
|
// logs (console.log, e.g. the glm-fallback 429 churn) don't flood over the readline
|
|
1510
1631
|
// "Name this account" prompt. The prompts + errors use process.stdout.write, which
|
|
1511
1632
|
// BYPASSES the mirror, so they stay visible. Login runs only on the terminal-owning
|
|
1512
|
-
// primary (mirror baseline false), so the finally restores it to false.
|
|
1633
|
+
// primary (mirror baseline false), so the finally restores it to false. Muted BEFORE
|
|
1634
|
+
// the hand-off: its stop() hands console.log back to the mirror while the second
|
|
1635
|
+
// flush still waits, and those lines would print onto the user's normal screen.
|
|
1513
1636
|
setConsoleStdoutSuppressed(true);
|
|
1514
1637
|
try {
|
|
1638
|
+
if (wasRunning) await this.handOff(); // r2 B2: bounded flush + exit sequence before the prompt
|
|
1515
1639
|
process.stdout.write('\nOpening browser to log into Claude…\n');
|
|
1516
1640
|
const creds = await loginOAuth();
|
|
1517
1641
|
const profile = await fetchProfile(creds.accessToken);
|
|
@@ -1997,9 +2121,30 @@ export class TUI {
|
|
|
1997
2121
|
|
|
1998
2122
|
// ── rendering ──────────────────────────────────────
|
|
1999
2123
|
|
|
2124
|
+
/** Draw on the next tick at the latest — N calls inside one tick draw ONE frame
|
|
2125
|
+
* (2026-11-12: every log line used to redraw the full screen synchronously and the
|
|
2126
|
+
* pool froze on a terminal that fell behind). */
|
|
2127
|
+
requestRender() {
|
|
2128
|
+
if (!this.running || this._renderTimer) return;
|
|
2129
|
+
const wait = Math.max(0, RENDER_MIN_INTERVAL_MS - (Date.now() - this._lastRenderAt));
|
|
2130
|
+
this._renderTimer = setTimeout(() => {
|
|
2131
|
+
this._renderTimer = null;
|
|
2132
|
+
this.render();
|
|
2133
|
+
}, wait);
|
|
2134
|
+
this._renderTimer.unref?.();
|
|
2135
|
+
}
|
|
2136
|
+
|
|
2137
|
+
/** Draw NOW (keystroke, resize, takeover): the user is waiting on this frame. */
|
|
2138
|
+
forceRender() {
|
|
2139
|
+
if (!this.running) return;
|
|
2140
|
+
if (this._renderTimer) { clearTimeout(this._renderTimer); this._renderTimer = null; }
|
|
2141
|
+
this._forceNext = true;
|
|
2142
|
+
this.render();
|
|
2143
|
+
}
|
|
2144
|
+
|
|
2000
2145
|
render() {
|
|
2001
2146
|
if (!this.running) return;
|
|
2002
|
-
// Guard against re-entry: clearing an expired quota logs, and _addLog
|
|
2147
|
+
// Guard against re-entry: clearing an expired quota logs, and _addLog schedules
|
|
2003
2148
|
// render() again — without this the nested call would render twice.
|
|
2004
2149
|
if (this._rendering) return;
|
|
2005
2150
|
this._rendering = true;
|
|
@@ -2018,7 +2163,7 @@ export class TUI {
|
|
|
2018
2163
|
const H = process.stdout.rows || 24;
|
|
2019
2164
|
|
|
2020
2165
|
if (W < 40 || H < 8) {
|
|
2021
|
-
|
|
2166
|
+
this._writeScreen(`${ESC}H${ESC}2JTerminal too small (need 40x8+)\r\n`);
|
|
2022
2167
|
return;
|
|
2023
2168
|
}
|
|
2024
2169
|
|
|
@@ -2288,15 +2433,33 @@ export class TUI {
|
|
|
2288
2433
|
}
|
|
2289
2434
|
|
|
2290
2435
|
/** Full-screen buffer write: cursor home, H rows fitted to W, then hide the cursor
|
|
2291
|
-
* (shown only in input mode).
|
|
2436
|
+
* (shown only in input mode). The ONLY frame writer: it coalesces (an unchanged
|
|
2437
|
+
* frame is skipped unless forced / REPAINT_MS-old) and never writes into a terminal
|
|
2438
|
+
* that has not drained the last frame — it owes one render on 'drain' instead
|
|
2439
|
+
* (2026-11-12 freeze fix). Memory stays bounded: the stream's own high-water mark
|
|
2440
|
+
* stops us, and at most one frame is owed. */
|
|
2292
2441
|
_writeScreen(lines, W, H) {
|
|
2293
|
-
let buf
|
|
2294
|
-
|
|
2295
|
-
buf
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2442
|
+
let buf;
|
|
2443
|
+
if (typeof lines === 'string') {
|
|
2444
|
+
buf = lines; // pre-built screen (e.g. "Terminal too small")
|
|
2445
|
+
} else {
|
|
2446
|
+
buf = `${ESC}H`;
|
|
2447
|
+
for (let i = 0; i < H; i++) {
|
|
2448
|
+
buf += fitLine(lines[i] || '', W);
|
|
2449
|
+
if (i < H - 1) buf += '\r\n';
|
|
2450
|
+
}
|
|
2451
|
+
// Show cursor only in input mode
|
|
2452
|
+
buf += this.mode === 'input' ? `${ESC}?25h` : `${ESC}?25l`;
|
|
2453
|
+
}
|
|
2454
|
+
this._lastRenderAt = Date.now();
|
|
2455
|
+
if (!this._forceNext && this._lastFrame === buf && Date.now() - this._lastFullWriteAt < REPAINT_MS) return;
|
|
2456
|
+
if (this._nb && process.stdout.writableNeedDrain) {
|
|
2457
|
+
if (!this._drainArmed) { this._drainArmed = true; process.stdout.once('drain', this._onDrain); }
|
|
2458
|
+
return; // frame owed: written once, on 'drain'
|
|
2459
|
+
}
|
|
2460
|
+
this._forceNext = false;
|
|
2461
|
+
this._lastFrame = buf;
|
|
2462
|
+
this._lastFullWriteAt = Date.now();
|
|
2300
2463
|
process.stdout.write(buf);
|
|
2301
2464
|
}
|
|
2302
2465
|
|