@rikcodes/teamclaude 1.1.20-rik.10 → 1.1.20-rik.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +20 -0
- package/src/config.js +1 -0
- package/src/index.js +130 -9
- package/src/restart.js +186 -0
- package/src/server.js +107 -4
- package/src/session-tracker.js +14 -0
- package/src/terminal-title.js +18 -7
- package/src/tui.js +63 -7
- package/src/update-watch.js +232 -0
- package/src/updater.js +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rikcodes/teamclaude",
|
|
3
|
-
"version": "1.1.20-rik.
|
|
3
|
+
"version": "1.1.20-rik.12",
|
|
4
4
|
"description": "Multi-account proxy for Claude Code and Codex: pools Claude Max, ChatGPT/Codex, API-key and third-party backend accounts, and rotates on quota",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
package/src/account-manager.js
CHANGED
|
@@ -1437,6 +1437,26 @@ export class AccountManager {
|
|
|
1437
1437
|
return { ...this.sessionTracker.stats(), mode: this.distributionMode, draining: this.drainingCount() };
|
|
1438
1438
|
}
|
|
1439
1439
|
|
|
1440
|
+
/**
|
|
1441
|
+
* Client requests running right now, fleet-wide. What a drain waits on.
|
|
1442
|
+
*
|
|
1443
|
+
* Two counters, because neither one spans a whole request. A session's hold
|
|
1444
|
+
* is taken across the ENTIRE client request, a multi-minute stream included
|
|
1445
|
+
* (see beginSession), which is precisely what a restart must not cut — but it
|
|
1446
|
+
* is only taken for a request that carries a session id. An account's
|
|
1447
|
+
* `inFlight` covers the rest, though only as far as the response headers:
|
|
1448
|
+
* storm control releases that slot there so streaming bodies do not tie up
|
|
1449
|
+
* concurrency. Summing them double-counts a session's request while it is
|
|
1450
|
+
* upstream-bound, which can only make a drain wait longer than it strictly
|
|
1451
|
+
* must — and the drain deadline bounds that.
|
|
1452
|
+
*/
|
|
1453
|
+
inFlightRequests() {
|
|
1454
|
+
// The tracker is a plain object to the checker here (see the constructor),
|
|
1455
|
+
// so the cast is how its counter is reached — not a claim about the value.
|
|
1456
|
+
const sessions = /** @type {any} */ (this.sessionTracker).inFlightCount();
|
|
1457
|
+
return sessions + this.accounts.reduce((n, a) => n + (a.inFlight || 0), 0);
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1440
1460
|
/**
|
|
1441
1461
|
* Like getActiveAccount, but if the selected account's OAuth token has ALREADY
|
|
1442
1462
|
* expired it blocks on a refresh before returning — so a caller that injects
|
package/src/config.js
CHANGED
package/src/index.js
CHANGED
|
@@ -37,6 +37,8 @@ import { SessionTitles } from './session-titles.js';
|
|
|
37
37
|
import { RemoteControl, createAttachSession } from './tui-remote.js';
|
|
38
38
|
import { SxManager } from './sx.js';
|
|
39
39
|
import { autoUpdate, checkForUpdate, currentVersion, resolveVersionLabel, runUpdate, installKind, updateAvailableFromCache, PKG_NAME } from './updater.js';
|
|
40
|
+
import { drainServer, superviseServer, DRAIN_DEADLINE_MS, RESTART_COUNT_ENV, RESTART_EXIT_CODE, SUPERVISED_ENV } from './restart.js';
|
|
41
|
+
import { createVersionSource, UpdateWatcher } from './update-watch.js';
|
|
40
42
|
import { renderStatus, formatPercent } from './status-renderer.js';
|
|
41
43
|
import { sanitizeText } from './safe-text.js';
|
|
42
44
|
import { ClientUsageTracker, UsageDimensionTracker } from './client-usage.js';
|
|
@@ -223,6 +225,12 @@ switch (command) {
|
|
|
223
225
|
// ── server ──────────────────────────────────────────────────
|
|
224
226
|
|
|
225
227
|
async function serverCommand() {
|
|
228
|
+
// --supervise: this process supervises, it does not serve. The config, the
|
|
229
|
+
// accounts, the certificates and the terminal all belong to the child it
|
|
230
|
+
// starts, so this branch comes before any of them is touched.
|
|
231
|
+
if (args.includes('--supervise')) {
|
|
232
|
+
process.exit(await superviseServer());
|
|
233
|
+
}
|
|
226
234
|
// Installed first: the server is the long-lived process, it runs under a TUI
|
|
227
235
|
// that repaints over anything Node prints on the way out, and a crash here
|
|
228
236
|
// takes every routed session with it. Without this, a proxy that vanished
|
|
@@ -331,6 +339,7 @@ async function serverCommand() {
|
|
|
331
339
|
const persistQuotaState = () =>
|
|
332
340
|
saveState({ quota: accountManager.exportQuotaState(), clients: clientUsage.export(), usageDimensions: dimensionUsage.export(), sidecars: sidecar?.exportPids() || savedState?.sidecars || {} })
|
|
333
341
|
.catch(err => console.error(`[TeamClaude] Failed to save quota state: ${err.message}`));
|
|
342
|
+
/** @type {ReturnType<typeof setInterval>|null} */
|
|
334
343
|
let quotaSaveInterval = null;
|
|
335
344
|
|
|
336
345
|
// Persist refreshed tokens back to config (re-read from disk to avoid clobbering
|
|
@@ -376,12 +385,23 @@ async function serverCommand() {
|
|
|
376
385
|
const headless = args.includes('--headless') || args.includes('--no-tui');
|
|
377
386
|
const useTUI = !headless && process.stdout.isTTY && process.stdin.isTTY;
|
|
378
387
|
|
|
388
|
+
// Is anything waiting to relaunch this process? Exit 75 is a request, not a
|
|
389
|
+
// mechanism: with nothing supervising, it is merely an exit. So the two things
|
|
390
|
+
// that can ask for one — the `u` key and autoRestart — are wired only when the
|
|
391
|
+
// answer is yes. `teamclaude server --supervise` sets this on its child, and
|
|
392
|
+
// the shell loop in docs/usage.md exports it for the same reason.
|
|
393
|
+
const supervised = process.env[SUPERVISED_ENV] === '1';
|
|
394
|
+
const restartCount = Number(process.env[RESTART_COUNT_ENV]) || 0;
|
|
395
|
+
|
|
379
396
|
// Opt-in background quota probe (config.quotaProbeSeconds, default 0 = off).
|
|
397
|
+
/** @type {Prober|null} */
|
|
380
398
|
let prober = null;
|
|
381
399
|
// Opt-in keep-warm scheduler (interval or persisted reset-target schedule).
|
|
400
|
+
/** @type {Warmer|null} */
|
|
382
401
|
let warmer = null;
|
|
383
402
|
// Supervised sidecar processes (config.sidecars, default none) — e.g. a local
|
|
384
403
|
// Anthropic→OpenAI translating proxy that a third-party account routes to.
|
|
404
|
+
/** @type {Sidecar|null} */
|
|
385
405
|
let sidecar = null;
|
|
386
406
|
const serverStartedAt = Date.now();
|
|
387
407
|
// Read once here, not per request: `teamclaude update` swaps package.json on
|
|
@@ -491,7 +511,9 @@ async function serverCommand() {
|
|
|
491
511
|
};
|
|
492
512
|
|
|
493
513
|
let tui = null;
|
|
494
|
-
|
|
514
|
+
// A bag of optional callbacks the application installs on a shared object the
|
|
515
|
+
// server, the MITM listener and the control endpoints all read through.
|
|
516
|
+
/** @type {Record<string, any>} */
|
|
495
517
|
let hooks = {};
|
|
496
518
|
|
|
497
519
|
if (useTUI) {
|
|
@@ -531,6 +553,10 @@ async function serverCommand() {
|
|
|
531
553
|
// POSIX signals (defined below). In raw mode ctrl-c never reaches the OS as
|
|
532
554
|
// a signal, so without this the process would only tear down via keypress.
|
|
533
555
|
onQuit: () => shutdown(),
|
|
556
|
+
// `u`. Null without a supervisor: draining to an exit nothing acts on
|
|
557
|
+
// would take the proxy — and every session on it — down, which is the
|
|
558
|
+
// opposite of what a key labelled "update" offers.
|
|
559
|
+
onRestart: supervised ? () => { drainAndRestart('Restart requested'); } : null,
|
|
534
560
|
});
|
|
535
561
|
hooks = {
|
|
536
562
|
onRequestStart: (id, info) => tui.onRequestStart(id, info),
|
|
@@ -662,13 +688,14 @@ async function serverCommand() {
|
|
|
662
688
|
}
|
|
663
689
|
if (tui) {
|
|
664
690
|
tui.start();
|
|
665
|
-
console.log(`Listening on port ${port} with ${accounts.length} account(s)`);
|
|
691
|
+
console.log(`Listening on port ${port} with ${accounts.length} account(s) on ${versionLabel}`);
|
|
666
692
|
} else {
|
|
667
693
|
const sep = '='.repeat(60);
|
|
668
694
|
console.log('');
|
|
669
695
|
console.log(sep);
|
|
670
696
|
console.log(' TeamClaude Proxy');
|
|
671
697
|
console.log(sep);
|
|
698
|
+
console.log(` Version: ${versionLabel}`);
|
|
672
699
|
console.log(` Bind: ${bindHost}:${port}${bindHost === '127.0.0.1' ? ' (localhost only)' : ' (reachable off-box — ensure proxy.apiKey is set)'}`);
|
|
673
700
|
console.log(` Accounts: ${accounts.length}`);
|
|
674
701
|
console.log(` Threshold: ${(threshold * 100).toFixed(0)}%`);
|
|
@@ -683,11 +710,20 @@ async function serverCommand() {
|
|
|
683
710
|
console.log(sep);
|
|
684
711
|
console.log('');
|
|
685
712
|
}
|
|
713
|
+
// Said once, plainly. The complaint this whole path exists to answer is
|
|
714
|
+
// that the build changes under a window nobody is watching and nothing
|
|
715
|
+
// anywhere admits it: the title carries the version from here on, and this
|
|
716
|
+
// line is the moment it changed.
|
|
717
|
+
if (restartCount > 0) {
|
|
718
|
+
console.log(`[TeamClaude] Restarted on ${versionLabel} — relaunch #${restartCount} of this supervised run.`);
|
|
719
|
+
}
|
|
686
720
|
});
|
|
687
721
|
|
|
688
|
-
// Reflect the active account in the terminal title so a
|
|
689
|
-
// server is glanceable. Works in both TUI and headless
|
|
690
|
-
|
|
722
|
+
// Reflect the active account and the running build in the terminal title so a
|
|
723
|
+
// backgrounded/tabbed server is glanceable. Works in both TUI and headless
|
|
724
|
+
// modes, and is the surface an unattended restart announces itself on: the
|
|
725
|
+
// title is all that is readable when the window is not the one in front.
|
|
726
|
+
const stopTitle = startTerminalTitleUpdater(accountManager, versionLabel);
|
|
691
727
|
|
|
692
728
|
// Persist quota every minute; unref so it never keeps the process alive.
|
|
693
729
|
quotaSaveInterval = setInterval(persistQuotaState, 60_000);
|
|
@@ -760,6 +796,82 @@ async function serverCommand() {
|
|
|
760
796
|
}
|
|
761
797
|
process.on('SIGINT', shutdown);
|
|
762
798
|
process.on('SIGTERM', shutdown);
|
|
799
|
+
|
|
800
|
+
// The graceful counterpart to shutdown(), for the other request an operator
|
|
801
|
+
// can make: not "stop now" but "come back on the new build". shutdown() is
|
|
802
|
+
// left exactly as it is — destroying live streams is the right answer to
|
|
803
|
+
// ctrl-c — while this path has to cost the fleet nothing, because the point
|
|
804
|
+
// of automating it is that it happens while nobody is watching.
|
|
805
|
+
//
|
|
806
|
+
// Order matters and is not the same as shutdown()'s. The flag goes up FIRST,
|
|
807
|
+
// so every answer still to be written carries the header that retires its
|
|
808
|
+
// socket (markDraining, server.js) — that, not the waiting, is what stops a
|
|
809
|
+
// restart from breaking sessions that were only ever idle. Then the listener
|
|
810
|
+
// stops accepting while the connections already open keep serving; then the
|
|
811
|
+
// bounded wait; then the sidecar, whose replacement the next process spawns;
|
|
812
|
+
// then quota state, exactly as shutdown() persists it. Exit 75 is the ask.
|
|
813
|
+
let draining = false;
|
|
814
|
+
hooks.isDraining = () => draining;
|
|
815
|
+
/** @param {string} why what put the restart in motion, for the line on the way out */
|
|
816
|
+
async function drainAndRestart(why) {
|
|
817
|
+
if (shuttingDown) return; // ctrl-c beat us here, or a second trigger did
|
|
818
|
+
shuttingDown = true;
|
|
819
|
+
draining = true;
|
|
820
|
+
try { tui?.stop(); } catch { /* terminal already restored */ }
|
|
821
|
+
stopTitle();
|
|
822
|
+
console.log(`\n[TeamClaude] ${why} — draining, up to ${Math.round(DRAIN_DEADLINE_MS / 1000)}s for requests in flight.`);
|
|
823
|
+
prober?.stop();
|
|
824
|
+
warmer?.stop();
|
|
825
|
+
eventLoopMonitor.stop();
|
|
826
|
+
// Nothing below may throw its way out. Neither caller awaits this — the TUI
|
|
827
|
+
// key returns to its handler and the watcher fires from a timer — so an
|
|
828
|
+
// escaping rejection would be an unhandled one, and crash-log.js turns that
|
|
829
|
+
// into exit 1: the supervisor would read a crash and stop, on the one path
|
|
830
|
+
// whose entire purpose is to come back.
|
|
831
|
+
try {
|
|
832
|
+
const { drained, waitedMs, inFlight } = await drainServer({
|
|
833
|
+
server,
|
|
834
|
+
inFlight: () => accountManager.inFlightRequests(),
|
|
835
|
+
});
|
|
836
|
+
console.log(drained
|
|
837
|
+
? `[TeamClaude] Drained in ${(waitedMs / 1000).toFixed(1)}s. Restarting on the new build.`
|
|
838
|
+
: `[TeamClaude] ${inFlight} request(s) still in flight after ${(waitedMs / 1000).toFixed(0)}s — restarting anyway.`);
|
|
839
|
+
sidecar?.stop();
|
|
840
|
+
if (quotaSaveInterval) clearInterval(quotaSaveInterval);
|
|
841
|
+
await persistQuotaState();
|
|
842
|
+
} catch (err) {
|
|
843
|
+
// Committed from the moment the display came down and the listener
|
|
844
|
+
// closed: there is no serving state left to return to, so say what broke
|
|
845
|
+
// and let the relaunch be the recovery.
|
|
846
|
+
console.error(`[TeamClaude] Drain failed: ${err.message}`);
|
|
847
|
+
}
|
|
848
|
+
process.exit(RESTART_EXIT_CODE);
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
// Opt-in, and only where a restart would actually happen. Nothing below runs
|
|
852
|
+
// on a default config, so the install probe behind createVersionSource is not
|
|
853
|
+
// paid for by anyone who did not ask for this.
|
|
854
|
+
if (config.autoRestart && !supervised) {
|
|
855
|
+
console.error('[TeamClaude] autoRestart is set, but nothing will relaunch this process — start it with "teamclaude server --supervise". Auto-restart is off for this run.');
|
|
856
|
+
} else if (config.autoRestart) {
|
|
857
|
+
const source = await createVersionSource();
|
|
858
|
+
if (!source) {
|
|
859
|
+
console.error('[TeamClaude] autoRestart is set, but this copy is a local or npx install — nothing rewrites it, so a restart would come back on the same build. Auto-restart is off for this run.');
|
|
860
|
+
} else {
|
|
861
|
+
new UpdateWatcher({
|
|
862
|
+
source,
|
|
863
|
+
// Nothing running and no session still counted active: a restart now
|
|
864
|
+
// costs a reconnect and nothing else.
|
|
865
|
+
isIdle: () => accountManager.inFlightRequests() === 0 && accountManager.sessionStats().active === 0,
|
|
866
|
+
onRestart: ({ build, forced }) => {
|
|
867
|
+
drainAndRestart(forced
|
|
868
|
+
? `Build ${build} is waiting and the fleet has not gone idle`
|
|
869
|
+
: `Build ${build} is waiting`);
|
|
870
|
+
},
|
|
871
|
+
}).start();
|
|
872
|
+
console.log(`[TeamClaude] Auto-restart is on, watching ${source.describes}.`);
|
|
873
|
+
}
|
|
874
|
+
}
|
|
763
875
|
}
|
|
764
876
|
|
|
765
877
|
// ── import ──────────────────────────────────────────────────
|
|
@@ -2192,6 +2304,9 @@ Options:
|
|
|
2192
2304
|
--log-to DIR Log requests/responses to DIR (server, one file per request)
|
|
2193
2305
|
--activity-log FILE Append TUI activity lines to FILE (server; works in headless mode too)
|
|
2194
2306
|
--headless Run the server without the interactive TUI (for backgrounding)
|
|
2307
|
+
--supervise (server) run the proxy as a child process and relaunch it
|
|
2308
|
+
whenever it drains for a new build (the TUI's 'u' key, or
|
|
2309
|
+
the autoRestart setting). Without it neither can restart
|
|
2195
2310
|
--no-mitm (run) skip the forward proxy; route via ANTHROPIC_BASE_URL only
|
|
2196
2311
|
--auto-fallback (run) if the proxy is down, launch claude directly instead
|
|
2197
2312
|
of erroring out (bypasses the proxy: no rotation)
|
|
@@ -2403,13 +2518,19 @@ function argValue(flag) {
|
|
|
2403
2518
|
return (i >= 0 && args[i + 1]) ? args[i + 1] : null;
|
|
2404
2519
|
}
|
|
2405
2520
|
|
|
2406
|
-
// Keep the terminal title in sync with the active account
|
|
2407
|
-
// work") so a backgrounded or tabbed
|
|
2521
|
+
// Keep the terminal title in sync with the active account and the running build
|
|
2522
|
+
// (e.g. "teamclaude 2/4 work 1.1.20-rik.11") so a backgrounded or tabbed
|
|
2523
|
+
// `teamclaude server` is glanceable — and so a restart onto a new build is
|
|
2524
|
+
// visible there without anyone going looking. TTY-only
|
|
2408
2525
|
// — never emit escapes into a pipe, a `--log-to` redirect, or a systemd journal;
|
|
2409
2526
|
// opt out entirely with TEAMCLAUDE_NO_TITLE. Polls (rather than hooking every
|
|
2410
2527
|
// currentIndex mutation) and writes only when the title actually changes.
|
|
2411
2528
|
// Returns an idempotent stop() that restores the shell's previous title.
|
|
2412
|
-
|
|
2529
|
+
/**
|
|
2530
|
+
* @param {AccountManager} accountManager
|
|
2531
|
+
* @param {string|null} [version]
|
|
2532
|
+
*/
|
|
2533
|
+
function startTerminalTitleUpdater(accountManager, version = null) {
|
|
2413
2534
|
const out = process.stdout;
|
|
2414
2535
|
if (!out.isTTY || process.env.TEAMCLAUDE_NO_TITLE) return () => {};
|
|
2415
2536
|
|
|
@@ -2418,7 +2539,7 @@ function startTerminalTitleUpdater(accountManager) {
|
|
|
2418
2539
|
const total = accountManager.accounts.length;
|
|
2419
2540
|
const index = Math.min(accountManager.currentIndex || 0, Math.max(0, total - 1));
|
|
2420
2541
|
const name = accountManager.accounts[index]?.name || null;
|
|
2421
|
-
const title = formatTerminalTitle({ index, total, name });
|
|
2542
|
+
const title = formatTerminalTitle({ index, total, name, version });
|
|
2422
2543
|
if (title !== last) { last = title; out.write(titleSequence(title)); }
|
|
2423
2544
|
};
|
|
2424
2545
|
|
package/src/restart.js
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
// Drain-and-restart: applying a new build without cutting a live session.
|
|
2
|
+
//
|
|
3
|
+
// Restarting the proxy used to mean ctrl-c and a fast re-run, which breaks
|
|
4
|
+
// every Claude Code session going through it, for two separate reasons:
|
|
5
|
+
//
|
|
6
|
+
// 1. shutdown() calls server.closeAllConnections(), which DESTROYS in-flight
|
|
7
|
+
// streaming responses. That abruptness is deliberate — a person holding
|
|
8
|
+
// ctrl-c wants out now — so it is left exactly as it is, and the graceful
|
|
9
|
+
// path lives here instead.
|
|
10
|
+
// 2. A restart kills idle keep-alive sockets the client still holds pooled.
|
|
11
|
+
// The client finds out by writing to a corpse, which is the same failure
|
|
12
|
+
// family as the two keep-alive fixes before this one.
|
|
13
|
+
//
|
|
14
|
+
// A drain answers both: stop accepting connections, tell every response on the
|
|
15
|
+
// way out that its socket is finished (`Connection: close`, set in server.js
|
|
16
|
+
// off `hooks.isDraining`), and wait for the requests already running to end.
|
|
17
|
+
// Clients retire their pooled sockets cooperatively and reconnect into the
|
|
18
|
+
// relaunched process.
|
|
19
|
+
//
|
|
20
|
+
// The relaunch itself is exit code 75 plus a supervisor, because a foreground
|
|
21
|
+
// TUI cannot re-exec itself: the parent exits, the shell prints a prompt, and
|
|
22
|
+
// the child fights it for the terminal.
|
|
23
|
+
|
|
24
|
+
import { spawn } from 'node:child_process';
|
|
25
|
+
|
|
26
|
+
// "Restart me." Chosen from the sysexits.h range (EX_TEMPFAIL) so it cannot
|
|
27
|
+
// collide with the 0/1 a crash or a clean quit already uses, and so a
|
|
28
|
+
// supervisor that does not know about it treats it as an ordinary failure.
|
|
29
|
+
export const RESTART_EXIT_CODE = 75;
|
|
30
|
+
|
|
31
|
+
// How long a drain waits for in-flight requests before going anyway. A stuck
|
|
32
|
+
// stream — an upstream that stopped sending without closing — must never be
|
|
33
|
+
// able to block a restart forever, and the requests that outlive this get the
|
|
34
|
+
// abrupt end they would have got from ctrl-c.
|
|
35
|
+
export const DRAIN_DEADLINE_MS = 30_000;
|
|
36
|
+
const DRAIN_POLL_MS = 100;
|
|
37
|
+
|
|
38
|
+
// A child that asks to be restarted this many times inside the window is
|
|
39
|
+
// looping, not updating: something makes the new build ask for a restart the
|
|
40
|
+
// moment it is up, and relaunching it forever would hide that behind a
|
|
41
|
+
// flickering terminal.
|
|
42
|
+
export const RESTART_LOOP_LIMIT = 5;
|
|
43
|
+
export const RESTART_LOOP_WINDOW_MS = 60_000;
|
|
44
|
+
|
|
45
|
+
// Set on a supervised child. Two things read it: the `u` key and the automatic
|
|
46
|
+
// restart, neither of which may exit 75 unless something is actually waiting to
|
|
47
|
+
// relaunch this process — otherwise "apply the update" reads as "kill the proxy
|
|
48
|
+
// and every session on it". The documented shell one-liner exports it too.
|
|
49
|
+
export const SUPERVISED_ENV = 'TEAMCLAUDE_SUPERVISED';
|
|
50
|
+
// How many relaunches this process is, so it can say so once it is listening.
|
|
51
|
+
export const RESTART_COUNT_ENV = 'TEAMCLAUDE_RESTARTS';
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Stop taking work and wait for what is running to finish.
|
|
55
|
+
*
|
|
56
|
+
* `server.close()` and NOT `closeAllConnections()`: close stops the listener,
|
|
57
|
+
* while the connections already open keep streaming — and keep serving, which
|
|
58
|
+
* is the point. A client whose pooled socket is still good sends its next
|
|
59
|
+
* request into this window and gets a real answer carrying `Connection: close`,
|
|
60
|
+
* so it retires that socket itself rather than discovering it dead later. Only
|
|
61
|
+
* the sockets that stayed idle through the whole drain are closed, at the end,
|
|
62
|
+
* where the gap between the close and the relaunch is as small as it can be.
|
|
63
|
+
*
|
|
64
|
+
* Every side effect is injectable so the deadline can be tested without
|
|
65
|
+
* spending it.
|
|
66
|
+
*
|
|
67
|
+
* @param {Object} opts
|
|
68
|
+
* @param {{ close?: Function, closeIdleConnections?: Function }} opts.server
|
|
69
|
+
* @param {() => number} opts.inFlight requests still running, fleet-wide
|
|
70
|
+
* @param {number} [opts.deadlineMs]
|
|
71
|
+
* @param {number} [opts.pollMs]
|
|
72
|
+
* @param {() => number} [opts.now]
|
|
73
|
+
* @param {(ms: number) => Promise<void>} [opts.sleep]
|
|
74
|
+
* @returns {Promise<{ drained: boolean, waitedMs: number, inFlight: number }>}
|
|
75
|
+
*/
|
|
76
|
+
export async function drainServer({
|
|
77
|
+
server, inFlight, deadlineMs = DRAIN_DEADLINE_MS, pollMs = DRAIN_POLL_MS,
|
|
78
|
+
now = Date.now, sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)),
|
|
79
|
+
}) {
|
|
80
|
+
const started = now();
|
|
81
|
+
// No callback: a CONNECT tunnel is not tracked as a connection Node will tell
|
|
82
|
+
// us about, so waiting for close() to call back could wait forever. What we
|
|
83
|
+
// wait on is the request count below, which the deadline bounds.
|
|
84
|
+
server.close?.();
|
|
85
|
+
let open = inFlight();
|
|
86
|
+
while (open > 0 && now() - started < deadlineMs) {
|
|
87
|
+
await sleep(pollMs);
|
|
88
|
+
open = inFlight();
|
|
89
|
+
}
|
|
90
|
+
// The sockets that never carried a response during the drain, and so never
|
|
91
|
+
// got the `Connection: close` that retires them politely. Closing them is
|
|
92
|
+
// unavoidable — the process is going — so it happens here, last, rather than
|
|
93
|
+
// at the start where it would yank a socket the client was about to use.
|
|
94
|
+
server.closeIdleConnections?.();
|
|
95
|
+
return { drained: open <= 0, waitedMs: now() - started, inFlight: Math.max(0, open) };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Run the real server as a child and relaunch it when it asks (exit 75). Any
|
|
100
|
+
* other exit is the child's answer and ends the supervisor with it.
|
|
101
|
+
*
|
|
102
|
+
* `stdio: 'inherit'` hands over the actual terminal, so the child's TUI owns
|
|
103
|
+
* raw mode, the tty size and the title exactly as it would unsupervised. The
|
|
104
|
+
* supervisor itself must therefore touch neither stdin nor the screen.
|
|
105
|
+
*
|
|
106
|
+
* Resolves with the exit code this process should use.
|
|
107
|
+
*
|
|
108
|
+
* @param {Object} [opts]
|
|
109
|
+
* @param {string[]} [opts.argv] what to run: [script, ...args], `--supervise` removed
|
|
110
|
+
* @param {typeof spawn} [opts.spawnFn]
|
|
111
|
+
* @param {string} [opts.execPath]
|
|
112
|
+
* @param {NodeJS.ProcessEnv} [opts.env]
|
|
113
|
+
* @param {(line: string) => void} [opts.log]
|
|
114
|
+
* @param {() => number} [opts.now]
|
|
115
|
+
* @param {number} [opts.limit]
|
|
116
|
+
* @param {number} [opts.windowMs]
|
|
117
|
+
* @param {(event: string, handler: Function) => void} [opts.onSignal]
|
|
118
|
+
*/
|
|
119
|
+
export function superviseServer({
|
|
120
|
+
argv = process.argv.slice(1), spawnFn = spawn, execPath = process.execPath,
|
|
121
|
+
env = process.env, log = console.error, now = Date.now,
|
|
122
|
+
limit = RESTART_LOOP_LIMIT, windowMs = RESTART_LOOP_WINDOW_MS,
|
|
123
|
+
onSignal = (event, handler) => process.on(event, /** @type {any} */ (handler)),
|
|
124
|
+
} = {}) {
|
|
125
|
+
const childArgv = argv.filter(a => a !== '--supervise');
|
|
126
|
+
/** @type {number[]} */
|
|
127
|
+
const relaunches = [];
|
|
128
|
+
/** @type {import('node:child_process').ChildProcess|null} */
|
|
129
|
+
let child = null;
|
|
130
|
+
let stopping = false;
|
|
131
|
+
|
|
132
|
+
// SIGTERM is aimed at this pid alone, so it has to be passed on or the child
|
|
133
|
+
// keeps the terminal with nothing supervising it. SIGINT deliberately is NOT:
|
|
134
|
+
// ctrl-c goes to the whole foreground process group, so the child already has
|
|
135
|
+
// it, and a second one tells its shutdown() to stop waiting and exit at once
|
|
136
|
+
// — the graceful teardown skipped by the very key meant to allow it.
|
|
137
|
+
onSignal('SIGTERM', () => { stopping = true; child?.kill('SIGTERM'); });
|
|
138
|
+
onSignal('SIGINT', () => { stopping = true; });
|
|
139
|
+
|
|
140
|
+
return new Promise((resolve) => {
|
|
141
|
+
const launch = (/** @type {number} */ restarts) => {
|
|
142
|
+
try {
|
|
143
|
+
child = spawnFn(execPath, childArgv, {
|
|
144
|
+
stdio: 'inherit',
|
|
145
|
+
env: { ...env, [SUPERVISED_ENV]: '1', [RESTART_COUNT_ENV]: String(restarts) },
|
|
146
|
+
});
|
|
147
|
+
} catch (err) {
|
|
148
|
+
log(`[TeamClaude] Could not start the server: ${/** @type {Error} */ (err).message}`);
|
|
149
|
+
resolve(1);
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
if (!child || typeof child.once !== 'function') { resolve(1); return; }
|
|
153
|
+
child.once('error', (err) => {
|
|
154
|
+
log(`[TeamClaude] Could not start the server: ${err.message}`);
|
|
155
|
+
resolve(1);
|
|
156
|
+
});
|
|
157
|
+
child.once('exit', (code, signal) => {
|
|
158
|
+
child = null;
|
|
159
|
+
// Killed rather than exited: the operator or the OS ended it, and there
|
|
160
|
+
// is nothing here to second-guess. 128+n is the shell's convention for
|
|
161
|
+
// reporting which signal it was.
|
|
162
|
+
if (code === null) { resolve(signal ? 128 + (signalNumber(signal) || 0) : 1); return; }
|
|
163
|
+
if (code !== RESTART_EXIT_CODE || stopping) { resolve(code); return; }
|
|
164
|
+
|
|
165
|
+
const at = now();
|
|
166
|
+
relaunches.push(at);
|
|
167
|
+
while (relaunches.length && at - relaunches[0] > windowMs) relaunches.shift();
|
|
168
|
+
if (relaunches.length > limit) {
|
|
169
|
+
log(`[TeamClaude] The server asked to restart ${relaunches.length} times in ${Math.round(windowMs / 1000)}s — it is looping, not updating. Giving up; start it again by hand once you know why.`);
|
|
170
|
+
resolve(1);
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
launch(restarts + 1);
|
|
174
|
+
});
|
|
175
|
+
};
|
|
176
|
+
launch(0);
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Signal name to number, for the 128+n exit convention. Unknown names report
|
|
181
|
+
* 0, which reads as "killed, we don't know by what" rather than throwing.
|
|
182
|
+
* @param {NodeJS.Signals|string} name */
|
|
183
|
+
function signalNumber(name) {
|
|
184
|
+
const table = /** @type {Record<string, number>} */ ({ SIGHUP: 1, SIGINT: 2, SIGQUIT: 3, SIGKILL: 9, SIGTERM: 15 });
|
|
185
|
+
return table[String(name)] || 0;
|
|
186
|
+
}
|
package/src/server.js
CHANGED
|
@@ -61,6 +61,29 @@ const INLINE_RETRY_AFTER_MAX_SECONDS = 15;
|
|
|
61
61
|
// the account so concurrent requests wait, then retries the same account.
|
|
62
62
|
const RATE_LIMIT_ABSORB_MAX_SECONDS =
|
|
63
63
|
Number(process.env.TEAMCLAUDE_RATE_LIMIT_ABSORB_MAX_SECONDS) || 60;
|
|
64
|
+
// How long to wait before the one retry of a headerless 429 — a 429 carrying no
|
|
65
|
+
// retry-after and no anthropic-ratelimit-* headers at all.
|
|
66
|
+
//
|
|
67
|
+
// Measured over a 32-minute window: these arrive in 0.6-0.8s, about once every
|
|
68
|
+
// 8 minutes on Fable traffic and never on any other model, and they follow the
|
|
69
|
+
// request onto whichever account the failover hop moves it to. That hop re-asks
|
|
70
|
+
// roughly 0.7s after the first refusal and is refused again, which is direct
|
|
71
|
+
// evidence that a wait shorter than that buys nothing but a third identical
|
|
72
|
+
// refusal. The ceiling is what the client does instead: Claude Code shows
|
|
73
|
+
// "will retry in 2m 38s" and then usually succeeds, so any wait measured in
|
|
74
|
+
// seconds trades a visible stall for an invisible one. 2s clears the interval
|
|
75
|
+
// already known to fail while keeping the worst case — the retry is refused too
|
|
76
|
+
// and the client gets its 429 anyway, just later — at about 4s.
|
|
77
|
+
//
|
|
78
|
+
// One delay, not a ladder: the limit's window is unknown, and a second guess at
|
|
79
|
+
// it would cost the client the wait without evidence that it helps. Override
|
|
80
|
+
// with TEAMCLAUDE_HEADERLESS_429_RETRY_DELAY_MS.
|
|
81
|
+
const DEFAULT_HEADERLESS_429_RETRY_DELAY_MS = 2000;
|
|
82
|
+
|
|
83
|
+
function resolveHeaderless429RetryDelayMs() {
|
|
84
|
+
const env = Number(process.env.TEAMCLAUDE_HEADERLESS_429_RETRY_DELAY_MS);
|
|
85
|
+
return env > 0 ? env : DEFAULT_HEADERLESS_429_RETRY_DELAY_MS;
|
|
86
|
+
}
|
|
64
87
|
const OAUTH_ENTITLEMENT_ERROR_CODE = 'oauth_not_allowed_for_organization';
|
|
65
88
|
const ERROR_BODY_INSPECTION_LIMIT = 64 * 1024;
|
|
66
89
|
// How long an idle keep-alive connection is held open.
|
|
@@ -121,6 +144,36 @@ const CONNECTION_SPECIFIC_HEADERS = new Set([
|
|
|
121
144
|
'proxy-connection', 'te', 'trailer',
|
|
122
145
|
]);
|
|
123
146
|
|
|
147
|
+
/**
|
|
148
|
+
* While the server is draining, tell this response's client that the socket is
|
|
149
|
+
* finished with.
|
|
150
|
+
*
|
|
151
|
+
* The cooperative half of a restart (see restart.js). A process that simply
|
|
152
|
+
* stops leaves every pooled keep-alive socket a corpse the client discovers
|
|
153
|
+
* only by writing to it — the failure this proxy has now been bitten by twice,
|
|
154
|
+
* and the reason a restart "breaks running sessions" even when the drain waits
|
|
155
|
+
* politely for the requests it can see. `Connection: close` retires the socket
|
|
156
|
+
* from the client's pool the moment this response lands, so the next request
|
|
157
|
+
* opens a fresh connection into the relaunched process.
|
|
158
|
+
*
|
|
159
|
+
* Set with setHeader rather than in a writeHead object: every answer below
|
|
160
|
+
* builds its own header object, and `connection` is stripped from all of them
|
|
161
|
+
* as hop-by-hop, so this survives the merge on each of the dozen exits instead
|
|
162
|
+
* of having to be added to each.
|
|
163
|
+
*
|
|
164
|
+
* HTTP/2 is left alone — the header is illegal there (Node refuses it) and a
|
|
165
|
+
* MITM tunnel's h2 session goes away with the CONNECT socket regardless.
|
|
166
|
+
*
|
|
167
|
+
* @param {import('node:http').IncomingMessage} req
|
|
168
|
+
* @param {import('node:http').ServerResponse} res
|
|
169
|
+
* @param {any} hooks the application's hook bag; `isDraining` is optional
|
|
170
|
+
*/
|
|
171
|
+
export function markDraining(req, res, hooks) {
|
|
172
|
+
if (!hooks.isDraining?.()) return;
|
|
173
|
+
if ((req.httpVersionMajor || 1) >= 2) return;
|
|
174
|
+
try { res.setHeader('Connection', 'close'); } catch { /* already answered */ }
|
|
175
|
+
}
|
|
176
|
+
|
|
124
177
|
// Constant-time proxy-API-key comparison (both the HTTP gate and the CONNECT
|
|
125
178
|
// gate use it). Returns false on any type/length mismatch without leaking timing.
|
|
126
179
|
export function safeKeyEqual(a, b) {
|
|
@@ -247,6 +300,9 @@ export function createProxyServer(accountManager, config, hooks = {}, sx = null,
|
|
|
247
300
|
|
|
248
301
|
const requestHandler = async (req, res) => {
|
|
249
302
|
try {
|
|
303
|
+
// Before any exit below writes a head, control endpoints included: a
|
|
304
|
+
// status poll and a dashboard refresh hold pooled sockets too.
|
|
305
|
+
markDraining(req, res, hooks);
|
|
250
306
|
// Dashboard page — served BEFORE the auth gate on purpose. The page is a
|
|
251
307
|
// static asset containing no data: everything it shows comes from
|
|
252
308
|
// /teamclaude/status, which stays behind the gate and is fetched by the
|
|
@@ -850,6 +906,9 @@ export function clientSessionId(headers) {
|
|
|
850
906
|
export function createProxyRequestListener({ accountManager, upstream, logDir = null, hooks = {}, sx = null, holdMs = 0, config = {}, forcedPin = null, egress = null, clientUsage = null, forcedClient = null, dimensionUsage = null }) {
|
|
851
907
|
let counter = 0;
|
|
852
908
|
return async (req, res) => {
|
|
909
|
+
// Again here, not only in the base server's wrapper: this listener is also
|
|
910
|
+
// the MITM tunnel's, where nothing above it has seen the request.
|
|
911
|
+
markDraining(req, res, hooks);
|
|
853
912
|
// The activity entry this request opened, while it is still open. Every
|
|
854
913
|
// consumer holds the row until it is told the request ended, so exactly one
|
|
855
914
|
// path must close it. Each closing site clears this first, which is how the
|
|
@@ -2419,7 +2478,45 @@ export async function forwardRequest(req, res, body, accountManager, upstream, r
|
|
|
2419
2478
|
} else if (ctx.rateLimitHopped && requestScoped) {
|
|
2420
2479
|
// Second headerless 429, on a different account: it followed the
|
|
2421
2480
|
// request. Nothing here is about either account.
|
|
2422
|
-
|
|
2481
|
+
//
|
|
2482
|
+
// That does not make it permanent. Measured: these land about once every
|
|
2483
|
+
// 8 minutes on Fable traffic, from four different starting accounts, and
|
|
2484
|
+
// the client's own retry usually succeeds — so most are a transient the
|
|
2485
|
+
// fleet cannot route around, not a model id upstream refuses. Returning
|
|
2486
|
+
// it straight away is what left Claude Code sitting on "will retry in
|
|
2487
|
+
// 2m 38s", with nothing in the session transcript to explain the pause.
|
|
2488
|
+
//
|
|
2489
|
+
// So take one short retry first, on THIS account. Not on a third one:
|
|
2490
|
+
// the limit is scoped to neither account, so another hop would pay a
|
|
2491
|
+
// cold prompt cache to learn what the first hop already established —
|
|
2492
|
+
// the same argument that bounds the hop budget above. ctx.hopTo keeps
|
|
2493
|
+
// the attempt on the account it landed on and off the fleet cursor
|
|
2494
|
+
// (#286), and `route` keeps its egress, since the wait is the only
|
|
2495
|
+
// variable being tested. A fresh IP is a different hypothesis and the sx
|
|
2496
|
+
// retry below still owns it: when it is armed it goes first, for free,
|
|
2497
|
+
// and this retry takes the attempt after it.
|
|
2498
|
+
//
|
|
2499
|
+
// Only for a caller that actually waits. A Codex one does not: it gives
|
|
2500
|
+
// the response head 60s, then retries the whole request itself (see
|
|
2501
|
+
// holdsConnection, and the incident recorded in
|
|
2502
|
+
// test/codex-no-inline-hold.test.js). Holding it here would stack our
|
|
2503
|
+
// wait underneath its own, which is the trade that file exists to refuse.
|
|
2504
|
+
const retryDelayMs = resolveHeaderless429RetryDelayMs();
|
|
2505
|
+
const callerWaits = holdsConnection(ctx.provider);
|
|
2506
|
+
if (!ctx.headerless429Retried && !switchingToSx && retryCount < maxRetries
|
|
2507
|
+
&& !res.headersSent && !clientGone(res) && !ctx.signal?.aborted && callerWaits) {
|
|
2508
|
+
// Once per request: a retry that is refused too has made the point.
|
|
2509
|
+
ctx.headerless429Retried = true;
|
|
2510
|
+
console.log(`[TeamClaude] 429 followed the request onto "${account.name}" with no rate-limit headers — retrying it once on the same account in ${retryDelayMs}ms`
|
|
2511
|
+
+ (refusal ? ` (${safeLine(refusal)})` : ''));
|
|
2512
|
+
await waitForRetry(retryDelayMs, ctx.signal);
|
|
2513
|
+
if (clientGone(res)) { ctx.abandoned = true; return; }
|
|
2514
|
+
ctx.hopTo = account.index;
|
|
2515
|
+
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, route);
|
|
2516
|
+
}
|
|
2517
|
+
console.log(`[TeamClaude] 429 followed the request onto "${account.name}" with no rate-limit headers — `
|
|
2518
|
+
+ (callerWaits ? 'it is about the request, not the accounts' : `${ctx.provider} caller does not wait`)
|
|
2519
|
+
+ '; returning it to the client'
|
|
2423
2520
|
+ (refusal ? ` (${safeLine(refusal)})` : ''));
|
|
2424
2521
|
} else if (ctx.rateLimitHopped) {
|
|
2425
2522
|
// Second 429 this request, on a different account. Say so once: the
|
|
@@ -2445,10 +2542,16 @@ export async function forwardRequest(req, res, body, accountManager, upstream, r
|
|
|
2445
2542
|
// third time is the other half of #288. With no sibling to hop to, one
|
|
2446
2543
|
// short retry covers a momentary blip, and then it is the client's turn.
|
|
2447
2544
|
if (requestScoped) {
|
|
2448
|
-
|
|
2545
|
+
// Same rule as the post-hop retry and the inline absorb below: a wait is
|
|
2546
|
+
// only invisible to a caller that waits longer than we do.
|
|
2547
|
+
if (!ctx.rateLimitHopped && !ctx.requestScopedRetried && retryCount < maxRetries
|
|
2548
|
+
&& holdsConnection(ctx.provider)) {
|
|
2449
2549
|
ctx.requestScopedRetried = true;
|
|
2450
|
-
|
|
2451
|
-
|
|
2550
|
+
// The same number as the post-hop retry above: one phenomenon, one
|
|
2551
|
+
// delay, one env var to move both.
|
|
2552
|
+
const retryDelayMs = resolveHeaderless429RetryDelayMs();
|
|
2553
|
+
console.log(`[TeamClaude] 429 with no rate-limit headers on "${account.name}" — retrying once in ${retryDelayMs}ms${refusal ? ` (${safeLine(refusal)})` : ''}`);
|
|
2554
|
+
await waitForRetry(retryDelayMs, ctx.signal);
|
|
2452
2555
|
if (clientGone(res)) { ctx.abandoned = true; return; }
|
|
2453
2556
|
return forwardRequest(req, res, body, accountManager, upstream, retryCount + 1, hooks, reqId, ctx, logDir, sx, nextUseSx);
|
|
2454
2557
|
}
|
package/src/session-tracker.js
CHANGED
|
@@ -341,6 +341,20 @@ export class SessionTracker {
|
|
|
341
341
|
return s.inFlight > 0 || now - s.lastSeen <= this.activeTtlMs;
|
|
342
342
|
}
|
|
343
343
|
|
|
344
|
+
/** Requests in flight across every session, for a drain to wait on.
|
|
345
|
+
*
|
|
346
|
+
* The hold this counts spans the whole client request, a long stream
|
|
347
|
+
* included (see beginRequest), so zero here means no client is mid-answer —
|
|
348
|
+
* which is the one thing a restart has to be sure of. Counted on demand
|
|
349
|
+
* rather than kept as a running total: this is asked a few times a second
|
|
350
|
+
* during a drain and never otherwise, while the map is bounded by MAX_SESSIONS.
|
|
351
|
+
*/
|
|
352
|
+
inFlightCount() {
|
|
353
|
+
let n = 0;
|
|
354
|
+
for (const s of this.sessions.values()) n += s.inFlight;
|
|
355
|
+
return n;
|
|
356
|
+
}
|
|
357
|
+
|
|
344
358
|
// Expired = idle past the known window AND nothing in flight (a long-running
|
|
345
359
|
// request keeps the session alive no matter how old lastSeen is).
|
|
346
360
|
_isExpired(s, now) {
|
package/src/terminal-title.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
// Reflect the active account in the terminal title (e.g.
|
|
2
|
-
// so a backgrounded or tabbed
|
|
3
|
-
// switching to it. Pure/side-effect-
|
|
4
|
-
// caller owns the TTY gate and the
|
|
1
|
+
// Reflect the active account and the running build in the terminal title (e.g.
|
|
2
|
+
// "teamclaude 2/4 work 1.1.20-rik.11"), so a backgrounded or tabbed
|
|
3
|
+
// `teamclaude server` is glanceable without switching to it. Pure/side-effect-
|
|
4
|
+
// free here so it can be unit-tested; the caller owns the TTY gate and the
|
|
5
|
+
// interval.
|
|
5
6
|
|
|
6
7
|
const OSC_TITLE = '\x1b]0;'; // OSC 0 — set icon name + window title
|
|
7
8
|
const BEL = '\x07';
|
|
@@ -20,11 +21,21 @@ function truncate(s, max) {
|
|
|
20
21
|
return s.length <= max ? s : `${s.slice(0, max - 1)}…`;
|
|
21
22
|
}
|
|
22
23
|
|
|
23
|
-
// Short, glanceable title: "teamclaude <pos>/<total> <name>". `index`
|
|
24
|
-
|
|
24
|
+
// Short, glanceable title: "teamclaude <pos>/<total> <name> <version>". `index`
|
|
25
|
+
// is 0-based.
|
|
26
|
+
//
|
|
27
|
+
// The version is here because the title is the only surface still readable
|
|
28
|
+
// while the window is backgrounded — which is exactly the state a server is in
|
|
29
|
+
// when a drain-and-restart swaps the build under it. Without it the only
|
|
30
|
+
// evidence that anything happened is a screen that scrolled while nobody was
|
|
31
|
+
// looking.
|
|
32
|
+
export function formatTerminalTitle({ index = 0, total = 0, name = null, version = null } = {}) {
|
|
25
33
|
const pos = total > 0 ? `${index + 1}/${total}` : '0/0';
|
|
26
34
|
const who = name ? ` ${truncate(name, 24)}` : '';
|
|
27
|
-
|
|
35
|
+
// Bounded like the account name, for the same reason: a tab strip gives a
|
|
36
|
+
// title a few dozen columns at best, and neither string was chosen here.
|
|
37
|
+
const build = version ? ` ${truncate(version, 20)}` : '';
|
|
38
|
+
return `teamclaude ${pos}${who}${build}`;
|
|
28
39
|
}
|
|
29
40
|
|
|
30
41
|
// Wrap a title string in the OSC set-title sequence, stripping control chars so a
|
package/src/tui.js
CHANGED
|
@@ -213,11 +213,15 @@ const BAR_MAX = 20;
|
|
|
213
213
|
// terminal lays the table out exactly as it did before the column could grow.
|
|
214
214
|
const NAME_MIN = 12;
|
|
215
215
|
|
|
216
|
-
// Clear space the
|
|
217
|
-
//
|
|
218
|
-
// so the whole label is dropped rather than squeezed.
|
|
216
|
+
// Clear space the version label needs on each side before it is drawn at all.
|
|
217
|
+
// Below that it reads as a collision with the title or the port block.
|
|
219
218
|
const HEAD_GAP = 2;
|
|
220
219
|
|
|
220
|
+
// Narrowest label still worth drawing: an ellipsis and three columns of build.
|
|
221
|
+
// Under that the header goes back to naming no build at all, which at those
|
|
222
|
+
// widths is the honest answer.
|
|
223
|
+
const HEAD_LABEL_MIN = 4;
|
|
224
|
+
|
|
221
225
|
// Which pair of bars a row draws: the subscription buckets (Ses/Wk, plus the
|
|
222
226
|
// S7/F7 family bars) when any unified reading exists, else the metered Tok/Req
|
|
223
227
|
// pair an API-key account reports. The account row budget is drawn per
|
|
@@ -281,6 +285,23 @@ export function fitLine(s, w) {
|
|
|
281
285
|
return s;
|
|
282
286
|
}
|
|
283
287
|
|
|
288
|
+
/** The build label at `max` columns, or '' when nothing legible fits.
|
|
289
|
+
*
|
|
290
|
+
* Cut from the LEFT, unlike every other truncation here, because what tells
|
|
291
|
+
* one build from the next is its tail: `…rik.11` still identifies the build,
|
|
292
|
+
* `1.1.2…` identifies the three before it just as well. Sliced by code unit
|
|
293
|
+
* against a display-width budget — a version or a sha is ASCII, and a label
|
|
294
|
+
* arriving over the wire is measured again by the caller before it is placed,
|
|
295
|
+
* so a wide glyph costs the label its slot rather than the header its width.
|
|
296
|
+
* @param {string} label
|
|
297
|
+
* @param {number} max */
|
|
298
|
+
export function fitHeadLabel(label, max) {
|
|
299
|
+
const w = vw(label);
|
|
300
|
+
if (max >= w) return label;
|
|
301
|
+
if (max < HEAD_LABEL_MIN) return '';
|
|
302
|
+
return `…${label.slice(label.length - (max - 1))}`;
|
|
303
|
+
}
|
|
304
|
+
|
|
284
305
|
function formatReset(resetTs) {
|
|
285
306
|
if (!resetTs) return '';
|
|
286
307
|
const ms = resetTs - Date.now();
|
|
@@ -411,6 +432,10 @@ function timestamp() {
|
|
|
411
432
|
|
|
412
433
|
export class TUI {
|
|
413
434
|
constructor({ accountManager, config, saveConfig, syncAccounts, onQuit, sx = null, probeQuota = null, activityLogPath = null,
|
|
435
|
+
// `u`: drain and come back on the new build. Null when nothing would
|
|
436
|
+
// relaunch the process, which is what stops a key offering an update from
|
|
437
|
+
// meaning "kill the proxy and every session on it".
|
|
438
|
+
onRestart = /** @type {(() => void)|null} */ (null),
|
|
414
439
|
// Supervised sidecar state for the conduit lines. A getter, not a snapshot:
|
|
415
440
|
// the supervisor respawns on its own schedule and the TUI redraws on a timer.
|
|
416
441
|
getSidecars = null,
|
|
@@ -435,6 +460,7 @@ export class TUI {
|
|
|
435
460
|
this.saveConfig = saveConfig;
|
|
436
461
|
this.syncAccounts = syncAccounts;
|
|
437
462
|
this.onQuit = onQuit;
|
|
463
|
+
this.onRestart = onRestart; // drain-and-restart, when something supervises us
|
|
438
464
|
this.sx = sx; // sx.org proxy manager (may be null)
|
|
439
465
|
this.sxBalance = null; // last fetched sx.org balance, for the settings screen
|
|
440
466
|
this.probeQuota = probeQuota; // on-demand fleet-wide quota refresh (may be null)
|
|
@@ -704,6 +730,7 @@ export class TUI {
|
|
|
704
730
|
this.mode = 'select'; this.selAction = 'toggle'; this.selIdx = this.am.currentIndex; this.selReturn = 'normal';
|
|
705
731
|
}
|
|
706
732
|
else if (k === 'p' && this.am.accounts.length > 0) { this._doProbe(); }
|
|
733
|
+
else if (k === 'u' && this.onRestart) { this._doRestart(); }
|
|
707
734
|
else if (k === 'g') { this.mode = 'settings'; this.setIdx = 0; this._loadSxBalance(); }
|
|
708
735
|
}
|
|
709
736
|
|
|
@@ -1128,6 +1155,17 @@ export class TUI {
|
|
|
1128
1155
|
}
|
|
1129
1156
|
}
|
|
1130
1157
|
|
|
1158
|
+
// `u`: pick up a new build now instead of waiting for the next lull. The
|
|
1159
|
+
// server owns what happens next — it drains, then exits asking to be
|
|
1160
|
+
// relaunched — and it stops this TUI first, so the line below is on screen
|
|
1161
|
+
// only for the instant before the display goes and the drain reports itself
|
|
1162
|
+
// in plain text.
|
|
1163
|
+
_doRestart() {
|
|
1164
|
+
if (!this.onRestart) return;
|
|
1165
|
+
this._addLog('Draining before restart...');
|
|
1166
|
+
this.onRestart();
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1131
1169
|
// ── Network settings ───────────────────────────────
|
|
1132
1170
|
|
|
1133
1171
|
/**
|
|
@@ -1474,8 +1512,15 @@ export class TUI {
|
|
|
1474
1512
|
// server's. A local AccountManager has neither property.
|
|
1475
1513
|
const label = this.am.versionLabel ?? this.versionLabel;
|
|
1476
1514
|
const upd = this.am.updateAvailable ?? this.updateAvailable;
|
|
1477
|
-
const
|
|
1478
|
-
|
|
1515
|
+
const lw = vw(left), rw = vw(right);
|
|
1516
|
+
// Columns left between the two blocks, and what the label may take of them.
|
|
1517
|
+
// The marker is budgeted before the label is cut, so a shortened label and
|
|
1518
|
+
// its marker still fit the room they were measured against.
|
|
1519
|
+
const room = W - lw - rw;
|
|
1520
|
+
const markerW = upd ? 2 : 0;
|
|
1521
|
+
const text = label ? fitHeadLabel(label, room - 2 * HEAD_GAP - markerW) : '';
|
|
1522
|
+
const mid = text ? dim(text) + (upd ? ` ${green('▲')}` : '') : '';
|
|
1523
|
+
const mw = vw(mid);
|
|
1479
1524
|
// Centred on the line, not in the gap between the two blocks, so the label
|
|
1480
1525
|
// holds still as the session segment comes and goes.
|
|
1481
1526
|
const start = Math.floor((W - mw) / 2);
|
|
@@ -1484,9 +1529,20 @@ export class TUI {
|
|
|
1484
1529
|
// branch can never produce the over-wide line the other branch can, so the
|
|
1485
1530
|
// two are not interchangeable.
|
|
1486
1531
|
const midFits = mw > 0 && start - lw >= HEAD_GAP && (W - rw) - (start + mw) >= HEAD_GAP;
|
|
1532
|
+
// Line-centring is a position, not a fit. The two blocks are different
|
|
1533
|
+
// widths, so a label small enough for the gap can still be pushed inside
|
|
1534
|
+
// one of them by where the centre of the LINE falls — and that, not width,
|
|
1535
|
+
// is what used to drop the label every time the session segment grew,
|
|
1536
|
+
// leaving a header that silently stopped naming the build it exists to
|
|
1537
|
+
// name. Centre it in the GAP instead, which is exact by construction: the
|
|
1538
|
+
// two runs below sum to `room`. The label moves when sessions come and go,
|
|
1539
|
+
// which is the price; being able to read it is what that buys.
|
|
1540
|
+
const gapPad = mw > 0 && room - mw >= 2 * HEAD_GAP ? Math.floor((room - mw) / 2) : -1;
|
|
1487
1541
|
lines.push(midFits
|
|
1488
1542
|
? left + ' '.repeat(start - lw) + mid + ' '.repeat(W - rw - start - mw) + right
|
|
1489
|
-
:
|
|
1543
|
+
: gapPad >= 0
|
|
1544
|
+
? left + ' '.repeat(gapPad) + mid + ' '.repeat(room - mw - gapPad) + right
|
|
1545
|
+
: left + ' '.repeat(Math.max(1, W - lw - rw)) + right);
|
|
1490
1546
|
lines.push(' ' + dim('─'.repeat(W - 2)));
|
|
1491
1547
|
|
|
1492
1548
|
const footerH = 2;
|
|
@@ -2266,7 +2322,7 @@ export class TUI {
|
|
|
2266
2322
|
case 'normal':
|
|
2267
2323
|
return this.remote
|
|
2268
2324
|
? ` ${bold('s')}witch ${bold('R')}eload ${bold('q')}uit`
|
|
2269
|
-
: ` ${bold('s')}witch ${bold('d')}isable ${bold('p')}robe quota ${bold('R')}eload ${bold('g')} settings ${bold('q')}uit`;
|
|
2325
|
+
: ` ${bold('s')}witch ${bold('d')}isable ${bold('p')}robe quota ${bold('R')}eload${this.onRestart ? ` ${bold('u')}pdate` : ''} ${bold('g')} settings ${bold('q')}uit`;
|
|
2270
2326
|
case 'settings':
|
|
2271
2327
|
return ` ${dim('↑↓')} navigate ${dim('←→')} change ${bold('Enter')} edit ${bold('Esc')} back`;
|
|
2272
2328
|
case 'routes':
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
// What tells a running proxy that a newer build is waiting for it.
|
|
2
|
+
//
|
|
3
|
+
// The signal is not the same on every install, and picking only one of them
|
|
4
|
+
// would leave the primary user with dead code:
|
|
5
|
+
//
|
|
6
|
+
// global npm install — `npm install -g` replaces the files under the running
|
|
7
|
+
// process, so the version in the package's own package.json stops matching
|
|
8
|
+
// the version this process loaded. That difference, not the registry, is
|
|
9
|
+
// the signal: a published release nothing has installed yet is not
|
|
10
|
+
// something a restart can apply. autoUpdate (updater.js) does the
|
|
11
|
+
// installing; this only notices that it landed.
|
|
12
|
+
// git checkout — npm refuses to touch one by design, so there is no installed
|
|
13
|
+
// version to compare. The deploy clone's HEAD moving is the signal, read
|
|
14
|
+
// with one cheap `git rev-parse` on a timer.
|
|
15
|
+
//
|
|
16
|
+
// Both answer the same question — "would restarting run something else?" — so
|
|
17
|
+
// both feed the same drain-and-restart, and the install kind picks which is
|
|
18
|
+
// asked.
|
|
19
|
+
//
|
|
20
|
+
// Detecting a new build is not permission to apply it. A restart costs every
|
|
21
|
+
// live session its connection, so it waits for the fleet to go quiet; a fleet
|
|
22
|
+
// that never does gets restarted anyway, because a proxy that updates only
|
|
23
|
+
// when nobody is using it never updates at all.
|
|
24
|
+
|
|
25
|
+
import { execFile } from 'node:child_process';
|
|
26
|
+
import { promisify } from 'node:util';
|
|
27
|
+
import { compareVersions, currentVersion, installKind, isReleaseVersion, packageRoot } from './updater.js';
|
|
28
|
+
|
|
29
|
+
const pexec = promisify(execFile);
|
|
30
|
+
|
|
31
|
+
// How often the source is asked. A git checkout costs one `rev-parse` and a
|
|
32
|
+
// global install one small file read, so this is cheap enough to be frequent
|
|
33
|
+
// and slow enough to be invisible.
|
|
34
|
+
export const UPDATE_POLL_MS = 60_000;
|
|
35
|
+
|
|
36
|
+
// How long the fleet must stay quiet before a pending build is applied. The
|
|
37
|
+
// session tracker already calls a session inactive after two idle minutes
|
|
38
|
+
// (SESSION_ACTIVE_TTL_MS), so this is the quiet period ON TOP of that — proof
|
|
39
|
+
// that the lull held rather than a gap between two turns of one conversation.
|
|
40
|
+
export const FLEET_IDLE_MS = 60_000;
|
|
41
|
+
|
|
42
|
+
// ...and how long a pending build waits for that quiet before going anyway. A
|
|
43
|
+
// busy fleet must not be able to postpone an update forever; at this point the
|
|
44
|
+
// drain is what protects the sessions, not the wait.
|
|
45
|
+
export const RESTART_DEADLINE_MS = 30 * 60_000;
|
|
46
|
+
|
|
47
|
+
// Bounded like every other git call in this package: a checkout on a stalled
|
|
48
|
+
// network mount must cost the proxy a skipped check, not a stuck timer.
|
|
49
|
+
const GIT_TIMEOUT_MS = 2000;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* @typedef {Object} VersionSource
|
|
53
|
+
* @property {string} kind the install shape this reads
|
|
54
|
+
* @property {string} describes what it names, for the log line
|
|
55
|
+
* @property {() => Promise<string|null>} read the build a restart would run, or null when that is the one already running
|
|
56
|
+
*/
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* A git checkout: HEAD's sha, compared against the sha this process started on.
|
|
60
|
+
*
|
|
61
|
+
* The baseline is taken on the first successful read rather than in the
|
|
62
|
+
* constructor, because reading is asynchronous and a watcher that had no
|
|
63
|
+
* baseline yet would read the first pull as "nothing changed". A first read
|
|
64
|
+
* that fails (no git binary, a broken checkout) leaves the baseline unset, so
|
|
65
|
+
* the next successful one becomes it — one missed pull rather than a watcher
|
|
66
|
+
* that never works again.
|
|
67
|
+
*
|
|
68
|
+
* @param {Object} [opts]
|
|
69
|
+
* @param {string} [opts.root]
|
|
70
|
+
* @param {(file: string, args: string[], options: { cwd: string, encoding: 'utf8', timeout: number }) => Promise<{ stdout: string }>} [opts.exec]
|
|
71
|
+
* @returns {VersionSource}
|
|
72
|
+
*/
|
|
73
|
+
export function gitHeadSource({ root = packageRoot(), exec = pexec } = {}) {
|
|
74
|
+
let baseline = null;
|
|
75
|
+
return {
|
|
76
|
+
kind: 'git',
|
|
77
|
+
describes: 'the checkout HEAD',
|
|
78
|
+
async read() {
|
|
79
|
+
let head = null;
|
|
80
|
+
try {
|
|
81
|
+
/** @type {{ cwd: string, encoding: 'utf8', timeout: number }} */
|
|
82
|
+
const opts = { cwd: root, encoding: 'utf8', timeout: GIT_TIMEOUT_MS };
|
|
83
|
+
head = String((await exec('git', ['rev-parse', 'HEAD'], opts)).stdout).trim();
|
|
84
|
+
} catch { return null; } // no git, no repository, or it took too long
|
|
85
|
+
// Shape-checked before it is compared: anything else is git reporting a
|
|
86
|
+
// problem on stdout, and treating that as a sha would restart the proxy
|
|
87
|
+
// on every poll for as long as the problem lasted.
|
|
88
|
+
if (!/^[0-9a-f]{7,64}$/.test(head)) return null;
|
|
89
|
+
baseline ||= head;
|
|
90
|
+
return head === baseline ? null : head;
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* A global npm install: the version on disk at the package root.
|
|
97
|
+
*
|
|
98
|
+
* `currentVersion` is re-read every time on purpose — that file is what
|
|
99
|
+
* `npm install -g` rewrites under a running process, and the whole signal is
|
|
100
|
+
* that it stopped matching what this process loaded. Only a newer release
|
|
101
|
+
* counts: a reinstall of the same version changes nothing worth a restart, and
|
|
102
|
+
* a downgrade or a version string that is not a release is not something to
|
|
103
|
+
* relaunch into.
|
|
104
|
+
*
|
|
105
|
+
* @param {Object} [opts]
|
|
106
|
+
* @param {string} [opts.root]
|
|
107
|
+
* @param {string|null} [opts.running] the version this process is executing
|
|
108
|
+
* @returns {VersionSource}
|
|
109
|
+
*/
|
|
110
|
+
export function npmVersionSource({ root = packageRoot(), running = currentVersion(root) } = {}) {
|
|
111
|
+
return {
|
|
112
|
+
kind: 'global',
|
|
113
|
+
describes: 'the installed package',
|
|
114
|
+
async read() {
|
|
115
|
+
const installed = currentVersion(root);
|
|
116
|
+
if (!installed || !running || !isReleaseVersion(installed)) return null;
|
|
117
|
+
return compareVersions(installed, running) > 0 ? installed : null;
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* The source for how this copy was installed, or null when a restart could not
|
|
124
|
+
* pick anything up: a local dependency or an npx copy is nobody's deployment,
|
|
125
|
+
* and nothing rewrites it under us.
|
|
126
|
+
*
|
|
127
|
+
* @param {Object} [opts]
|
|
128
|
+
* @param {string} [opts.root]
|
|
129
|
+
* @param {Function} [opts.kind]
|
|
130
|
+
* @param {Function} [opts.exec]
|
|
131
|
+
* @param {string|null} [opts.running]
|
|
132
|
+
* @returns {Promise<VersionSource|null>}
|
|
133
|
+
*/
|
|
134
|
+
export async function createVersionSource({ root = packageRoot(), kind = installKind, exec = pexec, running = currentVersion(root) } = {}) {
|
|
135
|
+
const how = await kind({ root });
|
|
136
|
+
if (how === 'git') return gitHeadSource({ root, exec: /** @type {any} */ (exec) });
|
|
137
|
+
if (how === 'global') return npmVersionSource({ root, running });
|
|
138
|
+
return null;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Polls a source and asks for a restart once a new build is waiting AND the
|
|
143
|
+
* fleet is quiet enough to lose its connections cheaply.
|
|
144
|
+
*
|
|
145
|
+
* Fires exactly once: the restart it asks for ends the process, and a second
|
|
146
|
+
* request during the drain would be noise at best.
|
|
147
|
+
*/
|
|
148
|
+
export class UpdateWatcher {
|
|
149
|
+
/**
|
|
150
|
+
* @param {Object} opts
|
|
151
|
+
* @param {VersionSource|null} opts.source
|
|
152
|
+
* @param {() => boolean} opts.isIdle is the fleet serving nothing right now
|
|
153
|
+
* @param {(info: { build: string, forced: boolean, waitedMs: number }) => void} opts.onRestart
|
|
154
|
+
* @param {number} [opts.pollMs]
|
|
155
|
+
* @param {number} [opts.idleMs]
|
|
156
|
+
* @param {number} [opts.deadlineMs]
|
|
157
|
+
* @param {() => number} [opts.now]
|
|
158
|
+
* @param {(line: string) => void} [opts.log]
|
|
159
|
+
*/
|
|
160
|
+
constructor({
|
|
161
|
+
source, isIdle, onRestart, pollMs = UPDATE_POLL_MS, idleMs = FLEET_IDLE_MS,
|
|
162
|
+
deadlineMs = RESTART_DEADLINE_MS, now = Date.now, log = console.log,
|
|
163
|
+
}) {
|
|
164
|
+
this.source = source;
|
|
165
|
+
this.isIdle = isIdle;
|
|
166
|
+
this.onRestart = onRestart;
|
|
167
|
+
this.pollMs = pollMs;
|
|
168
|
+
this.idleMs = idleMs;
|
|
169
|
+
this.deadlineMs = deadlineMs;
|
|
170
|
+
this.now = now;
|
|
171
|
+
this.log = log;
|
|
172
|
+
/** @type {ReturnType<typeof setInterval>|null} */
|
|
173
|
+
this.timer = null;
|
|
174
|
+
/** @type {string|null} the build waiting to be picked up */
|
|
175
|
+
this._build = null;
|
|
176
|
+
// Timestamps, and null rather than 0 for "not yet": a clock can legitimately
|
|
177
|
+
// read 0, and a falsy check would then re-stamp both of these on every poll
|
|
178
|
+
// — the deadline would never arrive and the quiet period never accumulate.
|
|
179
|
+
/** @type {number|null} when the build was first seen, for the deadline */
|
|
180
|
+
this._pendingSince = null;
|
|
181
|
+
/** @type {number|null} when the fleet last went quiet, for the wait */
|
|
182
|
+
this._idleSince = null;
|
|
183
|
+
this._fired = false;
|
|
184
|
+
this._busy = false; // a read is outstanding; never overlap them
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** No-op without a source, so the caller needs no second condition. */
|
|
188
|
+
start() {
|
|
189
|
+
if (!this.source || this.timer) return;
|
|
190
|
+
// Primes the baseline (see gitHeadSource) rather than waiting a full poll
|
|
191
|
+
// to learn what is already running.
|
|
192
|
+
this.check();
|
|
193
|
+
this.timer = setInterval(() => { this.check(); }, this.pollMs);
|
|
194
|
+
this.timer.unref?.();
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
stop() {
|
|
198
|
+
if (this.timer) { clearInterval(this.timer); this.timer = null; }
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** One poll. Public so a test can drive the decision without spending the
|
|
202
|
+
* interval, the idle wait and the deadline in real time. */
|
|
203
|
+
async check() {
|
|
204
|
+
if (this._fired || this._busy || !this.source) return null;
|
|
205
|
+
this._busy = true;
|
|
206
|
+
try {
|
|
207
|
+
const build = await this.source.read();
|
|
208
|
+
if (build) this._build = build;
|
|
209
|
+
// Nothing waiting: whatever idleness was accumulating is not evidence
|
|
210
|
+
// about a build, so it starts again with the next one.
|
|
211
|
+
if (!this._build) { this._idleSince = null; return null; }
|
|
212
|
+
|
|
213
|
+
const now = this.now();
|
|
214
|
+
if (this._pendingSince === null) {
|
|
215
|
+
this._pendingSince = now;
|
|
216
|
+
this.log(`[TeamClaude] A newer build is waiting (${this._build}) — restarting once the fleet is idle, or in ${Math.round(this.deadlineMs / 60_000)} min regardless.`);
|
|
217
|
+
}
|
|
218
|
+
if (this.isIdle()) this._idleSince ??= now; else this._idleSince = null;
|
|
219
|
+
|
|
220
|
+
const settled = this._idleSince !== null && now - this._idleSince >= this.idleMs;
|
|
221
|
+
const overdue = now - this._pendingSince >= this.deadlineMs;
|
|
222
|
+
if (!settled && !overdue) return null;
|
|
223
|
+
|
|
224
|
+
this._fired = true;
|
|
225
|
+
this.stop();
|
|
226
|
+
this.onRestart({ build: this._build, forced: !settled, waitedMs: now - this._pendingSince });
|
|
227
|
+
return this._build;
|
|
228
|
+
} finally {
|
|
229
|
+
this._busy = false;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
package/src/updater.js
CHANGED
|
@@ -29,7 +29,7 @@ const LABEL_MAX = 32;
|
|
|
29
29
|
const pexec = promisify(execFile);
|
|
30
30
|
|
|
31
31
|
/** Package root = one directory above this file's src/ directory. */
|
|
32
|
-
function packageRoot() {
|
|
32
|
+
export function packageRoot() {
|
|
33
33
|
return resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
34
34
|
}
|
|
35
35
|
|