maxpool 1.5.80 → 1.5.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +9 -4
- package/src/index.js +7 -0
- package/src/restart-controller.js +44 -0
- package/src/secret-resolver.js +65 -5
- package/src/server.js +34 -9
- package/src/tui.js +106 -26
package/package.json
CHANGED
package/src/account-manager.js
CHANGED
|
@@ -1487,10 +1487,15 @@ export class AccountManager {
|
|
|
1487
1487
|
const c = { total: 0, quota: 0, transient: 0, network: 0, disabled: 0, error: 0, other: 0 };
|
|
1488
1488
|
for (const a of this.accounts) {
|
|
1489
1489
|
if (a.type === 'provider') continue;
|
|
1490
|
+
// DISABLED accounts are out of the serving pool — counting them in `total`
|
|
1491
|
+
// made the 429 say "all 9 Claude accounts are momentarily busy" when ONE
|
|
1492
|
+
// enabled account was reconnecting and EIGHT were disabled with dead tokens
|
|
1493
|
+
// (reported 2026-08-13). The user reads 9-busy as fleet saturation; the
|
|
1494
|
+
// truth is 1-busy + 8-off. Track them for the message, exclude from cause math.
|
|
1495
|
+
if (a.enabled === false) { c.disabled++; continue; }
|
|
1490
1496
|
c.total++;
|
|
1491
1497
|
const cause = this._retryInfo(a, model)?.cause || 'unavailable';
|
|
1492
|
-
if (cause === '
|
|
1493
|
-
else if (cause === 'error') c.error++;
|
|
1498
|
+
if (cause === 'error') c.error++;
|
|
1494
1499
|
else if (QUOTA.has(cause)) c.quota++;
|
|
1495
1500
|
else if (TRANSIENT.has(cause)) {
|
|
1496
1501
|
c.transient++;
|
|
@@ -1500,7 +1505,7 @@ export class AccountManager {
|
|
|
1500
1505
|
&& (a.cooldownUntil - Date.now()) <= this.scheduler.networkCooldownMs) c.network++;
|
|
1501
1506
|
} else c.other++;
|
|
1502
1507
|
}
|
|
1503
|
-
const eligible = c.total - c.
|
|
1508
|
+
const eligible = c.total - c.error;
|
|
1504
1509
|
c.dominant = eligible <= 0 ? 'none'
|
|
1505
1510
|
: c.quota >= Math.max(1, Math.ceil(eligible / 2)) ? 'quota'
|
|
1506
1511
|
: c.transient > 0 ? 'transient' : 'other';
|
|
@@ -3185,7 +3190,7 @@ export class AccountManager {
|
|
|
3185
3190
|
// Default model maps — match what `cc all` sends via headers. Users can override
|
|
3186
3191
|
// per-provider in config. Without these z.ai returns [1210 Invalid API parameter].
|
|
3187
3192
|
const defaultModelMap = entry.provider === 'zai'
|
|
3188
|
-
? { opus: 'glm-5.
|
|
3193
|
+
? { opus: 'glm-5.3', sonnet: 'glm-5.3', haiku: 'glm-5.3', default: 'glm-5.3' }
|
|
3189
3194
|
: null;
|
|
3190
3195
|
const defaultModel = entry.provider === 'kimi' ? 'kimi-k3' : null;
|
|
3191
3196
|
this.upsertRuntimeAccount({
|
package/src/index.js
CHANGED
|
@@ -985,6 +985,13 @@ async function serverWorkerCommand() {
|
|
|
985
985
|
pauseAdmission: () => accountManager.setAdmissionPaused(true),
|
|
986
986
|
resumeAdmission: () => accountManager.setAdmissionPaused(false),
|
|
987
987
|
restartNow: requestReload,
|
|
988
|
+
// The pre-drain only earns its cost on the COLD path (the socket closes, so an
|
|
989
|
+
// in-flight request would be severed). Seamless keeps serving them post-baton.
|
|
990
|
+
isSeamless: () => {
|
|
991
|
+
const forceCold = process.env.MAXPOOL_TEST_FORCE_COLD_RESTART === '1'
|
|
992
|
+
|| process.env.MAXPOOL_TUI_COLD_RESTART === '1';
|
|
993
|
+
return !forceCold && reloadStrategy({ supervised }) === 'seamless';
|
|
994
|
+
},
|
|
988
995
|
// Configurable so ops can tune the bounded pre-restart drain, and so the
|
|
989
996
|
// integration test can exercise the force-restart path without a 10s wait.
|
|
990
997
|
...(Number.isFinite(config.restartDrainTimeoutMs) ? { drainTimeoutMs: config.restartDrainTimeoutMs } : {}),
|
|
@@ -12,16 +12,31 @@ export class RestartController {
|
|
|
12
12
|
// heartbeat — pins the restart on "Restart pending…" forever (the stuck-`r`
|
|
13
13
|
// bug). Dropped requests reconnect after restart, as the message promises.
|
|
14
14
|
drainTimeoutMs = 10_000,
|
|
15
|
+
// When the reload is SEAMLESS, the old worker keeps serving its in-flight
|
|
16
|
+
// requests through the baton handoff (releaseBatonAndDrain, 60s+ cap) — so
|
|
17
|
+
// pre-draining here buys nothing and costs every OTHER session a hard 503 for
|
|
18
|
+
// the whole window. With ~30 sessions running 30-60s requests the 10s drain
|
|
19
|
+
// NEVER completes naturally; it times out every time. So it was 10 guaranteed
|
|
20
|
+
// seconds of "Maxpool is restarting" across the fleet in exchange for zero
|
|
21
|
+
// drained requests — the reported 503 storm. Returns true when the restart
|
|
22
|
+
// path will hand off seamlessly, in which case we skip straight to the swap.
|
|
23
|
+
isSeamless = () => false,
|
|
15
24
|
setTimeoutFn = setTimeout,
|
|
16
25
|
clearTimeoutFn = clearTimeout,
|
|
26
|
+
setIntervalFn = setInterval,
|
|
27
|
+
clearIntervalFn = clearInterval,
|
|
17
28
|
}) {
|
|
18
29
|
this.pauseAdmission = pauseAdmission;
|
|
19
30
|
this.resumeAdmission = resumeAdmission;
|
|
20
31
|
this.restartNow = restartNow;
|
|
21
32
|
this.log = log;
|
|
22
33
|
this.drainTimeoutMs = drainTimeoutMs;
|
|
34
|
+
this.isSeamless = isSeamless;
|
|
23
35
|
this.setTimeoutFn = setTimeoutFn;
|
|
24
36
|
this.clearTimeoutFn = clearTimeoutFn;
|
|
37
|
+
this.setIntervalFn = setIntervalFn;
|
|
38
|
+
this.clearIntervalFn = clearIntervalFn;
|
|
39
|
+
this._progressTimer = null;
|
|
25
40
|
this.activeRequests = new Set();
|
|
26
41
|
this.upstreamRequests = new Set();
|
|
27
42
|
this.pending = false;
|
|
@@ -48,6 +63,7 @@ export class RestartController {
|
|
|
48
63
|
this.pending = false;
|
|
49
64
|
this.restarting = false;
|
|
50
65
|
if (this._drainTimer) { this.clearTimeoutFn(this._drainTimer); this._drainTimer = null; }
|
|
66
|
+
if (this._progressTimer) { this.clearIntervalFn(this._progressTimer); this._progressTimer = null; }
|
|
51
67
|
this.resumeAdmission();
|
|
52
68
|
return true;
|
|
53
69
|
}
|
|
@@ -77,9 +93,33 @@ export class RestartController {
|
|
|
77
93
|
return;
|
|
78
94
|
}
|
|
79
95
|
|
|
96
|
+
// Seamless handoff → do not pre-drain (see isSeamless above). The in-flight
|
|
97
|
+
// requests finish on the OLD worker after the baton passes; holding admission
|
|
98
|
+
// here would only 503 the rest of the fleet for nothing.
|
|
99
|
+
if (this.isSeamless()) {
|
|
100
|
+
this.log(`[Maxpool] Restarting now — ${this.upstreamRequests.size} in-flight request(s) finish on the current version; new requests go to the updated one.`);
|
|
101
|
+
this._restart();
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
|
|
80
105
|
this.pending = true;
|
|
81
106
|
const queuedOrIdle = Math.max(0, this.activeRequests.size - this.upstreamRequests.size);
|
|
82
107
|
this.log(`[Maxpool] Restart pending; admission paused while ${this.upstreamRequests.size} upstream request(s) finish (up to ${Math.round(this.drainTimeoutMs / 1000)}s). ${queuedOrIdle} queued/idle request(s) will reconnect after restart.`);
|
|
108
|
+
// Progress ticks while draining. Without these the TUI shows one line and then
|
|
109
|
+
// silence for the whole window, which reads as "nothing happened" — the reported
|
|
110
|
+
// complaint. A countdown makes the wait legible and bounded.
|
|
111
|
+
this._tick = 0;
|
|
112
|
+
const tickMs = 2000;
|
|
113
|
+
this._progressTimer = this.setIntervalFn?.(() => {
|
|
114
|
+
if (!this.pending || this.restarting) return;
|
|
115
|
+
this._tick += tickMs;
|
|
116
|
+
const left = Math.max(0, Math.ceil((this.drainTimeoutMs - this._tick) / 1000));
|
|
117
|
+
const n = this.upstreamRequests.size;
|
|
118
|
+
if (n === 0) return; // _maybeRestart is about to fire
|
|
119
|
+
this.log(`[Maxpool] Restarting — waiting for ${n} request(s) to finish (${left}s left)…`);
|
|
120
|
+
}, tickMs);
|
|
121
|
+
this._progressTimer?.unref?.();
|
|
122
|
+
|
|
83
123
|
// Force the restart if the drain overruns — never hang on a long stream.
|
|
84
124
|
this._drainTimer = this.setTimeoutFn(() => {
|
|
85
125
|
if (!this.pending || this.restarting) return;
|
|
@@ -102,6 +142,10 @@ export class RestartController {
|
|
|
102
142
|
this.clearTimeoutFn(this._drainTimer);
|
|
103
143
|
this._drainTimer = null;
|
|
104
144
|
}
|
|
145
|
+
if (this._progressTimer) {
|
|
146
|
+
this.clearIntervalFn(this._progressTimer);
|
|
147
|
+
this._progressTimer = null;
|
|
148
|
+
}
|
|
105
149
|
this.restartNow();
|
|
106
150
|
}
|
|
107
151
|
}
|
package/src/secret-resolver.js
CHANGED
|
@@ -10,13 +10,59 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { execFile } from 'node:child_process';
|
|
12
12
|
import { promisify } from 'node:util';
|
|
13
|
+
import { readFileSync } from 'node:fs';
|
|
14
|
+
import { homedir } from 'node:os';
|
|
15
|
+
import { join } from 'node:path';
|
|
13
16
|
|
|
14
17
|
const execFileAsync = promisify(execFile);
|
|
15
18
|
|
|
16
19
|
const DEFAULT_PROJECT = 'mokka-business-automations';
|
|
17
20
|
|
|
18
|
-
|
|
21
|
+
// A single `gcloud secrets versions access` costs ~17-33s on this machine (python
|
|
22
|
+
// interpreter start + an ADC round-trip), and running them CONCURRENTLY makes it
|
|
23
|
+
// worse, not better: measured 2026-08-10, 5 secrets in parallel → 3 of 5 hit the
|
|
24
|
+
// 45s timeout; the same 5 sequentially → 4 of 5. Either way providers boot as
|
|
25
|
+
// "secret-unresolved", which is what stranded `kimi max@gomokka.com` and let the
|
|
26
|
+
// header path create a duplicate `kimi-fallback` row beside it.
|
|
27
|
+
//
|
|
28
|
+
// The workspace already maintains a local plaintext cache of these same secrets
|
|
29
|
+
// (written by scripts/load-secrets.sh, mode 0600, sanctioned by the secrets-directory
|
|
30
|
+
// rule). Reading it is sub-millisecond and needs no auth, so try it FIRST and fall
|
|
31
|
+
// back to gcloud only for what it does not carry.
|
|
32
|
+
let cacheMemo;
|
|
33
|
+
let cachePathMemo;
|
|
34
|
+
|
|
35
|
+
function cachePath() {
|
|
36
|
+
if (!cachePathMemo) cachePathMemo = join(homedir(), '.claude', '.credentials-cache');
|
|
37
|
+
return cachePathMemo;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Test seam: drop the memo so a re-read picks up a freshly-written cache. */
|
|
41
|
+
export function __resetSecretCache() { cacheMemo = undefined; cachePathMemo = undefined; }
|
|
42
|
+
|
|
43
|
+
function readCredentialCache() {
|
|
44
|
+
if (cacheMemo !== undefined) return cacheMemo;
|
|
45
|
+
try {
|
|
46
|
+
const parsed = JSON.parse(readFileSync(cachePath(), 'utf8'));
|
|
47
|
+
cacheMemo = (parsed && typeof parsed === 'object') ? parsed : null;
|
|
48
|
+
} catch {
|
|
49
|
+
cacheMemo = null; // absent/unreadable/corrupt → gcloud path, never a crash
|
|
50
|
+
}
|
|
51
|
+
return cacheMemo;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function fromCache(secretName) {
|
|
55
|
+
const cache = readCredentialCache();
|
|
56
|
+
const v = cache?.[secretName];
|
|
57
|
+
return (typeof v === 'string' && v.trim()) ? v.trim() : null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export async function resolveSecret(secretName, { project = DEFAULT_PROJECT, timeoutMs = 45_000, useCache = true } = {}) {
|
|
19
61
|
if (!secretName || typeof secretName !== 'string') return null;
|
|
62
|
+
if (useCache) {
|
|
63
|
+
const cached = fromCache(secretName);
|
|
64
|
+
if (cached) return cached;
|
|
65
|
+
}
|
|
20
66
|
try {
|
|
21
67
|
const { stdout } = await execFileAsync(
|
|
22
68
|
'gcloud',
|
|
@@ -42,8 +88,22 @@ export async function resolveSecret(secretName, { project = DEFAULT_PROJECT, tim
|
|
|
42
88
|
export async function resolveSecrets(secretNames, opts = {}) {
|
|
43
89
|
const unique = [...new Set(secretNames.filter(Boolean))];
|
|
44
90
|
if (!unique.length) return {};
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
91
|
+
|
|
92
|
+
// Cache hits first — free, and on this machine that is usually all of them.
|
|
93
|
+
const out = {};
|
|
94
|
+
const misses = [];
|
|
95
|
+
for (const name of unique) {
|
|
96
|
+
const cached = opts.useCache === false ? null : fromCache(name);
|
|
97
|
+
if (cached) out[name] = cached;
|
|
98
|
+
else misses.push(name);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Only genuine misses pay gcloud, and SEQUENTIALLY — concurrent invocations
|
|
102
|
+
// contend and time each other out (measured: 3 of 5 failed in parallel vs 1 of 5
|
|
103
|
+
// sequentially). Sequential over a handful of misses is strictly better here.
|
|
104
|
+
for (const name of misses) {
|
|
105
|
+
const v = await resolveSecret(name, { ...opts, useCache: false });
|
|
106
|
+
if (v != null) out[name] = v;
|
|
107
|
+
}
|
|
108
|
+
return out;
|
|
49
109
|
}
|
package/src/server.js
CHANGED
|
@@ -211,7 +211,7 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
211
211
|
type: 'error',
|
|
212
212
|
error: {
|
|
213
213
|
type: 'restart_in_progress',
|
|
214
|
-
message: 'Maxpool is restarting.
|
|
214
|
+
message: 'Maxpool is restarting — finishing in-flight requests first. This retries automatically; your session is not lost.',
|
|
215
215
|
},
|
|
216
216
|
}));
|
|
217
217
|
return;
|
|
@@ -247,7 +247,7 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
247
247
|
type: 'error',
|
|
248
248
|
error: {
|
|
249
249
|
type: 'restart_in_progress',
|
|
250
|
-
message: 'Maxpool is restarting.
|
|
250
|
+
message: 'Maxpool is restarting — finishing in-flight requests first. This retries automatically; your session is not lost.',
|
|
251
251
|
},
|
|
252
252
|
}));
|
|
253
253
|
return;
|
|
@@ -1555,7 +1555,9 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
|
|
|
1555
1555
|
return `This session is too large for the GLM/Kimi fallbacks (their ~256K limit) — it needs a 1M-context Claude account, and they're all busy right now.${eta} It sends as soon as one frees; /compact shortens the session if you'd rather not wait.`;
|
|
1556
1556
|
}
|
|
1557
1557
|
|
|
1558
|
-
|
|
1558
|
+
// ENABLED only — a disabled account is not "at its limit", it is off; counting it
|
|
1559
|
+
// made a 1-enabled-account pool report "all 9 accounts at their limit".
|
|
1560
|
+
const claudeCount = accountManager.accounts.filter(a => a.type !== 'provider' && a.enabled !== false).length;
|
|
1559
1561
|
// Only name the providers when this pool HAS them AND they are actually allowed to
|
|
1560
1562
|
// serve this request. With crossProviderFallbackPolicy 'never' (the default) they are
|
|
1561
1563
|
// barred by POLICY, not saturated — saying they are "at their limit" is a lie that
|
|
@@ -1592,12 +1594,20 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
|
|
|
1592
1594
|
if (census && census.dominant === 'transient') {
|
|
1593
1595
|
const netly = census.network > 0;
|
|
1594
1596
|
const what = netly
|
|
1595
|
-
? `${census.network} of ${census.total} Claude
|
|
1596
|
-
:
|
|
1597
|
-
|
|
1597
|
+
? `${census.network} of ${census.total} Claude account${census.total === 1 ? '' : 's'} are in a brief reconnect cooldown`
|
|
1598
|
+
: census.total === 1
|
|
1599
|
+
? 'the only enabled Claude account is momentarily busy'
|
|
1600
|
+
: `all ${census.total} Claude accounts are momentarily busy`;
|
|
1601
|
+
// When most of the pool is DISABLED, "1 account busy" begs the real question —
|
|
1602
|
+
// name the disabled share so the fix (re-enable/re-auth) is visible in the message
|
|
1603
|
+
// instead of reading like a fleet that mysteriously shrank.
|
|
1604
|
+
const offNote = census.disabled > 0
|
|
1605
|
+
? ` (${census.disabled} account${census.disabled === 1 ? '' : 's'} disabled)`
|
|
1606
|
+
: '';
|
|
1607
|
+
return `No account can take this request right now — ${what}, not out of quota${offNote}. Retry in ~${formatRetryDuration(retryAfter)}; it clears on its own.${providersBarredHint}`;
|
|
1598
1608
|
}
|
|
1599
1609
|
|
|
1600
|
-
return `No account can take this request right now — all ${claudeCount} Claude
|
|
1610
|
+
return `No account can take this request right now — all ${claudeCount} Claude account${claudeCount === 1 ? '' : 's'}${providersClause} are ${waitLong ? 'at their limit' : 'momentarily at their limit'}. Retry in ~${formatRetryDuration(retryAfter)}.${providersBarredHint}`;
|
|
1601
1611
|
}
|
|
1602
1612
|
|
|
1603
1613
|
// A provider (GLM/Kimi) rejecting a request whose token count exceeds its context
|
|
@@ -2612,9 +2622,9 @@ function prepareRuntimeProviders(accountManager, headers) {
|
|
|
2612
2622
|
|
|
2613
2623
|
const zaiToken = headerValue(headers, 'x-maxpool-zai-token');
|
|
2614
2624
|
if (zaiToken && !configTokens.has(zaiToken)) {
|
|
2615
|
-
const opus = headerValue(headers, 'x-maxpool-zai-opus-model') || headerValue(headers, 'x-maxpool-zai-model') || 'glm-5.
|
|
2625
|
+
const opus = headerValue(headers, 'x-maxpool-zai-opus-model') || headerValue(headers, 'x-maxpool-zai-model') || 'glm-5.3';
|
|
2616
2626
|
const sonnet = headerValue(headers, 'x-maxpool-zai-sonnet-model') || headerValue(headers, 'x-maxpool-zai-model') || opus;
|
|
2617
|
-
const haiku = headerValue(headers, 'x-maxpool-zai-haiku-model') || 'glm-5.
|
|
2627
|
+
const haiku = headerValue(headers, 'x-maxpool-zai-haiku-model') || 'glm-5.3';
|
|
2618
2628
|
accountManager.upsertRuntimeAccount({
|
|
2619
2629
|
name: 'glm-fallback',
|
|
2620
2630
|
type: 'provider',
|
|
@@ -2739,6 +2749,7 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
2739
2749
|
let committed = res.headersSent;
|
|
2740
2750
|
let readFailed = false;
|
|
2741
2751
|
|
|
2752
|
+
|
|
2742
2753
|
// We're now committed to streaming a real upstream response body onto this
|
|
2743
2754
|
// response — there is no more failover for this forward. Stop the queue
|
|
2744
2755
|
// heartbeat (if this was a resumed held stream) BEFORE the first real byte, so
|
|
@@ -2854,6 +2865,20 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
2854
2865
|
reader.cancel().catch(() => {});
|
|
2855
2866
|
if (!readFailed) {
|
|
2856
2867
|
if (!committed && !res.headersSent) res.writeHead(status, responseHeaders);
|
|
2868
|
+
// Truncation sentinel FIRST (before end()) — a write after end() throws and
|
|
2869
|
+
// would be swallowed, silently producing the truncated stream this guards.
|
|
2870
|
+
// Fires ONLY on a genuinely truncated stream: a non-empty residual SSE buffer
|
|
2871
|
+
// means the upstream died mid-event (an event without its '\n\n' terminator).
|
|
2872
|
+
// Complete-event streams without a terminal message_stop (z.ai compat shape,
|
|
2873
|
+
// test fixtures) end with an EMPTY residual and pass through unchanged.
|
|
2874
|
+
if ((committed || res.headersSent) && sseBuffer.trim() && !res.destroyed && !res.writableEnded) {
|
|
2875
|
+
try {
|
|
2876
|
+
res.write(`event: error\ndata: ${JSON.stringify({
|
|
2877
|
+
type: 'error',
|
|
2878
|
+
error: { type: 'api_error', message: 'Upstream closed the stream before the response completed. Retry the message.' },
|
|
2879
|
+
})}\n\n`);
|
|
2880
|
+
} catch { /* client already gone */ }
|
|
2881
|
+
}
|
|
2857
2882
|
if (!res.writableEnded) res.end();
|
|
2858
2883
|
}
|
|
2859
2884
|
}
|
package/src/tui.js
CHANGED
|
@@ -426,7 +426,12 @@ export class TUI {
|
|
|
426
426
|
} else if (k === 'r') {
|
|
427
427
|
this._confirm(
|
|
428
428
|
'Restart Maxpool?',
|
|
429
|
-
|
|
429
|
+
// Say what actually happens to the user's sessions. The old text ("pause new
|
|
430
|
+
// requests, drain active work") was accurate but hid the consequence: with
|
|
431
|
+
// several long requests in flight, NEW requests get a 503 for up to the drain
|
|
432
|
+
// window, and every running session sees a retry. Reported 2026-08-10: "I click
|
|
433
|
+
// restart, then yes, and NOTHING happens" — while ~30 sessions were 503-ing.
|
|
434
|
+
this._restartConfirmDetail(),
|
|
430
435
|
// Do NOT stop the TUI here — let the reload path own it: cold restart stops
|
|
431
436
|
// it in restartWorkerNow, a seamless reload in releaseBatonAndDrain (on
|
|
432
437
|
// MSG_RELEASE). If the reload ROLLS BACK (new worker fails to boot), neither
|
|
@@ -558,6 +563,22 @@ export class TUI {
|
|
|
558
563
|
this.mode = 'normal';
|
|
559
564
|
}
|
|
560
565
|
|
|
566
|
+
/** Live description of what pressing restart will do RIGHT NOW — counts the
|
|
567
|
+
* in-flight requests that must drain and states the worst-case pause, so the
|
|
568
|
+
* confirm is honest about the cost instead of hiding it behind "drain active work". */
|
|
569
|
+
_restartConfirmDetail() {
|
|
570
|
+
const inFlight = this.active?.size || 0;
|
|
571
|
+
const sessions = new Set([...(this.active?.values() || [])].map(r => r.sessionKey).filter(Boolean)).size;
|
|
572
|
+
const drainSec = Math.round((this.config?.restartDrainTimeoutMs ?? 10_000) / 1000);
|
|
573
|
+
if (inFlight === 0) {
|
|
574
|
+
return 'Nothing is in flight, so this restarts immediately. Running sessions reconnect on their next request.';
|
|
575
|
+
}
|
|
576
|
+
const s = sessions === 1 ? '' : 's';
|
|
577
|
+
return `${inFlight} request${inFlight === 1 ? '' : 's'} from ${sessions} session${s} are in flight. `
|
|
578
|
+
+ `They finish first (up to ${drainSec}s), and during that window NEW requests get a "restarting" retry. `
|
|
579
|
+
+ 'Sessions are not lost — they reconnect automatically.';
|
|
580
|
+
}
|
|
581
|
+
|
|
561
582
|
_keyAccounts(k) {
|
|
562
583
|
if (k === 'a') {
|
|
563
584
|
// ONE entry point for every account type. Previously `l` (browser) and `k`
|
|
@@ -783,12 +804,16 @@ export class TUI {
|
|
|
783
804
|
);
|
|
784
805
|
} else if (k === 'p' && this.am.accounts.some(account => account.type !== 'provider')) {
|
|
785
806
|
this._startSelection('prefer');
|
|
807
|
+
} else if (k === 'f' || k === 'F' || k === 'm' || k === 'M') {
|
|
808
|
+
// Cycle the named routing mode — reversible + non-destructive, so no confirm
|
|
809
|
+
// dialog (unlike restart/delete). `f` is kept for muscle memory from the old
|
|
810
|
+
// cross-provider knob this replaced; `m` is the mnemonic (mode).
|
|
811
|
+
this._cycleRoutingMode();
|
|
786
812
|
} else if (k === 'g' || k === 'k') {
|
|
813
|
+
// Per-provider Claude→provider fallback. Only affects routing under `sticky`
|
|
814
|
+
// mode — under balance/prefer-* the mode itself controls eligibility. Still
|
|
815
|
+
// safe to set (it'll apply if you switch back to sticky).
|
|
787
816
|
this._cycleProviderClaudeFallback(k === 'g' ? 'zai' : 'kimi');
|
|
788
|
-
} else if (k === 'f' || k === 'F') {
|
|
789
|
-
// Cycle the cross-provider fallback policy in place — reversible + non-destructive,
|
|
790
|
-
// so no confirm dialog (unlike restart/delete). Accept F too (Shift-f muscle memory).
|
|
791
|
-
this._cycleCrossProviderPolicy();
|
|
792
817
|
} else if (k === 'esc' || k === 'q') {
|
|
793
818
|
this.mode = 'normal';
|
|
794
819
|
}
|
|
@@ -822,14 +847,38 @@ export class TUI {
|
|
|
822
847
|
apply();
|
|
823
848
|
}
|
|
824
849
|
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
850
|
+
/** The five named routing modes, in cycle order. Each carries the plain-English
|
|
851
|
+
* line the header shows, so the name the operator picks and the behaviour they
|
|
852
|
+
* get are described by ONE string — the old `never/when-exhausted/always` knob
|
|
853
|
+
* read as "load balance across everything" and did nothing of the sort. */
|
|
854
|
+
static ROUTING_MODES = [
|
|
855
|
+
{ id: 'balance', label: 'Balance all',
|
|
856
|
+
help: 'Score every request across Claude, GLM and Kimi. Busiest accounts get less.' },
|
|
857
|
+
{ id: 'prefer-claude', label: 'Prefer Claude',
|
|
858
|
+
help: 'Claude first; GLM/Kimi pick up the overflow when Claude is loaded.' },
|
|
859
|
+
{ id: 'prefer-zai', label: 'Prefer GLM',
|
|
860
|
+
help: 'GLM first; Claude/Kimi pick up the overflow when GLM is loaded.' },
|
|
861
|
+
{ id: 'prefer-kimi', label: 'Prefer Kimi',
|
|
862
|
+
help: 'Kimi first; Claude/GLM pick up the overflow when Kimi is loaded.' },
|
|
863
|
+
{ id: 'sticky', label: 'One account per session',
|
|
864
|
+
help: 'Each session stays on the account it started on. No spreading.' },
|
|
865
|
+
];
|
|
866
|
+
|
|
867
|
+
_routingModeDef(id) {
|
|
868
|
+
return TUI.ROUTING_MODES.find(m => m.id === id) || TUI.ROUTING_MODES[0];
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
async _cycleRoutingMode() {
|
|
872
|
+
const modes = TUI.ROUTING_MODES;
|
|
873
|
+
const cur = this.am.scheduler?.routingMode || 'sticky';
|
|
874
|
+
const next = modes[(modes.findIndex(m => m.id === cur) + 1) % modes.length];
|
|
875
|
+
this.am.setProviderRoutingMode?.(next.id);
|
|
876
|
+
this.config.scheduler = { ...(this.config.scheduler || {}), routingMode: next.id };
|
|
877
|
+
// Clear the stale legacy policy so it can't contradict the new mode in
|
|
878
|
+
// /maxpool/status or be read by code that still references it.
|
|
879
|
+
delete this.config.scheduler.crossProviderFallbackPolicy;
|
|
831
880
|
await this.saveConfig(this.config);
|
|
832
|
-
this._addLog(`
|
|
881
|
+
this._addLog(`Routing: ${next.label} — ${next.help}`);
|
|
833
882
|
}
|
|
834
883
|
|
|
835
884
|
_keySelect(k) {
|
|
@@ -912,7 +961,20 @@ export class TUI {
|
|
|
912
961
|
const nonProv = [];
|
|
913
962
|
const prov = [];
|
|
914
963
|
this.am.accounts.forEach((a, i) => (a.type === 'provider' ? prov : nonProv).push(i));
|
|
915
|
-
|
|
964
|
+
// GROUP providers by family (all GLM together, all Kimi together). Providers land
|
|
965
|
+
// in the config array in the order they were ADDED, so a GLM account added after a
|
|
966
|
+
// Kimi rendered below it — same-provider accounts split across the table (reported
|
|
967
|
+
// 2026-08-17: "accounts need to be grouped by provider... you shouldn't be mixing
|
|
968
|
+
// Anthropic with Z.ai and Moonshot"). Stable: group by first-seen order, preserving
|
|
969
|
+
// the within-family order and never reordering OAuth rows.
|
|
970
|
+
const familyFirstSeen = new Map();
|
|
971
|
+
for (const idx of prov) {
|
|
972
|
+
const fam = this.am.accounts[idx].provider || 'other';
|
|
973
|
+
if (!familyFirstSeen.has(fam)) familyFirstSeen.set(fam, []);
|
|
974
|
+
familyFirstSeen.get(fam).push(idx);
|
|
975
|
+
}
|
|
976
|
+
const provGrouped = [...familyFirstSeen.values()].flat();
|
|
977
|
+
return [...nonProv, ...provGrouped];
|
|
916
978
|
}
|
|
917
979
|
|
|
918
980
|
_selectableIndexes(action) {
|
|
@@ -1400,19 +1462,35 @@ export class TUI {
|
|
|
1400
1462
|
const how = this._autoUpdateOn() ? 'applying automatically · or press u now' : 'press u to update now';
|
|
1401
1463
|
lines.push(' ' + yellow(`↑ Update available: v${v.current} → v${v.latest}`) + dim(` · ${how}`));
|
|
1402
1464
|
}
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1465
|
+
// Routing header: name the mode + show the one-line description the operator
|
|
1466
|
+
// picked when cycling. Under `preferred`, show the manual-preference line (still
|
|
1467
|
+
// available via the 'p' key on the routing screen).
|
|
1468
|
+
let routing;
|
|
1469
|
+
if (this.am.routingMode === 'preferred') {
|
|
1470
|
+
routing = `Manual preference: ${this.am.preferredAccountName} (automatic failover)`;
|
|
1471
|
+
} else {
|
|
1472
|
+
// Label ONLY here. The one-line description belongs on the routing screen (and in
|
|
1473
|
+
// the log line when the mode changes) — inlining it here pushed the provider-volume
|
|
1474
|
+
// and cross-provider fragments off the right edge at 120 columns.
|
|
1475
|
+
routing = this._routingModeDef(this.am.scheduler?.routingMode || 'sticky').label;
|
|
1476
|
+
}
|
|
1406
1477
|
// Cross-provider fallback policy — only meaningful when GLM/Kimi providers are in
|
|
1407
1478
|
// the pool (profile=all). never=strict pin (yellow), when-exhausted=default (cyan),
|
|
1408
1479
|
// always=peer (green).
|
|
1409
1480
|
const hasProviders = this.am.accounts.some(a => a.type === 'provider');
|
|
1481
|
+
const mode = this.am.scheduler?.routingMode || 'sticky';
|
|
1410
1482
|
let xpText = '';
|
|
1483
|
+
// Only show the cross-provider fragment under sticky — under balance/prefer-*, the
|
|
1484
|
+
// MODE controls routing and claudeFallback is inert. Showing it there made the
|
|
1485
|
+
// header read "Balance all · Cross-provider: always" which looked like two
|
|
1486
|
+
// conflicting settings when only the first one does anything.
|
|
1411
1487
|
if (hasProviders) {
|
|
1412
|
-
|
|
1488
|
+
// Cross-provider fragment is sticky-only — under other modes it is inert and
|
|
1489
|
+
// reading "always" next to "Balance all" looked like two conflicting controls.
|
|
1490
|
+
if (mode === 'sticky') xpText = this._crossProviderText();
|
|
1413
1491
|
// Overflow visibility: when GLM/Kimi actually served requests recently, surface the
|
|
1414
|
-
// volume so provider traffic isn't a mystery
|
|
1415
|
-
//
|
|
1492
|
+
// volume so provider traffic isn't a mystery. This is DATA, not a control — shown
|
|
1493
|
+
// in every mode. Reuses the SAME 15m load window as the per-row "15m Nr" column.
|
|
1416
1494
|
const now = Date.now();
|
|
1417
1495
|
let provReq = 0;
|
|
1418
1496
|
for (const a of this.am.accounts) {
|
|
@@ -1732,7 +1810,7 @@ export class TUI {
|
|
|
1732
1810
|
if (headerFresh) return '';
|
|
1733
1811
|
}
|
|
1734
1812
|
const s = q.lastProbeErrorStatus;
|
|
1735
|
-
if (s === 429) return ` ${yellow('stale·
|
|
1813
|
+
if (s === 429) return ` ${yellow('stale·probe throttled')}`;
|
|
1736
1814
|
if (s) return ` ${yellow('stale·probe ' + s)}`;
|
|
1737
1815
|
return ` ${dim('stale')}`;
|
|
1738
1816
|
}
|
|
@@ -1808,12 +1886,14 @@ export class TUI {
|
|
|
1808
1886
|
case 'addtype':
|
|
1809
1887
|
return ` ${bold('1')}-${bold('4')} pick a type ${bold('Esc')} Back`;
|
|
1810
1888
|
case 'routing': {
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
//
|
|
1814
|
-
|
|
1815
|
-
const
|
|
1816
|
-
|
|
1889
|
+
const mode = this._routingModeDef(this.am.scheduler?.routingMode || 'sticky');
|
|
1890
|
+
const hasProviders = this.am.accounts.some(a => a.type === 'provider');
|
|
1891
|
+
// g/k controls only affect routing under sticky — hide them under other modes
|
|
1892
|
+
// so the operator never cycles a knob that does nothing.
|
|
1893
|
+
const provPart = (hasProviders && mode.id === 'sticky')
|
|
1894
|
+
? ` ${bold('g')} GLM: ${cyan(this.am._claudeFallbackFor?.('zai') || 'never')} ${bold('k')} Kimi: ${cyan(this.am._claudeFallbackFor?.('kimi') || 'never')}`
|
|
1895
|
+
: '';
|
|
1896
|
+
return ` ${bold('f')} Routing: ${cyan(mode.label)} ↻${provPart} ${bold('p')} Manual preference ${bold('Esc')} Back`;
|
|
1817
1897
|
}
|
|
1818
1898
|
case 'select': {
|
|
1819
1899
|
const act = this.selAction === 'prefer'
|