maxpool 1.5.31 → 1.5.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +49 -2
- package/src/index.js +46 -13
- package/src/prober.js +101 -8
- package/src/server.js +2 -1
- package/src/tui.js +51 -7
- package/src/updater.js +20 -3
package/package.json
CHANGED
package/src/account-manager.js
CHANGED
|
@@ -55,6 +55,18 @@ function emptyQuota() {
|
|
|
55
55
|
// marker — a swallowed failing probe no longer silently freezes a stale tag.
|
|
56
56
|
// Header-driven updates do NOT stamp this (headers can't refresh scoped/provider).
|
|
57
57
|
lastProbeOkAt: null, // ms
|
|
58
|
+
// Last background probe FAILURE — surfaced in the TUI/status so a persistently
|
|
59
|
+
// failing probe (e.g. the usage endpoint rate-limiting us) is VISIBLE instead of
|
|
60
|
+
// silently swallowed (which let a stale weekly keep looking fresh). Cleared on
|
|
61
|
+
// the next successful probe. Not persisted (transient).
|
|
62
|
+
lastProbeError: null, // string
|
|
63
|
+
lastProbeErrorAt: null, // ms
|
|
64
|
+
lastProbeErrorStatus: null, // http status (429, 500, …) or null
|
|
65
|
+
// Last time the DISPLAYED bars (unified5h/7d for OAuth, tokens/requests for
|
|
66
|
+
// API-key) were refreshed from a RESPONSE HEADER (updateQuota). Lets the TUI
|
|
67
|
+
// staleness marker tell "probe stale but bars fresh from live traffic" (a busy
|
|
68
|
+
// account — NOT stale) from "genuinely nothing refreshed it" (idle — stale).
|
|
69
|
+
lastHeaderQuotaAt: null, // ms
|
|
58
70
|
};
|
|
59
71
|
}
|
|
60
72
|
|
|
@@ -1942,6 +1954,10 @@ export class AccountManager {
|
|
|
1942
1954
|
}
|
|
1943
1955
|
|
|
1944
1956
|
q.lastProbeOkAt = Date.now();
|
|
1957
|
+
// A successful probe clears any recorded failure — freshness confirmed.
|
|
1958
|
+
q.lastProbeError = null;
|
|
1959
|
+
q.lastProbeErrorAt = null;
|
|
1960
|
+
q.lastProbeErrorStatus = null;
|
|
1945
1961
|
|
|
1946
1962
|
// If we just learned this account's weekly window while probing, re-evaluate
|
|
1947
1963
|
// selection (same path as learning it from a live response).
|
|
@@ -1951,6 +1967,22 @@ export class AccountManager {
|
|
|
1951
1967
|
}
|
|
1952
1968
|
}
|
|
1953
1969
|
|
|
1970
|
+
/**
|
|
1971
|
+
* Record a background probe FAILURE for an account (called by the prober instead
|
|
1972
|
+
* of swallowing the error). Surfaced in getStatus()/the TUI so a persistently
|
|
1973
|
+
* failing probe is visible; the stored quota values are left untouched (they age
|
|
1974
|
+
* into the staleness marker rather than being blanked — see applyUsageData's
|
|
1975
|
+
* `!= null` guards, which are load-bearing for weekly routing).
|
|
1976
|
+
*/
|
|
1977
|
+
recordProbeError(accountIndex, message, status = null) {
|
|
1978
|
+
const account = this.accounts[accountIndex];
|
|
1979
|
+
if (!account) return;
|
|
1980
|
+
const q = account.quota;
|
|
1981
|
+
q.lastProbeError = message ? String(message).slice(0, 160) : 'probe failed';
|
|
1982
|
+
q.lastProbeErrorAt = Date.now();
|
|
1983
|
+
q.lastProbeErrorStatus = Number.isFinite(status) ? status : null;
|
|
1984
|
+
}
|
|
1985
|
+
|
|
1954
1986
|
/**
|
|
1955
1987
|
* Update a PROVIDER account's quota from a provider usage probe
|
|
1956
1988
|
* (fetchProviderUsage). z.ai maps to Ses/Wk token windows; Kimi has no pollable
|
|
@@ -1987,7 +2019,7 @@ export class AccountManager {
|
|
|
1987
2019
|
}
|
|
1988
2020
|
|
|
1989
2021
|
/**
|
|
1990
|
-
* True when the background quota probe hasn't succeeded in >
|
|
2022
|
+
* True when the background quota probe hasn't succeeded in > 3× its interval —
|
|
1991
2023
|
* the last-known scoped/provider values are aging with no confirmation. Returns
|
|
1992
2024
|
* false when the probe is off (nothing to be stale against) or has never yet
|
|
1993
2025
|
* succeeded (startup — shown as "no data", not "stale").
|
|
@@ -1997,7 +2029,10 @@ export class AccountManager {
|
|
|
1997
2029
|
if (!interval || interval <= 0) return false;
|
|
1998
2030
|
const last = account?.quota?.lastProbeOkAt;
|
|
1999
2031
|
if (last == null) return false;
|
|
2000
|
-
|
|
2032
|
+
// 3× interval (min 3 min): the probe now rolls through accounts one at a time
|
|
2033
|
+
// (a full sweep ~ N × interval/6), so a healthy account refreshes well inside
|
|
2034
|
+
// this window — the marker only fires on a genuine multi-sweep probe failure.
|
|
2035
|
+
return (now - last) > Math.max(3 * interval, 180_000);
|
|
2001
2036
|
}
|
|
2002
2037
|
|
|
2003
2038
|
/**
|
|
@@ -2088,6 +2123,15 @@ export class AccountManager {
|
|
|
2088
2123
|
if (genericRemaining != null) account.quota.genericRemaining = genericRemaining;
|
|
2089
2124
|
if (genericReset != null) account.quota.genericReset = genericReset;
|
|
2090
2125
|
|
|
2126
|
+
// Stamp header-freshness ONLY when a real quota header actually arrived — so a
|
|
2127
|
+
// header-less response never falsely marks the bars fresh. Drives the TUI's
|
|
2128
|
+
// "busy account isn't stale even if its background probe is 429-throttled".
|
|
2129
|
+
const gotQuotaHeader = !isNaN(u5h) || !isNaN(u7d)
|
|
2130
|
+
|| !isNaN(tokensLimit) || !isNaN(tokensRemaining)
|
|
2131
|
+
|| !isNaN(requestsLimit) || !isNaN(requestsRemaining)
|
|
2132
|
+
|| genericLimit != null || genericRemaining != null;
|
|
2133
|
+
if (gotQuotaHeader) account.quota.lastHeaderQuotaAt = Date.now();
|
|
2134
|
+
|
|
2091
2135
|
account.usage.totalRequests++;
|
|
2092
2136
|
account.usage.lastUsed = new Date().toISOString();
|
|
2093
2137
|
|
|
@@ -2598,6 +2642,9 @@ export class AccountManager {
|
|
|
2598
2642
|
getStatus() {
|
|
2599
2643
|
const now = Date.now();
|
|
2600
2644
|
return {
|
|
2645
|
+
// Running version + npm update state (set at startup by maybeCheckForUpdate).
|
|
2646
|
+
// null until the check resolves; `current` is known even offline.
|
|
2647
|
+
version: this.versionInfo || null,
|
|
2601
2648
|
currentAccount: this.accounts[this.currentIndex]?.name,
|
|
2602
2649
|
switchThreshold: this.switchThreshold,
|
|
2603
2650
|
routing: {
|
package/src/index.js
CHANGED
|
@@ -6,13 +6,13 @@ import { loadOrCreateConfig, loadConfig, saveConfig, atomicConfigUpdate, getConf
|
|
|
6
6
|
import { setEventLogPath, installConsoleMirror, setConsoleStdoutSuppressed } from './event-log.js';
|
|
7
7
|
import { SleepGuard } from './sleep-guard.js';
|
|
8
8
|
import { AccountManager } from './account-manager.js';
|
|
9
|
-
import { createProxyServer } from './server.js';
|
|
9
|
+
import { createProxyServer, REQUEST_IDLE_MAX_MS } from './server.js';
|
|
10
10
|
import { Prober } from './prober.js';
|
|
11
11
|
import { loginOAuth, fetchProfile, refreshAccessToken, isTokenExpiringSoon, tokenFingerprint } from './oauth.js';
|
|
12
12
|
import { TUI } from './tui.js';
|
|
13
13
|
import { RestartController } from './restart-controller.js';
|
|
14
14
|
import { resolveAccounts } from './account-config.js';
|
|
15
|
-
import { maybeCheckForUpdate } from './updater.js';
|
|
15
|
+
import { maybeCheckForUpdate, getCurrentVersion } from './updater.js';
|
|
16
16
|
import {
|
|
17
17
|
runReloadBaton,
|
|
18
18
|
RELOAD_SWAPPED, RELOAD_ROLLED_BACK,
|
|
@@ -27,6 +27,18 @@ const SERVER_RESTART_EXIT_CODE = 75;
|
|
|
27
27
|
// worker rolled back — self-heal admission so we don't 503 forever. > the baton
|
|
28
28
|
// readiness timeout (10s) + margin.
|
|
29
29
|
const RELOAD_ROLLBACK_SELFHEAL_MS = Math.max(15_000, Number(process.env.MAXPOOL_RELOAD_SELFHEAL_MS) || 30_000);
|
|
30
|
+
// Seamless-reload drain cap. On a seamless reload the NEW worker already serves
|
|
31
|
+
// ALL new traffic while the OLD worker only finishes its own in-flight requests,
|
|
32
|
+
// so a long old-worker drain has zero request-facing cost — let a long streaming
|
|
33
|
+
// response complete instead of getting cut ("Connection closed mid-response").
|
|
34
|
+
// Sized ABOVE the per-request idle reaper (REQUEST_IDLE_MAX_MS, imported from
|
|
35
|
+
// server.js so both sizes share ONE source of truth) so an actively-progressing
|
|
36
|
+
// stream is never cut mid-flight; the reaper bounds a truly-stuck request, and
|
|
37
|
+
// this flat cap is only the last-resort ceiling. Quit/Ctrl-C keeps the short
|
|
38
|
+
// drainTimeoutMs (the user wants to exit now). An explicit MAXPOOL_RELOAD_DRAIN_MS
|
|
39
|
+
// override is honored down to a 60s floor (a shorter drain trades stream-
|
|
40
|
+
// completeness for faster reloads); leave it unset to auto-track the reaper +60s.
|
|
41
|
+
const RELOAD_DRAIN_MS = Math.max(60_000, Number(process.env.MAXPOOL_RELOAD_DRAIN_MS) || (REQUEST_IDLE_MAX_MS + 60_000));
|
|
30
42
|
const SERVER_WORKER_ENV = 'MAXPOOL_SERVER_WORKER';
|
|
31
43
|
// Set by the supervisor when it spawns a worker for a seamless reload: that
|
|
32
44
|
// worker boots HEADLESS (plain logs, no writer lease) and waits for the baton.
|
|
@@ -325,7 +337,7 @@ async function supervisorCommand() {
|
|
|
325
337
|
// sent PRIMARY during the baton; re-assert the master-accept close.
|
|
326
338
|
monitorAsActive(newWorker);
|
|
327
339
|
closeMasterAccept().catch(() => {});
|
|
328
|
-
reapOldWorker(oldWorker
|
|
340
|
+
reapOldWorker(oldWorker);
|
|
329
341
|
return;
|
|
330
342
|
}
|
|
331
343
|
|
|
@@ -415,17 +427,20 @@ async function supervisorCommand() {
|
|
|
415
427
|
|
|
416
428
|
// Drain + reap a released worker. The worker exits itself once its bounded
|
|
417
429
|
// in-flight finishes; the supervisor SIGKILLs it if it outlives the drain cap.
|
|
418
|
-
function reapOldWorker(worker
|
|
430
|
+
function reapOldWorker(worker) {
|
|
419
431
|
if (!worker) return;
|
|
420
|
-
|
|
432
|
+
// +30s grace over the worker's own RELOAD_DRAIN_MS hardCap so its clean exit(0)
|
|
433
|
+
// (fired while its streaming socket keeps the loop alive) always wins the race —
|
|
434
|
+
// SIGKILL only ever reaps a genuinely wedged worker, never cuts a live stream.
|
|
435
|
+
const cap = RELOAD_DRAIN_MS + 30_000;
|
|
421
436
|
let reaped = false;
|
|
422
437
|
const finish = () => { if (reaped) return; reaped = true; clearTimeout(timer); };
|
|
423
438
|
const timer = setTimeout(() => {
|
|
424
439
|
if (reaped) return;
|
|
425
|
-
console.error(`[Maxpool] Old worker outlived ${Math.ceil(
|
|
440
|
+
console.error(`[Maxpool] Old worker outlived ${Math.ceil(cap / 1000)}s reload-drain cap; SIGKILL.`);
|
|
426
441
|
try { worker.child.kill('SIGKILL'); } catch { /* ignore */ }
|
|
427
442
|
finish();
|
|
428
|
-
},
|
|
443
|
+
}, cap);
|
|
429
444
|
timer.unref?.();
|
|
430
445
|
worker.child.once('exit', finish);
|
|
431
446
|
}
|
|
@@ -590,9 +605,16 @@ async function serverWorkerCommand() {
|
|
|
590
605
|
const probeSeconds = process.env.MAXPOOL_DISABLE_QUOTA_PROBE === '1' ? 0 : (config.quotaProbeSeconds || 0);
|
|
591
606
|
const prober = new Prober(accountManager, { intervalMs: probeSeconds * 1000 });
|
|
592
607
|
// Tell the AM the probe cadence so the TUI can flag a scoped/provider tag whose
|
|
593
|
-
// background probe has gone stale (>
|
|
608
|
+
// background probe has gone stale (> 3× interval since last success).
|
|
594
609
|
accountManager.quotaProbeIntervalMs = probeSeconds * 1000;
|
|
595
610
|
|
|
611
|
+
// Seed the running version immediately so the TUI header always shows it, even
|
|
612
|
+
// before (or without) the npm update check. The cold worker's update check below
|
|
613
|
+
// fills in latest/hasUpdate.
|
|
614
|
+
getCurrentVersion()
|
|
615
|
+
.then(v => { accountManager.versionInfo ||= { current: v, latest: null, hasUpdate: false, checkedAt: null }; })
|
|
616
|
+
.catch(() => {});
|
|
617
|
+
|
|
596
618
|
// Persist refreshed tokens back to config. Defense-in-depth: the updater reads
|
|
597
619
|
// the on-disk refresh token and SKIPS the rotation if a fresher writer already
|
|
598
620
|
// advanced it (generation guard), so a stale write can't double-spend a token.
|
|
@@ -1040,7 +1062,7 @@ async function serverWorkerCommand() {
|
|
|
1040
1062
|
if (!viaTakeover && !isReloadWorker) {
|
|
1041
1063
|
if (process.env.MAXPOOL_TEST_LOG_UPDATE_CHECK === '1') console.log('[Maxpool] UPDATE_CHECK_FIRED');
|
|
1042
1064
|
const notify = msg => (tui?._addLog ? tui._addLog(msg) : console.log(`[Maxpool] ${msg}`));
|
|
1043
|
-
maybeCheckForUpdate(config, notify).catch(() => {});
|
|
1065
|
+
maybeCheckForUpdate(config, notify, info => { accountManager.versionInfo = info; }).catch(() => {});
|
|
1044
1066
|
} else if (process.env.MAXPOOL_TEST_LOG_UPDATE_CHECK === '1') {
|
|
1045
1067
|
console.log('[Maxpool] UPDATE_CHECK_SKIPPED (reload)');
|
|
1046
1068
|
}
|
|
@@ -1101,17 +1123,28 @@ async function serverWorkerCommand() {
|
|
|
1101
1123
|
setConsoleStdoutSuppressed(true);
|
|
1102
1124
|
try { process.send({ type: MSG_RELEASED }); } catch { /* ignore */ }
|
|
1103
1125
|
|
|
1104
|
-
//
|
|
1105
|
-
//
|
|
1126
|
+
// (macOS) releaseLease() above dropped the wake-lock, but we still stream our
|
|
1127
|
+
// in-flight requests for up to RELOAD_DRAIN_MS. Re-arm the guard so an idle
|
|
1128
|
+
// Mac can't Maintenance-Sleep mid-drain and cut a long overnight stream (the
|
|
1129
|
+
// exact case the guard exists for). It re-checks getGlobalInFlight and
|
|
1130
|
+
// releases on its own once we're idle; caffeinate -w <pid> dies with us
|
|
1131
|
+
// regardless. Started AFTER the stdout muzzle so its log can't paint the new
|
|
1132
|
+
// worker's screen. No-op off macOS / when disabled.
|
|
1133
|
+
sleepGuard.start();
|
|
1134
|
+
|
|
1135
|
+
// Drain bounded in-flight on EXISTING access tokens (no refresh needed), then
|
|
1136
|
+
// exit(0). Cap = RELOAD_DRAIN_MS (above the idle reaper) so a long streaming
|
|
1137
|
+
// response finishes instead of being cut; the supervisor SIGKILLs us only if
|
|
1138
|
+
// we outlive its (slightly longer) cap.
|
|
1106
1139
|
const waitForDrain = () => {
|
|
1107
1140
|
if (restartController.activeRequests.size === 0) { process.exit(0); return; }
|
|
1108
1141
|
};
|
|
1109
1142
|
const drainPoll = setInterval(waitForDrain, 200);
|
|
1110
1143
|
drainPoll.unref?.();
|
|
1111
1144
|
const hardCap = setTimeout(() => {
|
|
1112
|
-
console.error(`[Maxpool] Released worker drain cap reached with ${restartController.activeRequests.size} active; exiting.`);
|
|
1145
|
+
console.error(`[Maxpool] Released worker reload-drain cap reached with ${restartController.activeRequests.size} active; exiting.`);
|
|
1113
1146
|
process.exit(0);
|
|
1114
|
-
},
|
|
1147
|
+
}, RELOAD_DRAIN_MS);
|
|
1115
1148
|
hardCap.unref?.();
|
|
1116
1149
|
waitForDrain();
|
|
1117
1150
|
};
|
package/src/prober.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
import { fetchUsage, fetchProviderUsage } from './oauth.js';
|
|
11
11
|
|
|
12
12
|
export class Prober {
|
|
13
|
-
constructor(accountManager, { intervalMs = 0, probeFn = fetchUsage, providerProbeFn = fetchProviderUsage, timeoutMs = 10_000, log = console.log } = {}) {
|
|
13
|
+
constructor(accountManager, { intervalMs = 0, probeFn = fetchUsage, providerProbeFn = fetchProviderUsage, timeoutMs = 10_000, log = console.log, usageGapMs = null } = {}) {
|
|
14
14
|
this.am = accountManager;
|
|
15
15
|
this.intervalMs = intervalMs;
|
|
16
16
|
this.probeFn = probeFn;
|
|
@@ -19,6 +19,14 @@ export class Prober {
|
|
|
19
19
|
this.log = log;
|
|
20
20
|
this.timer = null;
|
|
21
21
|
this._running = false;
|
|
22
|
+
// De-burst pacing for the shared, per-IP-rate-limited /api/oauth/usage
|
|
23
|
+
// endpoint (see probeAll). null gap → derive from intervalMs; 0 → no pacing.
|
|
24
|
+
this._configuredUsageGapMs = usageGapMs;
|
|
25
|
+
this._usageGapMs = null; // current adaptive gap (grows on 429, eases on OK)
|
|
26
|
+
this._lastUsageProbeAt = 0; // ms of the last usage request across ALL accounts
|
|
27
|
+
this._stopping = false;
|
|
28
|
+
this._pendingSleep = null; // canceller for an in-flight pacing sleep
|
|
29
|
+
this._sweepStart = 0; // rotates the per-sweep start account (fairness)
|
|
22
30
|
}
|
|
23
31
|
|
|
24
32
|
start() {
|
|
@@ -46,7 +54,9 @@ export class Prober {
|
|
|
46
54
|
* (the baton release) can be sure no probe-driven token rotation is pending
|
|
47
55
|
* before it hands the writer lease to another worker. */
|
|
48
56
|
async stop() {
|
|
57
|
+
this._stopping = true;
|
|
49
58
|
if (this.timer) { clearInterval(this.timer); this.timer = null; }
|
|
59
|
+
if (this._pendingSleep) this._pendingSleep(); // abort an in-flight pacing wait
|
|
50
60
|
if (this._inflight) { try { await this._inflight; } catch { /* swallow */ } }
|
|
51
61
|
}
|
|
52
62
|
|
|
@@ -57,16 +67,36 @@ export class Prober {
|
|
|
57
67
|
probeAll() {
|
|
58
68
|
if (this._running) return this._inflight || Promise.resolve();
|
|
59
69
|
this._running = true;
|
|
70
|
+
this._stopping = false;
|
|
60
71
|
this._inflight = (async () => {
|
|
61
72
|
try {
|
|
62
73
|
// Skip auth-dead accounts (dead refresh token): probing them just re-POSTs
|
|
63
74
|
// the rejected token every cycle — a 400 storm. They recover only on re-auth.
|
|
64
75
|
const oauth = this.am.accounts.filter(a => a.type === 'oauth' && a.credential && !a.refreshDead);
|
|
65
76
|
const providers = this.am.accounts.filter(a => a.type === 'provider' && a.credential);
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
77
|
+
|
|
78
|
+
// Providers read DISTINCT hosts (z.ai monitor / Kimi) — no collision — so
|
|
79
|
+
// they run concurrently. Every OAuth account, though, reads the SAME
|
|
80
|
+
// /api/oauth/usage, which is tightly rate-limited PER SOURCE IP: firing them
|
|
81
|
+
// all at once (the old Promise.all) 429'd all but ~one per cycle, so only a
|
|
82
|
+
// single account's 5h/7d refreshed each tick and the weekly looked frozen.
|
|
83
|
+
// Probe the OAuth accounts ONE AT A TIME, paced across the interval, and back
|
|
84
|
+
// off on a 429 — so the whole fleet refreshes instead of self-throttling.
|
|
85
|
+
const providerWork = Promise.all(providers.map(a => this.probeProvider(a)));
|
|
86
|
+
// Rotate the start account each sweep so no account is systematically the
|
|
87
|
+
// oldest-refreshed (the serialized sweep would otherwise always probe the
|
|
88
|
+
// last account last).
|
|
89
|
+
const start = oauth.length ? (this._sweepStart++ % oauth.length) : 0;
|
|
90
|
+
for (let i = 0; i < oauth.length; i++) {
|
|
91
|
+
if (this._stopping) break;
|
|
92
|
+
await this._paceUsage();
|
|
93
|
+
if (this._stopping) break;
|
|
94
|
+
const r = await this.probeOne(oauth[(start + i) % oauth.length]);
|
|
95
|
+
this._lastUsageProbeAt = Date.now();
|
|
96
|
+
if (r && r.status === 429) this._bumpUsageGap();
|
|
97
|
+
else if (r && r.ok) this._relaxUsageGap();
|
|
98
|
+
}
|
|
99
|
+
await providerWork.catch(() => {});
|
|
70
100
|
} finally {
|
|
71
101
|
this._running = false;
|
|
72
102
|
this._inflight = null;
|
|
@@ -75,6 +105,54 @@ export class Prober {
|
|
|
75
105
|
return this._inflight;
|
|
76
106
|
}
|
|
77
107
|
|
|
108
|
+
/** Wait until the current usage-probe gap has elapsed since the last usage
|
|
109
|
+
* request, so consecutive OAuth probes don't collide on the shared endpoint.
|
|
110
|
+
* No-op when pacing is disabled (gap 0 — manual/tests). Abortable via stop(). */
|
|
111
|
+
async _paceUsage() {
|
|
112
|
+
const gap = this._usageGapMs != null ? this._usageGapMs : this._baseUsageGap();
|
|
113
|
+
if (!gap || gap <= 0) return;
|
|
114
|
+
const wait = gap - (Date.now() - (this._lastUsageProbeAt || 0));
|
|
115
|
+
if (wait > 0) await this._sleep(wait);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Base spacing between usage probes. Spread ~all accounts across the interval
|
|
119
|
+
* (interval/6, clamped 6-20s) unless explicitly configured. 0 when the probe is
|
|
120
|
+
* off / driven manually so a direct probeAll() runs with no delay. */
|
|
121
|
+
_baseUsageGap() {
|
|
122
|
+
if (this._configuredUsageGapMs != null) return this._configuredUsageGapMs;
|
|
123
|
+
if (this.intervalMs > 0) return Math.max(6000, Math.min(20_000, Math.floor(this.intervalMs / 6)));
|
|
124
|
+
return 0;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** A usage 429 means we're probing the shared endpoint too fast — widen the gap
|
|
128
|
+
* (×1.5, cap 120s) so the fleet stops self-throttling. */
|
|
129
|
+
_bumpUsageGap() {
|
|
130
|
+
const base = this._baseUsageGap();
|
|
131
|
+
if (base <= 0) return; // pacing disabled — nothing to back off
|
|
132
|
+
const cur = this._usageGapMs != null ? this._usageGapMs : base;
|
|
133
|
+
this._usageGapMs = Math.min(120_000, Math.round(cur * 1.5));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** A clean probe — ease the gap back toward the base (never below it). */
|
|
137
|
+
_relaxUsageGap() {
|
|
138
|
+
const base = this._baseUsageGap();
|
|
139
|
+
if (base <= 0) { this._usageGapMs = null; return; }
|
|
140
|
+
const cur = this._usageGapMs != null ? this._usageGapMs : base;
|
|
141
|
+
this._usageGapMs = Math.max(base, Math.round(cur * 0.9));
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Abortable sleep — stop() cancels an in-flight pacing wait so a baton release
|
|
145
|
+
* never blocks on it. NOT unref'd: the wait is short (≤ gap), it only runs during
|
|
146
|
+
* an active sweep, and graceful shutdown always goes through stop() which clears
|
|
147
|
+
* it — so it can't delay exit, while staying ref'd keeps the awaited sweep alive. */
|
|
148
|
+
_sleep(ms) {
|
|
149
|
+
return new Promise(resolve => {
|
|
150
|
+
if (this._stopping) return resolve();
|
|
151
|
+
const t = setTimeout(() => { this._pendingSleep = null; resolve(); }, ms);
|
|
152
|
+
this._pendingSleep = () => { clearTimeout(t); this._pendingSleep = null; resolve(); };
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
|
|
78
156
|
/** Probe one PROVIDER account. Provider tokens are static API keys (no OAuth
|
|
79
157
|
* refresh). Best-effort; never throws. */
|
|
80
158
|
async probeProvider(account) {
|
|
@@ -85,6 +163,10 @@ export class Prober {
|
|
|
85
163
|
} catch { /* best-effort; never let a probe throw */ }
|
|
86
164
|
}
|
|
87
165
|
|
|
166
|
+
/** Probe one OAUTH account. Returns {ok, status} so probeAll can pace/back-off
|
|
167
|
+
* and so a persistent failure is RECORDED (surfaced in the TUI/status) rather
|
|
168
|
+
* than silently swallowed — a swallowed failing probe is what let a stale
|
|
169
|
+
* weekly look fresh. Never throws. */
|
|
88
170
|
async probeOne(account) {
|
|
89
171
|
try {
|
|
90
172
|
await this.am.ensureTokenFresh(account.index);
|
|
@@ -94,9 +176,20 @@ export class Prober {
|
|
|
94
176
|
await this.am.ensureTokenFresh(account.index, true);
|
|
95
177
|
usage = await this._withTimeout(this.probeFn(account.credential));
|
|
96
178
|
}
|
|
97
|
-
if (
|
|
98
|
-
|
|
99
|
-
|
|
179
|
+
if (usage == null) { // timed out
|
|
180
|
+
this.am.recordProbeError?.(account.index, 'probe timed out', null);
|
|
181
|
+
return { ok: false, status: null };
|
|
182
|
+
}
|
|
183
|
+
if (usage.error) { // HTTP error (e.g. 429) or fetch failure
|
|
184
|
+
this.am.recordProbeError?.(account.index, usage.error, usage.status ?? null);
|
|
185
|
+
return { ok: false, status: usage.status ?? null };
|
|
186
|
+
}
|
|
187
|
+
this.am.applyUsageData(account.index, usage); // clears the error + stamps freshness
|
|
188
|
+
return { ok: true, status: 200 };
|
|
189
|
+
} catch (e) { // best-effort; never let a probe throw
|
|
190
|
+
this.am.recordProbeError?.(account.index, e?.message || String(e), null);
|
|
191
|
+
return { ok: false, status: null };
|
|
192
|
+
}
|
|
100
193
|
}
|
|
101
194
|
|
|
102
195
|
_withTimeout(promise) {
|
package/src/server.js
CHANGED
|
@@ -29,7 +29,8 @@ const CLIENT_DRAIN_MS = Math.max(5_000, Number(process.env.MAXPOOL_DRAIN_MS) ||
|
|
|
29
29
|
// request can go silent (no heartbeat there): TTFB 120s + nonStreamMaxWaitMs 300s +
|
|
30
30
|
// UPSTREAM_BODY_MS 300s ≈ 720s — so a slow-but-alive non-streaming request is never
|
|
31
31
|
// reaped at completion. Streaming holds heartbeat, so they're safe at any ceiling.
|
|
32
|
-
|
|
32
|
+
// Exported so index.js sizes RELOAD_DRAIN_MS off the SAME value (no drift).
|
|
33
|
+
export const REQUEST_IDLE_MAX_MS = Math.max(900_000, Number(process.env.MAXPOOL_REQUEST_IDLE_MAX_MS) || 1_200_000);
|
|
33
34
|
|
|
34
35
|
const DEFAULT_QUEUE = {
|
|
35
36
|
enabled: true,
|
package/src/tui.js
CHANGED
|
@@ -415,6 +415,10 @@ export class TUI {
|
|
|
415
415
|
'Reload account credentials and newly added accounts from the config file.',
|
|
416
416
|
() => this._doSync(),
|
|
417
417
|
);
|
|
418
|
+
} else if (k === 't' && this.am.accounts.length > 0) {
|
|
419
|
+
// Manual on/off toggle straight from the top level (also under a → Accounts).
|
|
420
|
+
// select → confirm → returns to 'normal', so this never strands the user.
|
|
421
|
+
this._startSelection('toggle');
|
|
418
422
|
}
|
|
419
423
|
}
|
|
420
424
|
|
|
@@ -920,7 +924,12 @@ export class TUI {
|
|
|
920
924
|
const lines = [];
|
|
921
925
|
|
|
922
926
|
// ── Header
|
|
923
|
-
const
|
|
927
|
+
const v = this.am.versionInfo;
|
|
928
|
+
const verStr = v?.current ? ` ${dim('v' + v.current)}` : '';
|
|
929
|
+
// "↑ vX.Y.Z" when a newer npm version is published; nothing when on latest or
|
|
930
|
+
// the check hasn't resolved yet (so the header never falsely implies up-to-date).
|
|
931
|
+
const updStr = (v?.hasUpdate && v?.latest) ? ` ${yellow('↑ v' + v.latest)}` : '';
|
|
932
|
+
const left = bold(' Maxpool') + verStr + updStr;
|
|
924
933
|
const port = this.config.proxy?.port || 3456;
|
|
925
934
|
const right = `Port ${port} ${green('▲')} `;
|
|
926
935
|
lines.push(left + ' '.repeat(Math.max(1, W - vw(left) - vw(right))) + right);
|
|
@@ -976,7 +985,7 @@ export class TUI {
|
|
|
976
985
|
// Glossary FOOTER (expands the abbreviations the header + inline labels can't
|
|
977
986
|
// spell out). Below the rows so it never breaks the header↔column alignment.
|
|
978
987
|
if (W >= 88) {
|
|
979
|
-
lines.push(' ' + dim('Ses 5h · Wk 7d · Now in-flight (weight) · 15m/1h served (avg · f fails)'));
|
|
988
|
+
lines.push(' ' + dim('Legend Ses = 5h · Wk = 7d · Now = in-flight (weight) · 15m/1h = served (avg · f = fails)'));
|
|
980
989
|
}
|
|
981
990
|
}
|
|
982
991
|
|
|
@@ -1146,13 +1155,48 @@ export class TUI {
|
|
|
1146
1155
|
}
|
|
1147
1156
|
if (scopedTags.length) line += ` ${scopedTags.join(' ')}`;
|
|
1148
1157
|
// Freshness: scoped caps are refreshed ONLY by the background probe (response
|
|
1149
|
-
// headers don't carry them). If the probe has gone stale
|
|
1150
|
-
//
|
|
1151
|
-
|
|
1158
|
+
// headers don't carry them). If the probe has gone stale, say so — and name the
|
|
1159
|
+
// cause (e.g. rate-limited) when a probe failure is on record — rather than imply
|
|
1160
|
+
// the last-known value is current.
|
|
1161
|
+
line += this._probeHealthNote(a);
|
|
1152
1162
|
line += ` ${dim(loadText(this._accountLoad(a)))}`;
|
|
1153
1163
|
return line;
|
|
1154
1164
|
}
|
|
1155
1165
|
|
|
1166
|
+
/** Freshness suffix for an account row. Empty when the probe is current. When
|
|
1167
|
+
* stale, names the cause from the last recorded probe failure (e.g. a 429 from
|
|
1168
|
+
* the usage endpoint) so a self-throttling probe reads as "rate-limited" rather
|
|
1169
|
+
* than an unexplained "stale". Leading spaces included so callers append raw. */
|
|
1170
|
+
_probeHealthNote(a) {
|
|
1171
|
+
// A dead account (reauth / disabled) already surfaces its state in the status
|
|
1172
|
+
// column, and its quota reading is frozen and moot — the prober skips it, so a
|
|
1173
|
+
// "stale·probe 401" here is just the perpetual echo of the 401 that killed it.
|
|
1174
|
+
// Only annotate probe-staleness for LIVE accounts, where a failing probe
|
|
1175
|
+
// (e.g. a 429) is a real, actionable signal.
|
|
1176
|
+
if (a?.refreshDead || a?.enabled === false) return '';
|
|
1177
|
+
if (!this.am._quotaProbeStale?.(a)) return ''; // probe fresh (or off) → nothing to flag; interval>0 after this
|
|
1178
|
+
// The background probe IS stale — but only flag it if something DISPLAYED is
|
|
1179
|
+
// actually stale. An OAuth account's Ses/Wk bars come from unified5h/7d, which
|
|
1180
|
+
// response headers refresh on every served request (updateQuota → lastHeaderQuotaAt);
|
|
1181
|
+
// so a BUSY oauth account with a header-fresh reading and no probe-only scoped
|
|
1182
|
+
// cap is NOT stale, even if its background probe is 429-throttled. Provider bars
|
|
1183
|
+
// and per-model scoped caps ARE probe-only, so for those a stale probe genuinely
|
|
1184
|
+
// means stale data → keep flagging.
|
|
1185
|
+
const q = a.quota || {};
|
|
1186
|
+
const reserveFloor = this.am.scheduler?.weeklyReserveThreshold ?? 0.85; // mirrors _renderAcct's scoped-tag gate
|
|
1187
|
+
const hasDisplayedScopedCap = Object.values(q.scopedWeekly || {}).some(
|
|
1188
|
+
e => e && e.isActive !== false && e.utilization != null && e.utilization >= reserveFloor);
|
|
1189
|
+
if (a.type !== 'provider' && !hasDisplayedScopedCap) {
|
|
1190
|
+
const interval = this.am.quotaProbeIntervalMs; // > 0 here (else _quotaProbeStale returned false)
|
|
1191
|
+
const headerFresh = q.lastHeaderQuotaAt && (Date.now() - q.lastHeaderQuotaAt) <= Math.max(3 * interval, 180_000);
|
|
1192
|
+
if (headerFresh) return '';
|
|
1193
|
+
}
|
|
1194
|
+
const s = q.lastProbeErrorStatus;
|
|
1195
|
+
if (s === 429) return ` ${yellow('stale·rate-limited')}`;
|
|
1196
|
+
if (s) return ` ${yellow('stale·probe ' + s)}`;
|
|
1197
|
+
return ` ${dim('stale')}`;
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1156
1200
|
_renderProviderAcct(sel, cur, name, type, status, a, bw = 11, showBoth = true) {
|
|
1157
1201
|
const q = a.quota || {};
|
|
1158
1202
|
|
|
@@ -1166,7 +1210,7 @@ export class TUI {
|
|
|
1166
1210
|
if (q.providerSes != null || q.providerWk != null) {
|
|
1167
1211
|
sesCell = q.providerSes != null ? bar(q.providerSes, bw, q.providerSesReset) : emptyBar('—', bw);
|
|
1168
1212
|
wkCell = q.providerWk != null ? bar(q.providerWk, bw, q.providerWkReset) : emptyBar('—', bw);
|
|
1169
|
-
|
|
1213
|
+
note = this._probeHealthNote(a);
|
|
1170
1214
|
} else if (q.providerQuotaSource === 'console-only') {
|
|
1171
1215
|
sesCell = emptyBar('n/a', bw);
|
|
1172
1216
|
wkCell = emptyBar('n/a', bw);
|
|
@@ -1211,7 +1255,7 @@ export class TUI {
|
|
|
1211
1255
|
_renderFooter() {
|
|
1212
1256
|
switch (this.mode) {
|
|
1213
1257
|
case 'normal':
|
|
1214
|
-
return ` ${bold('a')} Accounts ${bold('m')} Routing ${bold('s')} Sync ${bold('r')} Restart ${bold('q')} Stop`;
|
|
1258
|
+
return ` ${bold('a')} Accounts ${bold('t')} On/off ${bold('m')} Routing ${bold('s')} Sync ${bold('r')} Restart ${bold('q')} Stop`;
|
|
1215
1259
|
case 'accounts':
|
|
1216
1260
|
return ` ${bold('l')} Login/re-auth (browser) ${bold('k')} API key ${bold('n')} Rename ${bold('t')} Enable/disable ${bold('d')} Delete ${bold('Esc')} Back`;
|
|
1217
1261
|
case 'routing':
|
package/src/updater.js
CHANGED
|
@@ -73,11 +73,28 @@ export async function selfUpdate({ timeoutMs = 120_000 } = {}) {
|
|
|
73
73
|
* (config.autoUpdate). Never auto-restarts a running proxy — the new version
|
|
74
74
|
* applies on the next restart, so in-flight sessions are never interrupted.
|
|
75
75
|
* Fire-and-forget; all failures are swallowed.
|
|
76
|
+
*
|
|
77
|
+
* `onVersionInfo` (optional) is ALWAYS invoked with { current, latest, hasUpdate,
|
|
78
|
+
* checkedAt } — even when up-to-date, offline, or updateCheck is off — so the TUI
|
|
79
|
+
* header / status can show the running version + whether an update is available.
|
|
76
80
|
*/
|
|
77
|
-
export async function maybeCheckForUpdate(config, notify) {
|
|
78
|
-
if (config?.updateCheck === false) return;
|
|
81
|
+
export async function maybeCheckForUpdate(config, notify, onVersionInfo) {
|
|
79
82
|
const current = await getCurrentVersion();
|
|
80
|
-
|
|
83
|
+
// Skip the npm round-trip when the user disabled update checks, but still report
|
|
84
|
+
// the running version so the indicator can show it.
|
|
85
|
+
const result = config?.updateCheck === false ? null : await checkForUpdate(current);
|
|
86
|
+
|
|
87
|
+
if (onVersionInfo) {
|
|
88
|
+
try {
|
|
89
|
+
onVersionInfo({
|
|
90
|
+
current,
|
|
91
|
+
latest: result?.latest ?? null,
|
|
92
|
+
hasUpdate: Boolean(result?.hasUpdate),
|
|
93
|
+
checkedAt: Date.now(),
|
|
94
|
+
});
|
|
95
|
+
} catch { /* the indicator is best-effort; never break startup */ }
|
|
96
|
+
}
|
|
97
|
+
|
|
81
98
|
if (!result || !result.hasUpdate) return;
|
|
82
99
|
|
|
83
100
|
notify(`Update available: ${result.current} → ${result.latest}`);
|