maxpool 1.7.1 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/account-manager.js +122 -1
- package/src/capacity-ledger.js +286 -0
- package/src/config.js +16 -0
- package/src/index.js +65 -3
- package/src/prober.js +6 -0
- package/src/server.js +36 -7
- package/src/tui.js +132 -5
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "maxpool",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.8.1",
|
|
4
4
|
"description": "Multi-account Claude Code proxy with adaptive, rate-aware load balancing across Claude accounts",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -44,4 +44,4 @@
|
|
|
44
44
|
"eslint": "^9.39.5",
|
|
45
45
|
"git-cliff": "2.13.1"
|
|
46
46
|
}
|
|
47
|
-
}
|
|
47
|
+
}
|
package/src/account-manager.js
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import { refreshAccessToken, isTokenExpiringSoon, modelFamily, tokenFingerprint } from './oauth.js';
|
|
2
|
+
import { CapacityLedger } from './capacity-ledger.js';
|
|
3
|
+
|
|
4
|
+
// A capacity window boundary must move by at least this much to count as a real
|
|
5
|
+
// window ADVANCE rather than reset-stamp jitter (see noteCapacityWindowAdvance).
|
|
6
|
+
const WINDOW_ADVANCE_EPSILON_MS = 60_000;
|
|
2
7
|
import { peakWindowState, DEFAULT_PEAK_CAP } from './peak-window.js';
|
|
3
8
|
|
|
4
9
|
// Bounded re-poll hold for an account blocked ONLY by a transient, self-clearing
|
|
@@ -332,6 +337,9 @@ export class AccountManager {
|
|
|
332
337
|
this.preferredAccountName = null;
|
|
333
338
|
this.sessionBindings = new Map();
|
|
334
339
|
this._peakCache = null; // per-UTC-minute peak-window memo (see _peakStateFor)
|
|
340
|
+
// CAPACITY LEDGER (2026-08-22): observed tokens per completed window cycle, so
|
|
341
|
+
// "true capacity" is comparable across providers. Restored from state on boot.
|
|
342
|
+
this.capacity = new CapacityLedger();
|
|
335
343
|
this.sessionPolicies = new Map();
|
|
336
344
|
this.upstreamThrottle = {
|
|
337
345
|
until: null,
|
|
@@ -1058,9 +1066,14 @@ export class AccountManager {
|
|
|
1058
1066
|
let changed = false;
|
|
1059
1067
|
let session = false;
|
|
1060
1068
|
|
|
1061
|
-
// Clear expired unified quotas
|
|
1069
|
+
// Clear expired unified quotas. The capacity cycle closes HERE, before the stamp
|
|
1070
|
+
// is nulled: this is the authoritative rollover moment, and it fires wherever the
|
|
1071
|
+
// rollover is noticed (TUI render tick, every routed request) — not only on a
|
|
1072
|
+
// prober sweep that happens to land inside the sub-second window before the stamp
|
|
1073
|
+
// disappears (without this, OAuth cycles essentially never close: red-team 2026-08-22).
|
|
1062
1074
|
if (q.unified5h != null && q.unified5hReset && now >= q.unified5hReset) {
|
|
1063
1075
|
console.log(`[Maxpool] Account "${account.name}" session quota reset`);
|
|
1076
|
+
this.capacity?.closeCycle?.(account.name, 'ses', q.unified5hReset, { resetAt: q.unified5hReset });
|
|
1064
1077
|
q.unified5h = null;
|
|
1065
1078
|
q.unified5hReset = null;
|
|
1066
1079
|
changed = true;
|
|
@@ -1068,6 +1081,7 @@ export class AccountManager {
|
|
|
1068
1081
|
}
|
|
1069
1082
|
if (q.unified7d != null && q.unified7dReset && now >= q.unified7dReset) {
|
|
1070
1083
|
console.log(`[Maxpool] Account "${account.name}" weekly quota reset`);
|
|
1084
|
+
this.capacity?.closeCycle?.(account.name, 'wk', q.unified7dReset, { resetAt: q.unified7dReset });
|
|
1071
1085
|
q.unified7d = null;
|
|
1072
1086
|
q.unified7dReset = null;
|
|
1073
1087
|
q.unifiedStatus = null;
|
|
@@ -1914,6 +1928,10 @@ export class AccountManager {
|
|
|
1914
1928
|
setAccountEnabled(index, enabled) {
|
|
1915
1929
|
const account = this.accounts[index];
|
|
1916
1930
|
if (!account) return false;
|
|
1931
|
+
// CAPACITY LEDGER: a cycle that spent part of its life DISABLED did not get the
|
|
1932
|
+
// chance to deliver its true capacity, so it is not a capacity observation —
|
|
1933
|
+
// flag it partial (shown, excluded from the averages).
|
|
1934
|
+
if (!enabled && account.enabled) this.capacity.markPartial(account.name, { disabled: true });
|
|
1917
1935
|
account.enabled = Boolean(enabled);
|
|
1918
1936
|
if (!enabled && account.name === this.preferredAccountName) {
|
|
1919
1937
|
this.setRoutingMode('automatic');
|
|
@@ -2668,6 +2686,10 @@ export class AccountManager {
|
|
|
2668
2686
|
const account = this.accounts[accountIndex];
|
|
2669
2687
|
if (!account || !usage) return;
|
|
2670
2688
|
const q = account.quota;
|
|
2689
|
+
// CAPACITY LEDGER: prev stamps, so a probe observing the window ADVANCE closes
|
|
2690
|
+
// the old cycle (the OAuth twin of the applyProviderUsage hook).
|
|
2691
|
+
const prevSesReset = q.unified5hReset;
|
|
2692
|
+
const prevWkReset = q.unified7dReset;
|
|
2671
2693
|
|
|
2672
2694
|
if (usage.fiveHour) {
|
|
2673
2695
|
if (usage.fiveHour.utilization != null) q.unified5h = clamp01(usage.fiveHour.utilization);
|
|
@@ -2677,6 +2699,12 @@ export class AccountManager {
|
|
|
2677
2699
|
if (usage.sevenDay.utilization != null) q.unified7d = clamp01(usage.sevenDay.utilization);
|
|
2678
2700
|
if (usage.sevenDay.resetAt != null) q.unified7dReset = usage.sevenDay.resetAt;
|
|
2679
2701
|
}
|
|
2702
|
+
// Stamp-advance close (same guard as the provider path): a FRESHER stamp means
|
|
2703
|
+
// the old window rolled over — close its cycle at the old boundary.
|
|
2704
|
+
// noteCapacityWindowAdvance already no-ops on a missing/unchanged/older stamp, so
|
|
2705
|
+
// there is nothing to gate here — pass the raw (possibly undefined) value through.
|
|
2706
|
+
this.noteCapacityWindowAdvance(account.name, 'ses', prevSesReset, usage.fiveHour?.resetAt);
|
|
2707
|
+
this.noteCapacityWindowAdvance(account.name, 'wk', prevWkReset, usage.sevenDay?.resetAt);
|
|
2680
2708
|
// Only a SUCCESSFUL probe carrying the flag speaks to this. A header-driven update
|
|
2681
2709
|
// can't see the limits[] array, and a FAILED read knows nothing about the account's
|
|
2682
2710
|
// caps — either one claiming "uncapped" would mislabel a capped account as having no
|
|
@@ -2787,6 +2815,11 @@ export class AccountManager {
|
|
|
2787
2815
|
const account = this.accounts[accountIndex];
|
|
2788
2816
|
if (!account || !usage) return;
|
|
2789
2817
|
const q = account.quota;
|
|
2818
|
+
// CAPACITY LEDGER: snapshot the previous reset stamps so a probe observing the
|
|
2819
|
+
// window ADVANCE (new stamp) closes the capacity cycle at the old boundary —
|
|
2820
|
+
// covers windows whose old stamp was never learned (clock-close can't fire).
|
|
2821
|
+
const prevSesReset = q.providerSesReset;
|
|
2822
|
+
const prevWkReset = q.providerWkReset;
|
|
2790
2823
|
if (usage.error) {
|
|
2791
2824
|
// Distinguish "no pollable quota" (Kimi) from a transient probe failure.
|
|
2792
2825
|
// Never clear existing values on a transient error — let them age into the
|
|
@@ -2814,6 +2847,12 @@ export class AccountManager {
|
|
|
2814
2847
|
q.weeklyAbsent = true;
|
|
2815
2848
|
}
|
|
2816
2849
|
q.lastProbeOkAt = Date.now();
|
|
2850
|
+
// Stamp-advance close: a FRESHER reset stamp means the old window rolled over —
|
|
2851
|
+
// the tokens accrued since the last close belong to the cycle that just ended.
|
|
2852
|
+
// noteCapacityWindowAdvance already no-ops on a missing/unchanged/older stamp
|
|
2853
|
+
// (same-window re-report, clock-skew backward re-report) — pass values through.
|
|
2854
|
+
this.noteCapacityWindowAdvance(account.name, 'ses', prevSesReset, usage.ses?.resetAt);
|
|
2855
|
+
this.noteCapacityWindowAdvance(account.name, 'wk', prevWkReset, usage.wk?.resetAt);
|
|
2817
2856
|
}
|
|
2818
2857
|
|
|
2819
2858
|
/**
|
|
@@ -2949,6 +2988,88 @@ export class AccountManager {
|
|
|
2949
2988
|
}
|
|
2950
2989
|
}
|
|
2951
2990
|
|
|
2991
|
+
/** Restore the ledger from persisted state. A BOOT GAP (the open cycle's last
|
|
2992
|
+
* accrual is far behind now) means maxpool was down for part of that cycle, so the
|
|
2993
|
+
* cycle is no longer a truthful capacity observation — flag it partial (B2/SC6).
|
|
2994
|
+
* It still displays; it is excluded from averages. */
|
|
2995
|
+
restoreCapacityState(payload, now = Date.now(), downtimeMs = null) {
|
|
2996
|
+
this.capacity = CapacityLedger.fromSerialized(payload);
|
|
2997
|
+
// Partial is keyed on MAXPOOL'S OWN downtime, NEVER on the account's last request:
|
|
2998
|
+
// an account parked >10min mid-cycle is normal fleet rotation, and keying on its
|
|
2999
|
+
// last request discarded valid observations on every reload (red-team F3). The
|
|
3000
|
+
// caller passes measured downtime (state mtime at save → boot now); when it
|
|
3001
|
+
// cannot (a seamless reload handoff — the old worker was provably serving
|
|
3002
|
+
// throughout), null skips the check entirely.
|
|
3003
|
+
// 60s absorbs a restart's own turnaround; anything longer is real downtime, and
|
|
3004
|
+
// it disqualifies EVERY open cycle (maxpool was not serving, so no account could
|
|
3005
|
+
// deliver its true capacity) — the downtime is a property of the process, not of
|
|
3006
|
+
// any one account's traffic.
|
|
3007
|
+
if (!(downtimeMs > 60_000)) return;
|
|
3008
|
+
const spannedDays = new Set();
|
|
3009
|
+
for (let t = now - downtimeMs; t <= now; t += 3600_000) spannedDays.add(new Date(t).toISOString().slice(0, 10));
|
|
3010
|
+
spannedDays.add(new Date(now).toISOString().slice(0, 10));
|
|
3011
|
+
for (const name of this.capacity.accounts()) {
|
|
3012
|
+
if (this.capacity.openCycle(name, 'ses') || this.capacity.openCycle(name, 'wk')) {
|
|
3013
|
+
this.capacity.markPartial(name);
|
|
3014
|
+
}
|
|
3015
|
+
for (const day of spannedDays) this.capacity.markDayPartial(name, day);
|
|
3016
|
+
}
|
|
3017
|
+
}
|
|
3018
|
+
|
|
3019
|
+
/** Accrue ONE request's tokens into the capacity ledger (per-request values; the
|
|
3020
|
+
* server seam has already applied max-semantics for streamed output). */
|
|
3021
|
+
accrueCapacity(accountIndex, { input = 0, output = 0 } = {}) {
|
|
3022
|
+
const account = this.accounts[accountIndex];
|
|
3023
|
+
if (!account) return;
|
|
3024
|
+
this.capacity.accrue(account.name, { input, output });
|
|
3025
|
+
}
|
|
3026
|
+
|
|
3027
|
+
/** Close any window cycle whose reset time has passed — CLOCK-AUTHORITATIVE, so a
|
|
3028
|
+
* stale or dead probe can never leave a cycle open and mis-attribute the next
|
|
3029
|
+
* window's tokens to it (pre-mortem M5; worst case is the no-weekly account whose
|
|
3030
|
+
* probe latches refreshDead). Safe to call on every render tick and prober tick. */
|
|
3031
|
+
closeExpiredCapacityCycles(now = Date.now()) {
|
|
3032
|
+
for (const a of this.accounts) {
|
|
3033
|
+
const q = a.quota || {};
|
|
3034
|
+
const pairs = a.type === 'provider'
|
|
3035
|
+
? [['ses', 'providerSesReset'], ['wk', 'providerWkReset']]
|
|
3036
|
+
: [['ses', 'unified5hReset'], ['wk', 'unified7dReset']];
|
|
3037
|
+
for (const [win, stampKey] of pairs) {
|
|
3038
|
+
const resetAt = q[stampKey];
|
|
3039
|
+
// Close ONLY. This path deliberately does NOT null the stamp: the rollover
|
|
3040
|
+
// stays single-shot because closeCycle FOLDS a same-boundary repeat into the
|
|
3041
|
+
// already-closed cycle (one boundary, one cycle), and the very next
|
|
3042
|
+
// refreshExpiredQuotas / request-path _clearExpiredQuotas nulls the stamp with
|
|
3043
|
+
// its full original side effects — the reset log, the `session` signal that
|
|
3044
|
+
// drives _switchOnSessionReset, and the weekly `unifiedStatus` clear. Nulling
|
|
3045
|
+
// here as well (round-2) made a prober-first notice silently swallow all of
|
|
3046
|
+
// those whenever the sweep won the race (red-team round 3, RT3-1).
|
|
3047
|
+
if (resetAt && now >= resetAt) {
|
|
3048
|
+
this.capacity.closeCycle(a.name, win, resetAt, { resetAt });
|
|
3049
|
+
}
|
|
3050
|
+
}
|
|
3051
|
+
}
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
/** Close a cycle because a probe observed the window ADVANCE (a new reset stamp) —
|
|
3055
|
+
* covers the case where the old stamp was never learned. */
|
|
3056
|
+
noteCapacityWindowAdvance(accountName, window, prevResetAt, nextResetAt) {
|
|
3057
|
+
if (!prevResetAt || !nextResetAt) return;
|
|
3058
|
+
// EPSILON, not `>`. An OAuth reset stamp is derived per-response as
|
|
3059
|
+
// `Date.now() + delay_seconds * 1000` (parseResetHeader), so the SAME window
|
|
3060
|
+
// boundary reads a little later on every response — sub-second to ~2s of jitter.
|
|
3061
|
+
// A bare `nextResetAt > prevResetAt` fired the advance-close on each of those,
|
|
3062
|
+
// shredding one real 5h window into 9-10 fake cycles and closing a "weekly"
|
|
3063
|
+
// cycle whose endedAt was a week in the FUTURE. Measured on live state.json
|
|
3064
|
+
// 2026-08-23, 12h after v1.8.0 shipped: mk@dubner.io had 9 ses cycles between
|
|
3065
|
+
// 01:00 and 06:00, the shortest 0.2 minutes. A REAL advance is a whole window
|
|
3066
|
+
// (>=5h); 60s separates the two by three orders of magnitude.
|
|
3067
|
+
// Provider stamps (z.ai/Kimi probe) are absolute and did NOT jitter — which is
|
|
3068
|
+
// how the defect localized to the OAuth path.
|
|
3069
|
+
if (nextResetAt - prevResetAt < WINDOW_ADVANCE_EPSILON_MS) return;
|
|
3070
|
+
this.capacity.closeCycle(accountName, window, prevResetAt, { resetAt: prevResetAt });
|
|
3071
|
+
}
|
|
3072
|
+
|
|
2952
3073
|
/**
|
|
2953
3074
|
* Update cumulative token usage from response body data.
|
|
2954
3075
|
*/
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* True-capacity ledger — observed tokens per completed window cycle.
|
|
3
|
+
*
|
|
4
|
+
* The measurement model (user spec 2026-08-22): when a 5h or weekly window completes,
|
|
5
|
+
* the tokens delivered during that cycle ARE the account's measured capacity for the
|
|
6
|
+
* window. History per account per window: last / prev / prev-1 / avg3 / avg10 /
|
|
7
|
+
* all-time — in tokens, so a GLM row and a Claude row compare apples to apples.
|
|
8
|
+
*
|
|
9
|
+
* The no-weekly account (legacy z.ai TOKENS_LIMIT, weeklyAbsent) has no weekly TANK:
|
|
10
|
+
* it gets full 5h cycle history plus a rolling-7d THROUGHPUT from UTC day buckets —
|
|
11
|
+
* a volume, never a limit, rendered distinctly.
|
|
12
|
+
*
|
|
13
|
+
* PURE: no I/O, injected clock. Persistence + accrual seams live in account-manager /
|
|
14
|
+
* index; this module owns the data model and its invariants.
|
|
15
|
+
* Pre-mortem (2026-08-22) blockers B1/B2 + majors M3-M7 are the load-bearing comments.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
// v2 (2026-08-23): v1 histories are POISONED and are deliberately dropped on load.
|
|
19
|
+
// The OAuth reset-stamp jitter (see AccountManager.noteCapacityWindowAdvance) shredded
|
|
20
|
+
// real windows into slivers and recorded future-dated weekly cycles, so a v1 payload's
|
|
21
|
+
// averages are wrong for weeks. Starting empty costs one window; keeping it costs trust.
|
|
22
|
+
const SCHEMA_VERSION = 2;
|
|
23
|
+
|
|
24
|
+
// Two closes this far apart are the SAME boundary observed twice (a clock-close and a
|
|
25
|
+
// stamp-advance racing across the boundary second), not two windows.
|
|
26
|
+
const SAME_BOUNDARY_MS = 5_000;
|
|
27
|
+
const MAX_CYCLES_PER_WINDOW = 50;
|
|
28
|
+
const MAX_DAY_BUCKETS = 10;
|
|
29
|
+
|
|
30
|
+
export class CapacityLedger {
|
|
31
|
+
constructor({ now = () => Date.now() } = {}) {
|
|
32
|
+
this._now = now;
|
|
33
|
+
// accountName → { ses: {open?, closed: []}, wk: {open?, closed: []}, days: {utcDay: {tokens, partial}} }
|
|
34
|
+
this._accounts = new Map();
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Restore from a serialized payload (state.json). Tolerant: unknown schemaVersion →
|
|
38
|
+
* start empty and KEEP the file (the caller preserves it); a corrupt shape → empty. */
|
|
39
|
+
static fromSerialized(payload, { now = () => Date.now() } = {}) {
|
|
40
|
+
const l = new CapacityLedger({ now });
|
|
41
|
+
if (!payload || payload.schemaVersion !== SCHEMA_VERSION) return l;
|
|
42
|
+
try {
|
|
43
|
+
for (const [name, rec] of Object.entries(payload.accounts || {})) {
|
|
44
|
+
const a = { ses: { open: null, closed: [] }, wk: { open: null, closed: [] }, days: {} };
|
|
45
|
+
for (const w of ['ses', 'wk']) {
|
|
46
|
+
if (rec[w]?.open) a[w].open = { ...rec[w].open };
|
|
47
|
+
if (Array.isArray(rec[w]?.closed)) a[w].closed = rec[w].closed.slice(-MAX_CYCLES_PER_WINDOW);
|
|
48
|
+
}
|
|
49
|
+
if (rec.days && typeof rec.days === 'object') {
|
|
50
|
+
for (const [d, v] of Object.entries(rec.days)) a.days[d] = { ...v };
|
|
51
|
+
}
|
|
52
|
+
l._accounts.set(name, a);
|
|
53
|
+
}
|
|
54
|
+
} catch { return new CapacityLedger({ now }); }
|
|
55
|
+
return l;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
serialize() {
|
|
59
|
+
const accounts = {};
|
|
60
|
+
for (const [name, rec] of this._accounts) {
|
|
61
|
+
accounts[name] = {
|
|
62
|
+
ses: { open: rec.ses.open ? { ...rec.ses.open } : null, closed: rec.ses.closed.slice() },
|
|
63
|
+
wk: { open: rec.wk.open ? { ...rec.wk.open } : null, closed: rec.wk.closed.slice() },
|
|
64
|
+
days: Object.fromEntries(Object.entries(rec.days).map(([d, v]) => [d, { ...v }])),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
return { schemaVersion: SCHEMA_VERSION, accounts };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
_rec(name) {
|
|
71
|
+
let r = this._accounts.get(name);
|
|
72
|
+
if (!r) {
|
|
73
|
+
r = { ses: { open: null, closed: [] }, wk: { open: null, closed: [] }, days: {} };
|
|
74
|
+
this._accounts.set(name, r);
|
|
75
|
+
}
|
|
76
|
+
return r;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// ── Accrual ──────────────────────────────────────────────────────────────────
|
|
80
|
+
|
|
81
|
+
/** Accrue one served request's tokens. `tokens` = {input, output} for THIS request,
|
|
82
|
+
* already per-request-max semantics for output (the caller derives them; see
|
|
83
|
+
* M3 in the pre-mortem — Anthropic interim deltas are CUMULATIVE, so the SSE seam
|
|
84
|
+
* passes the running max, and this adds it exactly once per request).
|
|
85
|
+
* count_tokens requests are the CALLER's job to skip (M4) — they never reach here. */
|
|
86
|
+
accrue(name, { input, output }, at = this._now()) {
|
|
87
|
+
if (!(input > 0) && !(output > 0)) return;
|
|
88
|
+
const rec = this._rec(name);
|
|
89
|
+
for (const w of ['ses', 'wk']) {
|
|
90
|
+
// Open the window lazily if nothing is open (a mid-cycle boot or a window whose
|
|
91
|
+
// reset stamp was never learned). startedAt is the accrual time then — the cycle
|
|
92
|
+
// will be flagged partial by the caller if the boot gap warrants it (B1/B2).
|
|
93
|
+
if (!rec[w].open) {
|
|
94
|
+
rec[w].open = { startedAt: at, tokensSoFar: 0, lastAccrualAt: at, complete: true, disabledDuring: false };
|
|
95
|
+
}
|
|
96
|
+
rec[w].open.tokensSoFar += input + output;
|
|
97
|
+
rec[w].open.lastAccrualAt = at;
|
|
98
|
+
}
|
|
99
|
+
const day = new Date(at).toISOString().slice(0, 10);
|
|
100
|
+
rec.days[day] = rec.days[day] || { tokens: 0, partial: false };
|
|
101
|
+
rec.days[day].tokens += input + output;
|
|
102
|
+
this._evictDays(rec);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
_evictDays(rec) {
|
|
106
|
+
const keys = Object.keys(rec.days).sort();
|
|
107
|
+
while (keys.length > MAX_DAY_BUCKETS) {
|
|
108
|
+
delete rec.days[keys.shift()];
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// ── Cycle lifecycle ─────────────────────────────────────────────────────────
|
|
113
|
+
|
|
114
|
+
/** Mark the open cycles as partial (maxpool was down / an account was disabled).
|
|
115
|
+
* Called by the boot path when a gap is detected (B2), and by the disable hook. */
|
|
116
|
+
markPartial(name, { disabled = false } = {}) {
|
|
117
|
+
const rec = this._accounts.get(name);
|
|
118
|
+
if (!rec) return;
|
|
119
|
+
for (const w of ['ses', 'wk']) {
|
|
120
|
+
if (!rec[w].open) continue;
|
|
121
|
+
// TWO INDEPENDENT axes, deliberately not collapsed: `complete:false` = maxpool
|
|
122
|
+
// was down for part of the cycle; `disabledDuring` = the operator took the
|
|
123
|
+
// account out of rotation. Setting both for a disable made the disabled flag
|
|
124
|
+
// query-redundant and therefore untested (red-team F6-1) — each now excludes on
|
|
125
|
+
// its own, and each is pinned by its own test.
|
|
126
|
+
if (disabled) rec[w].open.disabledDuring = true;
|
|
127
|
+
else rec[w].open.complete = false;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Close the open cycle for a window (M5: clock-authoritative — the close is keyed
|
|
132
|
+
* on `endedAt`, which the caller derives from the reset stamp or the clock, and the
|
|
133
|
+
* cycle keeps its own book regardless of probe health). No-op if none open. */
|
|
134
|
+
closeCycle(name, window, endedAt = this._now(), { resetAt = null } = {}) {
|
|
135
|
+
const rec = this._accounts.get(name);
|
|
136
|
+
if (!rec || !rec[window]?.open) return null;
|
|
137
|
+
const open = rec[window].open;
|
|
138
|
+
// ONE CYCLE PER BOUNDARY CLOSURE — the structural backstop for the two-closer race
|
|
139
|
+
// (round-2 F1). If both closers observe the SAME reset stamp within the same
|
|
140
|
+
// rollover moment, the second close's tokens are a straddling tail of that same
|
|
141
|
+
// window, so they FOLD INTO that cycle instead of becoming a tiny fabricated second
|
|
142
|
+
// one that drags every average down. A LATER window reporting the same numeric stamp
|
|
143
|
+
// value (clock coincidence) is distinguished by endedAt. Pinned by I3.
|
|
144
|
+
const prev = rec[window].closed[rec[window].closed.length - 1];
|
|
145
|
+
// Same boundary within a few seconds, not just byte-identical stamps: the two
|
|
146
|
+
// closers can observe one rollover through slightly different stamps (jitter), and
|
|
147
|
+
// an exact-match-only fold then admitted a 0.2-minute sliver as its own cycle.
|
|
148
|
+
const sameBoundary = resetAt != null && prev
|
|
149
|
+
&& Math.abs((prev.resetAt ?? prev.endedAt) - resetAt) <= SAME_BOUNDARY_MS
|
|
150
|
+
&& Math.abs(prev.endedAt - endedAt) <= SAME_BOUNDARY_MS;
|
|
151
|
+
if (sameBoundary && open.complete && !open.disabledDuring) {
|
|
152
|
+
// Fold ONLY a complete tail: folding a partial/disabled tail would flip the
|
|
153
|
+
// flags on the prior legitimate observation and ERASE it from the averages
|
|
154
|
+
// (round 3, RT3-2) — strictly worse than leaving a tiny excluded cycle.
|
|
155
|
+
prev.tokens += open.tokensSoFar;
|
|
156
|
+
rec[window].open = null;
|
|
157
|
+
return prev;
|
|
158
|
+
}
|
|
159
|
+
rec[window].closed.push({
|
|
160
|
+
startedAt: open.startedAt,
|
|
161
|
+
endedAt,
|
|
162
|
+
tokens: open.tokensSoFar,
|
|
163
|
+
complete: open.complete,
|
|
164
|
+
disabledDuring: open.disabledDuring,
|
|
165
|
+
resetAt,
|
|
166
|
+
});
|
|
167
|
+
if (rec[window].closed.length > MAX_CYCLES_PER_WINDOW) rec[window].closed.shift();
|
|
168
|
+
rec[window].open = null;
|
|
169
|
+
return rec[window].closed[rec[window].closed.length - 1];
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// ── Queries ─────────────────────────────────────────────────────────────────
|
|
173
|
+
|
|
174
|
+
/** The columns the TUI renders: last, prev, prev1, avg3, avg10, allTime — over
|
|
175
|
+
* COMPLETE, not-disabled cycles only (D4/D5: partial and operator-disabled cycles
|
|
176
|
+
* are observations, not capacity). */
|
|
177
|
+
windowStats(name, window) {
|
|
178
|
+
const rec = this._accounts.get(name);
|
|
179
|
+
const closed = (rec?.[window]?.closed || []).filter(c => c.complete && !c.disabledDuring);
|
|
180
|
+
if (!closed.length) return null;
|
|
181
|
+
const avg = (arr) => arr.length ? Math.round(arr.reduce((a, b) => a + b, 0) / arr.length) : null;
|
|
182
|
+
const tokens = closed.map(c => c.tokens);
|
|
183
|
+
return {
|
|
184
|
+
last: closed[closed.length - 1].tokens,
|
|
185
|
+
prev: closed.length >= 2 ? closed[closed.length - 2].tokens : null,
|
|
186
|
+
prev1: closed.length >= 3 ? closed[closed.length - 3].tokens : null,
|
|
187
|
+
avg3: avg(tokens.slice(-3)),
|
|
188
|
+
avg10: avg(tokens.slice(-10)),
|
|
189
|
+
allTime: avg(tokens),
|
|
190
|
+
cycles: closed.length,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Rolling-7d throughput (the no-weekly account's weekly figure). The window is
|
|
195
|
+
* keyed on CALENDAR days — [today-6 .. today] UTC — NOT "the last 7 buckets":
|
|
196
|
+
* buckets exist only where accrual happened, so a bucket-slice silently stretches
|
|
197
|
+
* across idle gaps and over-reports (red-team F4). A missing day contributes 0 by
|
|
198
|
+
* ABSENCE (present in dayKeys, absent from days) — and with MAX_DAY_BUCKETS=10 an
|
|
199
|
+
* idle day no longer even evicts; only real activity ages out. `partial` is true
|
|
200
|
+
* when any bucket in the window is flagged partial — the figure is ≤ observed. */
|
|
201
|
+
rollingThroughput(name, days = 7) {
|
|
202
|
+
const rec = this._accounts.get(name);
|
|
203
|
+
if (!rec) return { tokens: 0, partial: false };
|
|
204
|
+
// The window is anchored on the ledger's OWN clock — the last `days` CALENDAR days
|
|
205
|
+
// ending today. An earlier "latest recorded day" anchor reported a weeks-old window
|
|
206
|
+
// as if it were current (idle GLM fallback showed a stale 25M "7d volume" with no
|
|
207
|
+
// disclosure — red-team round 2, F2). Now: idle → 0, honestly.
|
|
208
|
+
const today = new Date(this._now()).toISOString().slice(0, 10);
|
|
209
|
+
const cutoff = this._utcDayMinus(today, Math.min(days - 1, MAX_DAY_BUCKETS - 1));
|
|
210
|
+
let tokens = 0, partial = false;
|
|
211
|
+
for (const [d, v] of Object.entries(rec.days)) {
|
|
212
|
+
if (d >= cutoff && d <= today) { tokens += v.tokens; if (v.partial) partial = true; }
|
|
213
|
+
}
|
|
214
|
+
return { tokens, partial };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
_utcDayMinus(day, n) {
|
|
218
|
+
const d = new Date(`${day}T00:00:00Z`);
|
|
219
|
+
d.setUTCDate(d.getUTCDate() - n);
|
|
220
|
+
return d.toISOString().slice(0, 10);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Merge a DELTA of accrual into a base payload (B2 drain-exit merge-flush).
|
|
225
|
+
*
|
|
226
|
+
* The released worker keeps serving in-flight requests for up to RELOAD_DRAIN_MS
|
|
227
|
+
* AFTER its final flush, and then exits bare — so every token delivered during the
|
|
228
|
+
* drain was measured and thrown away. It cannot simply re-write its own ledger:
|
|
229
|
+
* the NEW worker owns the file by then and has its own accrual. So at exit it
|
|
230
|
+
* computes what it accrued SINCE its final flush (after − before) and adds only
|
|
231
|
+
* that delta onto whatever the new worker has on disk.
|
|
232
|
+
*
|
|
233
|
+
* Adds to the OPEN cycles and the day buckets — the only places drain-time tokens
|
|
234
|
+
* can land. A cycle the new worker already closed is not re-opened (the tokens
|
|
235
|
+
* belonged to a window that has since rolled; dropping them is correct, and the
|
|
236
|
+
* cycle is a completed observation we must not mutate after the fact).
|
|
237
|
+
*/
|
|
238
|
+
static mergeDelta(basePayload, beforePayload, afterPayload) {
|
|
239
|
+
const base = (basePayload && basePayload.schemaVersion === SCHEMA_VERSION)
|
|
240
|
+
? JSON.parse(JSON.stringify(basePayload))
|
|
241
|
+
: { schemaVersion: SCHEMA_VERSION, accounts: {} };
|
|
242
|
+
const before = beforePayload?.accounts || {};
|
|
243
|
+
const after = afterPayload?.accounts || {};
|
|
244
|
+
for (const [name, aRec] of Object.entries(after)) {
|
|
245
|
+
const bRec = before[name] || {};
|
|
246
|
+
for (const w of ['ses', 'wk']) {
|
|
247
|
+
const aOpen = aRec[w]?.open, bOpen = bRec[w]?.open;
|
|
248
|
+
if (!aOpen) continue;
|
|
249
|
+
const target = base.accounts[name]?.[w];
|
|
250
|
+
// The BASE's open cycle is the one being amended, so the same-cycle check is
|
|
251
|
+
// against IT — not against our own before-snapshot (which trivially matches
|
|
252
|
+
// our own after-snapshot and so never fired). A different startedAt means the
|
|
253
|
+
// window rolled during the drain: the delta belongs to a window the new worker
|
|
254
|
+
// has already closed, and crediting it to the fresh cycle would inflate the
|
|
255
|
+
// very next capacity reading by a whole window of traffic (red-team F6-3).
|
|
256
|
+
if (!target?.open) continue;
|
|
257
|
+
if (target.open.startedAt !== aOpen.startedAt) continue;
|
|
258
|
+
const delta = (bOpen && bOpen.startedAt === aOpen.startedAt)
|
|
259
|
+
? aOpen.tokensSoFar - bOpen.tokensSoFar
|
|
260
|
+
: aOpen.tokensSoFar;
|
|
261
|
+
if (!(delta > 0)) continue;
|
|
262
|
+
target.open.tokensSoFar += delta;
|
|
263
|
+
target.open.lastAccrualAt = Math.max(target.open.lastAccrualAt || 0, aOpen.lastAccrualAt || 0);
|
|
264
|
+
}
|
|
265
|
+
for (const [day, v] of Object.entries(aRec.days || {})) {
|
|
266
|
+
const bTok = bRec.days?.[day]?.tokens || 0;
|
|
267
|
+
const delta = (v.tokens || 0) - bTok;
|
|
268
|
+
if (!(delta > 0)) continue;
|
|
269
|
+
base.accounts[name] = base.accounts[name]
|
|
270
|
+
|| { ses: { open: null, closed: [] }, wk: { open: null, closed: [] }, days: {} };
|
|
271
|
+
const days = base.accounts[name].days;
|
|
272
|
+
days[day] = days[day] || { tokens: 0, partial: false };
|
|
273
|
+
days[day].tokens += delta;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
return base;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
dayKeys(name) { return Object.keys(this._accounts.get(name)?.days || {}).sort(); }
|
|
280
|
+
openCycle(name, window) { return this._accounts.get(name)?.[window]?.open || null; }
|
|
281
|
+
accounts() { return [...this._accounts.keys()]; }
|
|
282
|
+
markDayPartial(name, utcDay) {
|
|
283
|
+
const rec = this._accounts.get(name);
|
|
284
|
+
if (rec?.days[utcDay]) rec.days[utcDay].partial = true;
|
|
285
|
+
}
|
|
286
|
+
}
|
package/src/config.js
CHANGED
|
@@ -289,6 +289,22 @@ export async function readGeneration(path) {
|
|
|
289
289
|
*/
|
|
290
290
|
let _stateWriteChain = Promise.resolve();
|
|
291
291
|
|
|
292
|
+
/**
|
|
293
|
+
* Amended state write at an UNCHANGED generation (drain-exit capacity merge-flush).
|
|
294
|
+
* saveState always bumps `_generation`; a released worker amending the file the NEW
|
|
295
|
+
* lease-holder owns must NOT bump it, or the holder's `stateGeneration` is stale from
|
|
296
|
+
* that instant and every guarded periodic write is refused for its entire tenure.
|
|
297
|
+
* Serialized onto the same write chain; NEVER used for ordinary state updates.
|
|
298
|
+
*/
|
|
299
|
+
export function saveStateUnbumped(state, generation) {
|
|
300
|
+
const run = async () => {
|
|
301
|
+
await atomicWrite(getStatePath(), JSON.stringify({ ...state, _generation: generation }, null, 2) + '\n');
|
|
302
|
+
};
|
|
303
|
+
const result = _stateWriteChain.then(run, run);
|
|
304
|
+
_stateWriteChain = result.then(() => {}, () => {});
|
|
305
|
+
return result;
|
|
306
|
+
}
|
|
307
|
+
|
|
292
308
|
export function saveState(state, { expectedGeneration = null } = {}) {
|
|
293
309
|
// Serialize state writes (a 60s interval flush can otherwise race the final
|
|
294
310
|
// baton flush, read the same on-disk generation, and double-write).
|
package/src/index.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
3
|
import net from 'node:net';
|
|
4
|
+
import { stat } from 'node:fs/promises';
|
|
4
5
|
import { spawn, spawnSync } from 'node:child_process';
|
|
5
6
|
import { createInterface } from 'node:readline';
|
|
6
|
-
import { loadOrCreateConfig, loadConfig, saveConfig, atomicConfigUpdate, getConfigPath, loadState, saveState, getStatePath, getLogPath, readGeneration, flushConfigWrites, flushStateWrites } from './config.js';
|
|
7
|
+
import { loadOrCreateConfig, loadConfig, saveConfig, atomicConfigUpdate, getConfigPath, loadState, saveState, getStatePath, getLogPath, readGeneration, flushConfigWrites, flushStateWrites, saveStateUnbumped } from './config.js';
|
|
7
8
|
import { setEventLogPath, installConsoleMirror, setConsoleStdoutSuppressed } from './event-log.js';
|
|
8
9
|
import { SleepGuard } from './sleep-guard.js';
|
|
9
10
|
import { AccountManager } from './account-manager.js';
|
|
@@ -27,6 +28,7 @@ net.setDefaultAutoSelectFamilyAttemptTimeout(
|
|
|
27
28
|
Math.max(1000, Number(process.env.MAXPOOL_FAMILY_ATTEMPT_TIMEOUT_MS) || 5000),
|
|
28
29
|
);
|
|
29
30
|
import { Prober } from './prober.js';
|
|
31
|
+
import { CapacityLedger } from './capacity-ledger.js';
|
|
30
32
|
import { loginOAuth, fetchProfile, refreshAccessToken, isTokenExpiringSoon, tokenFingerprint } from './oauth.js';
|
|
31
33
|
import { TUI } from './tui.js';
|
|
32
34
|
import { RestartController } from './restart-controller.js';
|
|
@@ -606,6 +608,18 @@ async function serverWorkerCommand() {
|
|
|
606
608
|
// lease holder); a cold/direct worker reads the on-disk state file.
|
|
607
609
|
const savedState = await loadState();
|
|
608
610
|
if (savedState?.quota) accountManager.restoreQuotaState(savedState.quota);
|
|
611
|
+
if (savedState?.capacity) {
|
|
612
|
+
// Measured maxpool downtime = state-file last write → boot. A seamless reload
|
|
613
|
+
// worker's state was handed over live (no gap) → null skips the partial check.
|
|
614
|
+
let downtimeMs = null;
|
|
615
|
+
try {
|
|
616
|
+
if (!isReloadWorker) {
|
|
617
|
+
const st = await stat(getStatePath());
|
|
618
|
+
downtimeMs = Date.now() - st.mtimeMs;
|
|
619
|
+
}
|
|
620
|
+
} catch { downtimeMs = null; }
|
|
621
|
+
accountManager.restoreCapacityState(savedState.capacity, Date.now(), downtimeMs);
|
|
622
|
+
}
|
|
609
623
|
// Runtime fallback providers (glm-fallback/kimi-fallback) are created lazily from
|
|
610
624
|
// `cc all` request headers, not config — so without restoring them here a restart
|
|
611
625
|
// shows only the config OAuth accounts until the next `cc all` request re-sends the
|
|
@@ -675,12 +689,47 @@ async function serverWorkerCommand() {
|
|
|
675
689
|
// Runtime providers so glm-fallback/kimi-fallback survive a restart (they're
|
|
676
690
|
// header-derived, not in config). Empty [] when none — harmless.
|
|
677
691
|
runtimeProviders: accountManager.exportRuntimeProviders(),
|
|
692
|
+
// CAPACITY LEDGER (2026-08-22) rides the SAME payload on purpose: it inherits
|
|
693
|
+
// the writer lease, the generation guard and the serialized write chain. A
|
|
694
|
+
// separate file would need its own copy of all three (pre-mortem B2), and the
|
|
695
|
+
// OPEN cycle must persist or a mid-cycle restart records a near-empty cycle as
|
|
696
|
+
// a real capacity observation and poisons the averages forever (B1).
|
|
697
|
+
capacity: accountManager.capacity?.serialize?.() || null,
|
|
678
698
|
},
|
|
679
699
|
{ expectedGeneration },
|
|
680
700
|
)
|
|
681
701
|
.then(written => { if (written != null) stateGeneration = written; })
|
|
682
702
|
.catch(() => {});
|
|
683
703
|
};
|
|
704
|
+
// CAPACITY LEDGER drain-exit merge-flush (B2). `capacityFlushSnapshot` is taken at
|
|
705
|
+
// the released worker's FINAL flush; everything accrued after it is drain-time work
|
|
706
|
+
// whose tokens would otherwise be discarded when this worker exits bare.
|
|
707
|
+
let capacityFlushSnapshot = null;
|
|
708
|
+
const mergeFlushCapacityDelta = async () => {
|
|
709
|
+
if (!capacityFlushSnapshot) return;
|
|
710
|
+
const after = accountManager.capacity?.serialize?.();
|
|
711
|
+
if (!after) return;
|
|
712
|
+
const disk = await loadState();
|
|
713
|
+
if (!disk) { // a failed read must not be "merged" into a quota-less state file (RT2-4)
|
|
714
|
+
console.error('[Maxpool] Capacity drain-flush skipped: state file unreadable — drain-time token delta not persisted.');
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
const capacity = CapacityLedger.mergeDelta(disk.capacity, capacityFlushSnapshot, after);
|
|
718
|
+
// Re-write the FULL on-disk state with only `capacity` replaced: quota and
|
|
719
|
+
// runtimeProviders belong to the NEW worker now and must not be reverted to ours.
|
|
720
|
+
// No generation guard — we are deliberately amending a file another writer owns,
|
|
721
|
+
// and a same-instant race costs at most this drain's delta.
|
|
722
|
+
const { _generation, ...rest } = disk;
|
|
723
|
+
// NO generation bump: re-write the file at the SAME generation so the new
|
|
724
|
+
// worker's stateGeneration stays valid. saveState refuses a lower generation
|
|
725
|
+
// only — writing the observed one back is accepted and bumps nothing. Bumping
|
|
726
|
+
// here (the naive first version) wedged the new worker's periodic flush for its
|
|
727
|
+
// entire tenure: onDisk advanced past its stateGeneration and every guarded
|
|
728
|
+
// write was refused forever (red-team 2026-08-22).
|
|
729
|
+
await saveStateUnbumped({ ...rest, capacity }, _generation ?? 0);
|
|
730
|
+
await flushStateWrites();
|
|
731
|
+
};
|
|
732
|
+
|
|
684
733
|
// Persist quota every minute; unref so it never keeps the process alive.
|
|
685
734
|
let quotaSaveInterval = null;
|
|
686
735
|
|
|
@@ -1339,6 +1388,11 @@ async function serverWorkerCommand() {
|
|
|
1339
1388
|
// flipped hasLease=false, and persistQuotaState no-ops without the lease —
|
|
1340
1389
|
// so an unforced call here silently drops the reload's final quota flush.
|
|
1341
1390
|
await persistQuotaState(true);
|
|
1391
|
+
// 3b) CAPACITY LEDGER: snapshot what we had flushed. We keep serving in-flight
|
|
1392
|
+
// requests for up to RELOAD_DRAIN_MS after this, and those tokens are real
|
|
1393
|
+
// measured capacity — but the NEW worker owns the state file by then. At
|
|
1394
|
+
// exit we merge (now − this snapshot) onto whatever it has written (B2).
|
|
1395
|
+
capacityFlushSnapshot = accountManager.capacity?.serialize?.() || null;
|
|
1342
1396
|
// 4) Barrier: ensure every queued config write (a rotated-token persist is
|
|
1343
1397
|
// fire-and-forget) AND state write has actually hit disk before we hand
|
|
1344
1398
|
// off (M3 — otherwise the new worker boots from the invalidated token).
|
|
@@ -1368,14 +1422,22 @@ async function serverWorkerCommand() {
|
|
|
1368
1422
|
// exit(0). Cap = RELOAD_DRAIN_MS (above the idle reaper) so a long streaming
|
|
1369
1423
|
// response finishes instead of being cut; the supervisor SIGKILLs us only if
|
|
1370
1424
|
// we outlive its (slightly longer) cap.
|
|
1425
|
+
// Exit only AFTER folding the drain-time capacity delta into the new worker's
|
|
1426
|
+
// state file. Bounded (1s) and best-effort: losing the delta is a measurement
|
|
1427
|
+
// gap, hanging the released worker is an operational fault, so the timeout wins.
|
|
1428
|
+
const exitAfterMergeFlush = () => {
|
|
1429
|
+
Promise.race([mergeFlushCapacityDelta(), delay(1000)])
|
|
1430
|
+
.catch(() => {})
|
|
1431
|
+
.finally(() => process.exit(0));
|
|
1432
|
+
};
|
|
1371
1433
|
const waitForDrain = () => {
|
|
1372
|
-
if (restartController.activeRequests.size === 0) {
|
|
1434
|
+
if (restartController.activeRequests.size === 0) { exitAfterMergeFlush(); return; }
|
|
1373
1435
|
};
|
|
1374
1436
|
const drainPoll = setInterval(waitForDrain, 200);
|
|
1375
1437
|
drainPoll.unref?.();
|
|
1376
1438
|
const hardCap = setTimeout(() => {
|
|
1377
1439
|
console.error(`[Maxpool] Released worker reload-drain cap reached with ${restartController.activeRequests.size} active; exiting.`);
|
|
1378
|
-
|
|
1440
|
+
exitAfterMergeFlush();
|
|
1379
1441
|
}, RELOAD_DRAIN_MS);
|
|
1380
1442
|
hardCap.unref?.();
|
|
1381
1443
|
waitForDrain();
|
package/src/prober.js
CHANGED
|
@@ -70,6 +70,12 @@ export class Prober {
|
|
|
70
70
|
this._stopping = false;
|
|
71
71
|
this._inflight = (async () => {
|
|
72
72
|
try {
|
|
73
|
+
// CAPACITY LEDGER: close any open cycle whose window's reset stamp has
|
|
74
|
+
// passed. Runs at the top of every sweep (lease-holder only — probeAll
|
|
75
|
+
// fires from the prober, which start/stop with the lease) so a window that
|
|
76
|
+
// rolled over between requests still closes promptly, not only on the next
|
|
77
|
+
// accrual or stamp-advance.
|
|
78
|
+
try { this.am.closeExpiredCapacityCycles?.(); } catch { /* never block probing */ }
|
|
73
79
|
// DISABLED accounts are INTENTIONALLY still probed (no `a.enabled` filter):
|
|
74
80
|
// a user often disables an account precisely BECAUSE it's exhausted, and still
|
|
75
81
|
// wants to see its quota recover — so keep refreshing its usage for visibility
|
package/src/server.js
CHANGED
|
@@ -1285,7 +1285,19 @@ async function forwardRequest(
|
|
|
1285
1285
|
|
|
1286
1286
|
if (isStreaming) {
|
|
1287
1287
|
const streamLog = logDir ? [] : null;
|
|
1288
|
-
|
|
1288
|
+
// CAPACITY LEDGER: accrue in a FINALLY — a stream that dies mid-flight throws
|
|
1289
|
+
// from streamResponse, and the tokens genuinely delivered before the failure
|
|
1290
|
+
// are real capacity; skipping them under-counts exactly the longest
|
|
1291
|
+
// generations (red-team F5). One accrual per request, per-stream running max
|
|
1292
|
+
// (M3 — cumulative interim deltas must not be summed).
|
|
1293
|
+
try {
|
|
1294
|
+
await streamResponse(upstreamRes.body, res, upstreamRes.status, responseHeaders, account.index, accountManager, streamLog, requestInfo);
|
|
1295
|
+
} finally {
|
|
1296
|
+
accountManager.accrueCapacity?.(account.index, {
|
|
1297
|
+
input: requestInfo._capacityInput || 0,
|
|
1298
|
+
output: requestInfo._capacityOutput || 0,
|
|
1299
|
+
});
|
|
1300
|
+
}
|
|
1289
1301
|
accountManager.releaseAccount(lease, { success: true, status: upstreamRes.status });
|
|
1290
1302
|
if (logDir) {
|
|
1291
1303
|
logSections.push(`=== RESPONSE BODY (streamed) ===\n${streamLog.join('')}`);
|
|
@@ -1312,7 +1324,7 @@ async function forwardRequest(
|
|
|
1312
1324
|
clearTimeout(bodyTimer);
|
|
1313
1325
|
}
|
|
1314
1326
|
const buf = Buffer.from(arr);
|
|
1315
|
-
extractUsageFromBody(buf, account.index, accountManager);
|
|
1327
|
+
extractUsageFromBody(buf, account.index, accountManager, requestInfo);
|
|
1316
1328
|
markThinkingFromResponse(buf, accountManager, requestInfo);
|
|
1317
1329
|
accountManager.releaseAccount(lease, { success: upstreamRes.status < 500, status: upstreamRes.status });
|
|
1318
1330
|
if (logDir) {
|
|
@@ -2914,10 +2926,18 @@ function parseSSEEvent(event, accountIndex, accountManager, requestInfo = {}) {
|
|
|
2914
2926
|
|
|
2915
2927
|
try {
|
|
2916
2928
|
const data = JSON.parse(dataLine.slice(6));
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2929
|
+
// CAPACITY LEDGER (2026-08-22): stream-level usage feeds the per-cycle ledger.
|
|
2930
|
+
// M3: Anthropic interim message_delta usage is CUMULATIVE — add-semantics inflates
|
|
2931
|
+
// output on long generations. Track a per-stream RUNNING MAX, accrue once at end.
|
|
2932
|
+
// M4: count_tokens responses are prompt-size echoes, not delivered work — skip.
|
|
2933
|
+
if (!requestInfo.isCountTokens) {
|
|
2934
|
+
if (data.type === 'message_start' && data.message?.usage) {
|
|
2935
|
+
accountManager.updateUsage(accountIndex, data.message.usage.input_tokens, 0);
|
|
2936
|
+
requestInfo._capacityInput = Math.max(requestInfo._capacityInput || 0, data.message.usage.input_tokens || 0);
|
|
2937
|
+
} else if (data.type === 'message_delta' && data.usage) {
|
|
2938
|
+
accountManager.updateUsage(accountIndex, 0, data.usage.output_tokens);
|
|
2939
|
+
requestInfo._capacityOutput = Math.max(requestInfo._capacityOutput || 0, data.usage.output_tokens || 0);
|
|
2940
|
+
}
|
|
2921
2941
|
}
|
|
2922
2942
|
if (sseEventContainsThinking(data)) {
|
|
2923
2943
|
accountManager.markSessionThinkingProtected?.(requestInfo.sessionKey, requestInfo.model);
|
|
@@ -2933,11 +2953,20 @@ function sseEventContainsThinking(data) {
|
|
|
2933
2953
|
|| data?.delta?.type === 'signature_delta';
|
|
2934
2954
|
}
|
|
2935
2955
|
|
|
2936
|
-
function extractUsageFromBody(buffer, accountIndex, accountManager) {
|
|
2956
|
+
function extractUsageFromBody(buffer, accountIndex, accountManager, requestInfo = {}) {
|
|
2937
2957
|
try {
|
|
2938
2958
|
const json = JSON.parse(buffer.toString());
|
|
2939
2959
|
if (json.usage) {
|
|
2940
2960
|
accountManager.updateUsage(accountIndex, json.usage.input_tokens, json.usage.output_tokens);
|
|
2961
|
+
// CAPACITY LEDGER — M4: a /count_tokens response is a prompt-size echo with ZERO
|
|
2962
|
+
// work delivered. Claude Code calls it constantly; counting it would inflate every
|
|
2963
|
+
// cycle by whole prompt sizes.
|
|
2964
|
+
if (!requestInfo.isCountTokens) {
|
|
2965
|
+
accountManager.accrueCapacity?.(accountIndex, {
|
|
2966
|
+
input: json.usage.input_tokens || 0,
|
|
2967
|
+
output: json.usage.output_tokens || 0,
|
|
2968
|
+
});
|
|
2969
|
+
}
|
|
2941
2970
|
}
|
|
2942
2971
|
} catch {
|
|
2943
2972
|
// not JSON or no usage
|
package/src/tui.js
CHANGED
|
@@ -68,7 +68,10 @@ function rpad(s, w) {
|
|
|
68
68
|
return gap > 0 ? s + ' '.repeat(gap) : s;
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
-
/** Truncate a string with ANSI codes to exactly w visible characters, then reset.
|
|
71
|
+
/** Truncate a string with ANSI codes to exactly w visible characters, then reset.
|
|
72
|
+
* The trailing RESET is appended only when something was actually cut — a no-op
|
|
73
|
+
* truncate that unconditionally appended it shifted `.padEnd()` (raw-length) fields
|
|
74
|
+
* 4 columns narrow for every string shorter than the width (red-team round 3, RT3-4). */
|
|
72
75
|
function truncate(s, w) {
|
|
73
76
|
let visible = 0;
|
|
74
77
|
let out = '';
|
|
@@ -82,7 +85,7 @@ function truncate(s, w) {
|
|
|
82
85
|
visible++;
|
|
83
86
|
i++;
|
|
84
87
|
}
|
|
85
|
-
return out + RESET;
|
|
88
|
+
return visible < w ? out + ' '.repeat(w - visible) + RESET : out + RESET;
|
|
86
89
|
}
|
|
87
90
|
|
|
88
91
|
/** Fit a line to exactly w columns: truncate if too long, pad if too short. */
|
|
@@ -116,6 +119,16 @@ function quotaLabel(ratio, resetTs, width) {
|
|
|
116
119
|
return (rst || pct).slice(0, width);
|
|
117
120
|
}
|
|
118
121
|
|
|
122
|
+
/** Compact token count for the capacity page. Tokens are the apples-to-apples unit
|
|
123
|
+
* across Claude / GLM / Kimi, so the column must stay narrow and scannable. */
|
|
124
|
+
function formatTokens(n) {
|
|
125
|
+
if (n == null) return '--';
|
|
126
|
+
if (n >= 1_000_000_000) return `${(n / 1_000_000_000).toFixed(1)}B`;
|
|
127
|
+
if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(n >= 10_000_000 ? 0 : 1)}M`;
|
|
128
|
+
if (n >= 1_000) return `${Math.round(n / 1_000)}k`;
|
|
129
|
+
return String(n);
|
|
130
|
+
}
|
|
131
|
+
|
|
119
132
|
function formatMs(ms) {
|
|
120
133
|
if (ms == null || isNaN(ms)) return '-';
|
|
121
134
|
if (ms < 1000) return `${Math.round(ms)}ms`;
|
|
@@ -257,7 +270,8 @@ export class TUI {
|
|
|
257
270
|
|
|
258
271
|
this.log = []; // completed activity entries
|
|
259
272
|
this.active = new Map(); // in-flight requests
|
|
260
|
-
this.mode = 'normal'; // normal | accounts | routing | select | input | confirm | providers | addtype
|
|
273
|
+
this.mode = 'normal'; // normal | accounts | routing | capacity | select | input | confirm | providers | addtype
|
|
274
|
+
this.capacityWindow = 'ses'; // capacity page window: 'ses' (5h) | 'wk' (weekly)
|
|
261
275
|
// Hide disabled accounts from the table. With 8 dead/disabled accounts the live
|
|
262
276
|
// ones scroll off the top; `h` collapses them to a one-line summary.
|
|
263
277
|
this.hideDisabled = false;
|
|
@@ -408,6 +422,7 @@ export class TUI {
|
|
|
408
422
|
case 'accounts': this._keyAccounts(k); break;
|
|
409
423
|
case 'routing': this._keyRouting(k); break;
|
|
410
424
|
case 'updates': this._keyUpdates(k); break;
|
|
425
|
+
case 'capacity': this._keyCapacity(k); break;
|
|
411
426
|
case 'addtype': this._keyAddType(k); break;
|
|
412
427
|
case 'select': this._keySelect(k); break;
|
|
413
428
|
case 'input': this._keyInput(k); break;
|
|
@@ -451,6 +466,8 @@ export class TUI {
|
|
|
451
466
|
);
|
|
452
467
|
} else if (k === 'u') {
|
|
453
468
|
this.mode = 'updates';
|
|
469
|
+
} else if (k === 'y') {
|
|
470
|
+
this.mode = 'capacity';
|
|
454
471
|
} else if (k === 'h') {
|
|
455
472
|
this.hideDisabled = !this.hideDisabled;
|
|
456
473
|
this._addLog(this.hideDisabled ? 'Hiding disabled accounts' : 'Showing all accounts');
|
|
@@ -1598,6 +1615,19 @@ export class TUI {
|
|
|
1598
1615
|
lines.push(` ${cyan(' Parked')} ${dim(`${queuedCount} request${queuedCount === 1 ? '' : 's'} held oldest ${formatMs(oldest)} · ${why}`)}`);
|
|
1599
1616
|
}
|
|
1600
1617
|
|
|
1618
|
+
// ── Capacity page — a DEDICATED screen, not a footer panel. It replaces the
|
|
1619
|
+
// account/activity body wholesale so the six columns get the full width; the
|
|
1620
|
+
// header above (version, routing, throttle) stays so the operator never loses
|
|
1621
|
+
// where they are. Reads the in-memory ledger only — no fs per frame.
|
|
1622
|
+
if (this.mode === 'capacity') {
|
|
1623
|
+
lines.push(...this._renderCapacityPage(W));
|
|
1624
|
+
while (lines.length < H - 2) lines.push('');
|
|
1625
|
+
lines.push(' ' + dim('─'.repeat(W - 2)));
|
|
1626
|
+
lines.push(this._renderFooter());
|
|
1627
|
+
this._writeScreen(lines, W, H);
|
|
1628
|
+
return;
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1601
1631
|
// ── Accounts
|
|
1602
1632
|
if (this.am.accounts.length === 0) {
|
|
1603
1633
|
lines.push('');
|
|
@@ -1696,7 +1726,12 @@ export class TUI {
|
|
|
1696
1726
|
if (this.mode === 'updates') lines.push(...this._renderUpdatesDetail());
|
|
1697
1727
|
lines.push(this._renderFooter());
|
|
1698
1728
|
|
|
1699
|
-
|
|
1729
|
+
this._writeScreen(lines, W, H);
|
|
1730
|
+
}
|
|
1731
|
+
|
|
1732
|
+
/** Full-screen buffer write: cursor home, H rows fitted to W, then hide the cursor
|
|
1733
|
+
* (shown only in input mode). */
|
|
1734
|
+
_writeScreen(lines, W, H) {
|
|
1700
1735
|
let buf = `${ESC}H`;
|
|
1701
1736
|
for (let i = 0; i < H; i++) {
|
|
1702
1737
|
buf += fitLine(lines[i] || '', W);
|
|
@@ -1955,10 +1990,102 @@ export class TUI {
|
|
|
1955
1990
|
};
|
|
1956
1991
|
}
|
|
1957
1992
|
|
|
1993
|
+
_keyCapacity(k) {
|
|
1994
|
+
if (k === 'w') { this.capacityWindow = this.capacityWindow === 'wk' ? 'ses' : 'wk'; }
|
|
1995
|
+
else if (k === 'esc' || k === 'q') { this.mode = 'normal'; }
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
/**
|
|
1999
|
+
* Capacity page — how much an account ACTUALLY delivers before it runs out.
|
|
2000
|
+
*
|
|
2001
|
+
* Each completed 5h/weekly cycle's token total IS that account's measured capacity
|
|
2002
|
+
* for the window, so the columns compare a Claude account and a GLM account on the
|
|
2003
|
+
* same axis. Only COMPLETE, never-disabled cycles count (a cycle maxpool sat out
|
|
2004
|
+
* part of is an observation, not a capacity).
|
|
2005
|
+
*
|
|
2006
|
+
* The no-weekly account (legacy z.ai plan) has no weekly TANK to fill, so its weekly
|
|
2007
|
+
* row is a rolling-7d THROUGHPUT — a volume, labelled as such, never a limit.
|
|
2008
|
+
*/
|
|
2009
|
+
_renderCapacityPage(W) {
|
|
2010
|
+
const out = [];
|
|
2011
|
+
const win = this.capacityWindow === 'wk' ? 'wk' : 'ses';
|
|
2012
|
+
const ledger = this.am.capacity;
|
|
2013
|
+
const title = win === 'wk' ? 'Weekly (7d) capacity' : 'Session (5h) capacity';
|
|
2014
|
+
out.push('');
|
|
2015
|
+
out.push(` ${bold(title)} ${dim('— tokens delivered per completed cycle, per account')}`);
|
|
2016
|
+
out.push('');
|
|
2017
|
+
|
|
2018
|
+
if (!ledger) { out.push(yellow(' Capacity ledger unavailable on this worker.')); return out; }
|
|
2019
|
+
|
|
2020
|
+
// Narrow terminals: drop trailing columns rather than let fitLine chop a number
|
|
2021
|
+
// mid-digit (at W=80 the full 6-column row is 82+ chars — every row silently lost
|
|
2022
|
+
// its last two cells). The dropped ones are the aggregates, not the observations.
|
|
2023
|
+
const ALL_COLS = ['Last', 'Prev', 'Prev-1', 'Avg 3', 'Avg 10', 'All time'];
|
|
2024
|
+
const CW = 9;
|
|
2025
|
+
const nameW = 12;
|
|
2026
|
+
let COLS = ALL_COLS;
|
|
2027
|
+
while (COLS.length > 1 && (nameW + PROVIDER_W + 2 + COLS.length * CW + 14) > W) {
|
|
2028
|
+
COLS = COLS.slice(0, -1);
|
|
2029
|
+
}
|
|
2030
|
+
const header = ' ' + 'Account'.padEnd(nameW) + ' ' + 'Provider'.padEnd(PROVIDER_W) + ' '
|
|
2031
|
+
+ COLS.map(c => c.padStart(CW)).join('') + ' Cycles';
|
|
2032
|
+
out.push(dimUnderline(fitLine(header, W)));
|
|
2033
|
+
|
|
2034
|
+
let anyData = false;
|
|
2035
|
+
for (const i of this._displayOrder()) {
|
|
2036
|
+
const a = this.am.accounts[i];
|
|
2037
|
+
if (!a) continue;
|
|
2038
|
+
if (this.hideDisabled && a.enabled === false) continue;
|
|
2039
|
+
const noWeekly = win === 'wk' && a.type === 'provider' && a.quota?.weeklyAbsent;
|
|
2040
|
+
const name = truncate(a.name, nameW).padEnd(nameW);
|
|
2041
|
+
const prov = gray(providerLabel(a).padEnd(PROVIDER_W));
|
|
2042
|
+
if (noWeekly) {
|
|
2043
|
+
// No weekly limit — the favourite legacy plan. There is no cycle to complete,
|
|
2044
|
+
// so a capacity number would be a fiction. Show what it actually DID move.
|
|
2045
|
+
const t = ledger.rollingThroughput(a.name, 7);
|
|
2046
|
+
anyData = anyData || t.tokens > 0;
|
|
2047
|
+
const vol = t.tokens > 0 ? formatTokens(t.tokens) : '--';
|
|
2048
|
+
// Always disclose the window's age boundary: today is unfinished, so the 7d
|
|
2049
|
+
// figure grows through the day; a genuinely partial day adds the ≤-observed floor.
|
|
2050
|
+
const note = t.partial
|
|
2051
|
+
? ' (≤ observed — maxpool was down part of the window; includes today, in progress)'
|
|
2052
|
+
: ' (includes today, in progress)';
|
|
2053
|
+
out.push(' ' + name + ' ' + prov + ' '
|
|
2054
|
+
+ cyan(`no weekly limit · 7d volume ${vol}`) + dim(note));
|
|
2055
|
+
continue;
|
|
2056
|
+
}
|
|
2057
|
+
const st = ledger.windowStats(a.name, win);
|
|
2058
|
+
if (!st) {
|
|
2059
|
+
out.push(' ' + name + ' ' + prov + ' ' + dim('no completed cycle yet'));
|
|
2060
|
+
continue;
|
|
2061
|
+
}
|
|
2062
|
+
anyData = true;
|
|
2063
|
+
const all = { Last: st.last, Prev: st.prev, 'Prev-1': st.prev1, 'Avg 3': st.avg3, 'Avg 10': st.avg10, 'All time': st.allTime };
|
|
2064
|
+
const cells = COLS.map(c => formatTokens(all[c]).padStart(CW)).join('');
|
|
2065
|
+
out.push(' ' + name + ' ' + prov + ' ' + cells + ' ' + dim(String(st.cycles)));
|
|
2066
|
+
}
|
|
2067
|
+
|
|
2068
|
+
out.push('');
|
|
2069
|
+
if (!anyData) {
|
|
2070
|
+
// Fresh install: the page is honest about WHY it is empty and WHEN it fills,
|
|
2071
|
+
// instead of showing zeros that read like an account delivering nothing.
|
|
2072
|
+
out.push(' ' + yellow('No completed cycles yet.') + dim(
|
|
2073
|
+
win === 'wk'
|
|
2074
|
+
? ' A weekly figure appears after an account\'s 7d window resets once.'
|
|
2075
|
+
: ' A session figure appears after an account\'s 5h window resets once.'));
|
|
2076
|
+
}
|
|
2077
|
+
out.push(' ' + dim('A cycle counts only if maxpool ran for all of it and the account stayed enabled.'));
|
|
2078
|
+
return out;
|
|
2079
|
+
}
|
|
2080
|
+
|
|
1958
2081
|
_renderFooter() {
|
|
1959
2082
|
switch (this.mode) {
|
|
1960
2083
|
case 'normal':
|
|
1961
|
-
return ` ${bold('a')} Accounts ${bold('m')} Routing ${bold('h')} Hide disabled ${dim('│')} ${bold('u')} Updates ${bold('r')} Restart ${bold('q')} Stop server`;
|
|
2084
|
+
return ` ${bold('a')} Accounts ${bold('m')} Routing ${bold('y')} Capacity ${bold('h')} Hide disabled ${dim('│')} ${bold('u')} Updates ${bold('r')} Restart ${bold('q')} Stop server`;
|
|
2085
|
+
case 'capacity': {
|
|
2086
|
+
const other = this.capacityWindow === 'wk' ? '5h session' : 'weekly (7d)';
|
|
2087
|
+
return ` ${bold('w')} Show ${cyan(other)} ${bold('Esc')} Back`;
|
|
2088
|
+
}
|
|
1962
2089
|
case 'updates': {
|
|
1963
2090
|
const state = this._autoUpdateOn() ? green('on') : dim('off');
|
|
1964
2091
|
return ` ${bold('c')} Check & apply now ${bold('t')} Automatic updates: ${state} ↻ ${bold('Esc')} Back`;
|