maxpool 1.2.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -23
- package/package.json +1 -1
- package/src/account-config.js +5 -15
- package/src/account-manager.js +151 -25
- package/src/config.js +8 -2
- package/src/index.js +8 -63
- package/src/oauth.js +1 -67
- package/src/server.js +149 -26
- package/src/tui.js +36 -34
package/README.md
CHANGED
|
@@ -58,13 +58,6 @@ maxpool server
|
|
|
58
58
|
maxpool run
|
|
59
59
|
```
|
|
60
60
|
|
|
61
|
-
You can also import existing Claude Code credentials instead of logging in:
|
|
62
|
-
|
|
63
|
-
```bash
|
|
64
|
-
claude /login # Log into an account in Claude Code
|
|
65
|
-
maxpool import # Import its credentials
|
|
66
|
-
```
|
|
67
|
-
|
|
68
61
|
## Recommended setup
|
|
69
62
|
|
|
70
63
|
The cleanest way to use maxpool day-to-day: **keep your normal `claude` login untouched, and add a separate alias that routes through the pool.** Then plain `claude` still uses your default single account, and `ccmax` (call it whatever you like) spreads work across all your accounts.
|
|
@@ -121,20 +114,7 @@ Uses the same OAuth flow as Claude Code. Auto-detects the account email and subs
|
|
|
121
114
|
|
|
122
115
|
You can add accounts while the server is running — press **s** in the TUI to sync immediately, or wait for automatic sync.
|
|
123
116
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
If you already have Claude Code set up, you can import its credentials directly:
|
|
127
|
-
|
|
128
|
-
```bash
|
|
129
|
-
claude /login # Log into an account in Claude Code
|
|
130
|
-
maxpool import # Import its credentials
|
|
131
|
-
```
|
|
132
|
-
|
|
133
|
-
Re-importing the same account updates its credentials. You can also import from a custom path:
|
|
134
|
-
|
|
135
|
-
```bash
|
|
136
|
-
maxpool import --from /path/to/credentials.json
|
|
137
|
-
```
|
|
117
|
+
> **Note on adding accounts:** OAuth login adds whatever account you're currently signed into at claude.ai — there's no account picker. To add a *different* account, sign into that account at claude.ai first (or use a logged-out / incognito browser window), then run `maxpool login`. (Importing the Claude Code CLI's own local login was removed — it shares a single-use credential the CLI keeps rotating, which broke the pooled copy. Use browser login so maxpool holds its own independent grant.)
|
|
138
118
|
|
|
139
119
|
### API Key
|
|
140
120
|
|
|
@@ -175,8 +155,7 @@ The Accounts menu (`a`) lets you add and manage accounts without leaving the TUI
|
|
|
175
155
|
|
|
176
156
|
| Key | Action |
|
|
177
157
|
|-----|--------|
|
|
178
|
-
| `
|
|
179
|
-
| `l` | Log in via browser — add *any* account, then name it |
|
|
158
|
+
| `l` | Log in via browser — add an account, then name it |
|
|
180
159
|
| `k` | Add an Anthropic API key account |
|
|
181
160
|
| `n` | Rename the selected account |
|
|
182
161
|
| `t` | Enable or disable the selected account |
|
package/package.json
CHANGED
package/src/account-config.js
CHANGED
|
@@ -1,24 +1,14 @@
|
|
|
1
|
-
import { importCredentials } from './oauth.js';
|
|
2
|
-
|
|
3
1
|
export async function resolveAccounts(config) {
|
|
4
2
|
const accounts = [];
|
|
5
3
|
for (const acct of config.accounts) {
|
|
6
4
|
if (acct.type === 'oauth') {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
console.log(`Imported "${acct.name}" from ${acct.importFrom}`);
|
|
12
|
-
} catch (err) {
|
|
13
|
-
console.error(`Failed to import "${acct.name}": ${err.message}`);
|
|
14
|
-
// Fall back to a previously-stored token rather than dropping the
|
|
15
|
-
// account entirely when the import source is unreadable.
|
|
16
|
-
if (acct.accessToken) accounts.push(acct);
|
|
17
|
-
}
|
|
18
|
-
} else if (acct.accessToken) {
|
|
5
|
+
// Legacy import-sourced accounts keep their stored token (the file/Keychain
|
|
6
|
+
// re-import was removed — it snapshotted a credential other clients rotate,
|
|
7
|
+
// which bricked accounts). Re-add via `maxpool login` for an independent grant.
|
|
8
|
+
if (acct.accessToken) {
|
|
19
9
|
accounts.push(acct);
|
|
20
10
|
} else {
|
|
21
|
-
console.error(`No token for "${acct.name}", skipping`);
|
|
11
|
+
console.error(`No token for "${acct.name}", skipping — re-add it with: maxpool login`);
|
|
22
12
|
}
|
|
23
13
|
} else if (acct.type === 'apikey' && acct.apiKey) {
|
|
24
14
|
accounts.push(acct);
|
package/src/account-manager.js
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
import { refreshAccessToken, isTokenExpiringSoon } from './oauth.js';
|
|
2
2
|
|
|
3
|
+
// Bounded re-poll hold for an account blocked ONLY by a transient, self-clearing
|
|
4
|
+
// condition whose exact recovery time is unknown: (a) a weekly-critical account
|
|
5
|
+
// (last-resort usable, no learned reset), or (b) an otherwise-healthy account at
|
|
6
|
+
// its in-flight / global concurrency cap (a sibling completing frees a slot in
|
|
7
|
+
// seconds). Both are recoverable by definition, so they must HOLD finite and let
|
|
8
|
+
// waitForAvailableRoute's poll loop re-check real availability — never collapse to
|
|
9
|
+
// an Infinity session-kill / error-fast.
|
|
10
|
+
const BOUNDED_REPOLL_HOLD_MS = 60_000;
|
|
11
|
+
|
|
3
12
|
function emptyQuota() {
|
|
4
13
|
return {
|
|
5
14
|
// Standard API rate limits (API key accounts)
|
|
@@ -204,6 +213,8 @@ export class AccountManager {
|
|
|
204
213
|
let soonestTemporary = Infinity;
|
|
205
214
|
let temporaryCause = null;
|
|
206
215
|
let soonestWeekly = Infinity;
|
|
216
|
+
let soonestBoundedHold = Infinity; // recoverable-transient accounts (weekly-critical last-resort, or concurrency-capped): always a bounded re-poll (known short-term resets route through soonestTemporary instead)
|
|
217
|
+
let boundedHoldCause = null;
|
|
207
218
|
let weeklyUnknownReset = 0; // weekly-exhausted accounts whose reset time we don't know yet
|
|
208
219
|
let matchingRoutes = 0;
|
|
209
220
|
const reasons = {};
|
|
@@ -234,7 +245,7 @@ export class AccountManager {
|
|
|
234
245
|
|
|
235
246
|
const retry = this._retryInfo(account);
|
|
236
247
|
note(retry.cause);
|
|
237
|
-
if (retry.
|
|
248
|
+
if (retry.weeklyCritical && this._isAvailable(account, { allowWeeklyReserve: true, allowWeeklyCritical: true })) {
|
|
238
249
|
return {
|
|
239
250
|
available: true,
|
|
240
251
|
retryAfterMs: 0,
|
|
@@ -244,11 +255,27 @@ export class AccountManager {
|
|
|
244
255
|
};
|
|
245
256
|
}
|
|
246
257
|
if (retry.queueable && retry.retryAt) {
|
|
258
|
+
// A known, soon short-term reset (5h cap / rate-limit / cooldown) — even on
|
|
259
|
+
// a weekly-critical account, this is the REAL near-term recovery time, so
|
|
260
|
+
// it holds here with the true cause rather than the distant weekly reset.
|
|
247
261
|
const ms = retry.retryAt - Date.now();
|
|
248
262
|
if (ms < soonestTemporary) {
|
|
249
263
|
soonestTemporary = ms;
|
|
250
264
|
temporaryCause = retry.cause;
|
|
251
265
|
}
|
|
266
|
+
} else if (retry.weeklyCritical || retry.transientCap) {
|
|
267
|
+
// A recoverable-transient block with no queueable short-term reset — a
|
|
268
|
+
// weekly-critical account (last-resort usable) or an otherwise-healthy
|
|
269
|
+
// account at its concurrency cap. _retryInfo always reaches here with
|
|
270
|
+
// retryAt:null (a KNOWN short-term reset is queueable and routes through
|
|
271
|
+
// soonestTemporary above), so the hold is a bounded re-poll. Recoverable by
|
|
272
|
+
// definition — hold finite, never collapse to Infinity and KILL the session.
|
|
273
|
+
soonestBoundedHold = Math.min(soonestBoundedHold, BOUNDED_REPOLL_HOLD_MS);
|
|
274
|
+
// Label precedence: an account that is BOTH weekly-critical and short-term
|
|
275
|
+
// capped is fundamentally weekly_critical; concurrency_cap only labels the
|
|
276
|
+
// hold when no weekly-critical account contributed it.
|
|
277
|
+
if (retry.weeklyCritical) boundedHoldCause = 'weekly_critical';
|
|
278
|
+
else if (!boundedHoldCause) boundedHoldCause = 'concurrency_cap';
|
|
252
279
|
} else if (retry.cause === 'weekly_exhausted' && retry.retryAt) {
|
|
253
280
|
const ms = retry.retryAt - Date.now();
|
|
254
281
|
if (ms < soonestWeekly) soonestWeekly = ms;
|
|
@@ -260,21 +287,24 @@ export class AccountManager {
|
|
|
260
287
|
}
|
|
261
288
|
}
|
|
262
289
|
|
|
263
|
-
|
|
290
|
+
// Min-merge ALL THREE recovery buckets and emit the cause of the SOONEST one.
|
|
291
|
+
// A weekly-critical account is last-resort usable and frees when its
|
|
292
|
+
// short-term blocker clears (often ~minutes); a weekly-exhausted account is
|
|
293
|
+
// unusable until its full 7d reset. Picking any one bucket ahead of the others
|
|
294
|
+
// (the old temporary-then-weekly-then-critical order) could mask a sibling's
|
|
295
|
+
// sooner recovery behind a far reset — error-fasting a holdable request and
|
|
296
|
+
// emitting a misleading multi-day Retry-After.
|
|
297
|
+
const recoveries = [
|
|
298
|
+
{ ms: soonestTemporary, cause: temporaryCause || 'temporary_unavailable' },
|
|
299
|
+
{ ms: soonestWeekly, cause: 'weekly_exhausted' },
|
|
300
|
+
{ ms: soonestBoundedHold, cause: boundedHoldCause || 'weekly_critical' },
|
|
301
|
+
].filter(r => Number.isFinite(r.ms));
|
|
302
|
+
if (recoveries.length) {
|
|
303
|
+
const best = recoveries.reduce((a, b) => (b.ms < a.ms ? b : a));
|
|
264
304
|
return {
|
|
265
305
|
available: false,
|
|
266
|
-
retryAfterMs: Math.max(0,
|
|
267
|
-
cause:
|
|
268
|
-
reasons,
|
|
269
|
-
matchingRoutes,
|
|
270
|
-
};
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
if (Number.isFinite(soonestWeekly)) {
|
|
274
|
-
return {
|
|
275
|
-
available: false,
|
|
276
|
-
retryAfterMs: Math.max(0, soonestWeekly),
|
|
277
|
-
cause: 'weekly_exhausted',
|
|
306
|
+
retryAfterMs: Math.max(0, best.ms),
|
|
307
|
+
cause: best.cause,
|
|
278
308
|
reasons,
|
|
279
309
|
matchingRoutes,
|
|
280
310
|
};
|
|
@@ -619,9 +649,42 @@ export class AccountManager {
|
|
|
619
649
|
// Register a waiter. Returns the ticket, or null if a backpressure limit
|
|
620
650
|
// (maxConcurrentQueued / maxQueuedBytes) would be exceeded — the caller then
|
|
621
651
|
// rejects the request with a "queue full" error instead of holding it.
|
|
652
|
+
// Evict any waiting ticket(s) for a session key, releasing their slot + bytes.
|
|
653
|
+
// A client timeout-retry opens a fresh request for the same session; this lets
|
|
654
|
+
// the retry SUPERSEDE its own ghost instead of leaving a dead ticket occupying
|
|
655
|
+
// a queue slot for up to the hold ceiling (the steady-state ghost-leak DoS).
|
|
656
|
+
_evictQueuedSession(sessionKey) {
|
|
657
|
+
if (!sessionKey) return;
|
|
658
|
+
const q = this.queueState;
|
|
659
|
+
for (let i = q.waiting.length - 1; i >= 0; i--) {
|
|
660
|
+
const t = q.waiting[i];
|
|
661
|
+
if (t.sessionKey !== sessionKey) continue;
|
|
662
|
+
// Only supersede a GHOST — a prior hold whose client connection is already
|
|
663
|
+
// gone (a timeout-retry of the SAME logical request). NEVER evict a LIVE
|
|
664
|
+
// concurrent sibling: a single Claude Code process fires concurrent
|
|
665
|
+
// requests under ONE session id (the main stream + the haiku title/summary
|
|
666
|
+
// call + parallel subagents), and evicting a live one orphans it for days.
|
|
667
|
+
// Catch a half-dead EPIPE ghost too: after a client RST the ServerResponse
|
|
668
|
+
// may not have flipped destroyed/writableEnded yet (it's noticed on the next
|
|
669
|
+
// write), but its underlying socket is already destroyed. A LIVE sibling has a
|
|
670
|
+
// live socket (socket.destroyed===false), so this never evicts one. (Mock-live
|
|
671
|
+
// res objects leave socket undefined → not dead.) Uses socket.destroyed only —
|
|
672
|
+
// a stable terminal signal — not the transient res.writable.
|
|
673
|
+
const dead = !t.res || t.res.destroyed || t.res.writableEnded
|
|
674
|
+
|| t.res.socket?.destroyed === true;
|
|
675
|
+
if (!dead) continue;
|
|
676
|
+
if (t.requestInfo) t.requestInfo.queueTicket = null; // let its waiter exit fast
|
|
677
|
+
t.dead = true;
|
|
678
|
+
q.bytes = Math.max(0, q.bytes - (t.bytes || 0));
|
|
679
|
+
q.waiting.splice(i, 1);
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
|
|
622
683
|
registerQueuedRequest(requestInfo = {}, opts = {}) {
|
|
623
684
|
if (requestInfo.queueTicket) return requestInfo.queueTicket;
|
|
624
685
|
this._reapStaleQueueHead();
|
|
686
|
+
const sessionKey = opts.sessionKey || requestInfo.sessionKey || null;
|
|
687
|
+
this._evictQueuedSession(sessionKey); // a retry supersedes its own DEAD prior hold
|
|
625
688
|
const bytes = Math.max(0, Number(opts.bytes) || 0);
|
|
626
689
|
const { maxConcurrentQueued, maxQueuedBytes } = opts;
|
|
627
690
|
if (maxConcurrentQueued != null && this.queueState.waiting.length >= maxConcurrentQueued) return null;
|
|
@@ -632,10 +695,18 @@ export class AccountManager {
|
|
|
632
695
|
queuedAt: Date.now(),
|
|
633
696
|
bytes,
|
|
634
697
|
deadlineAt: opts.deadlineAt || null,
|
|
698
|
+
sessionKey,
|
|
699
|
+
res: opts.res || null,
|
|
700
|
+
requestInfo,
|
|
635
701
|
};
|
|
636
702
|
this.queueState.waiting.push(ticket);
|
|
637
703
|
this.queueState.bytes += bytes;
|
|
638
704
|
requestInfo.queueTicket = ticket;
|
|
705
|
+
// Re-queuing CONSUMES any prior admission: a request that was admitted
|
|
706
|
+
// (ticket cleared, queueAdmitted=true) but then failed to acquire the freed
|
|
707
|
+
// slot (lost the race) must re-enter the FIFO as a fair waiter, NOT keep
|
|
708
|
+
// bypassing the fairness gate forever and starve everyone behind it.
|
|
709
|
+
requestInfo.queueAdmitted = false;
|
|
639
710
|
return ticket;
|
|
640
711
|
}
|
|
641
712
|
|
|
@@ -782,22 +853,80 @@ export class AccountManager {
|
|
|
782
853
|
}
|
|
783
854
|
|
|
784
855
|
_isNearQuota(account) {
|
|
856
|
+
// RAW weekly state (not pace): a raw-healthy account with real headroom is
|
|
857
|
+
// never treated as near-quota just because it's burning fast. Pace stays a
|
|
858
|
+
// soft cost in _scoreAccount only.
|
|
785
859
|
return this._isSessionQuotaUnavailable(account)
|
|
786
|
-
|| ['reserve', 'critical', 'exhausted'].includes(this.
|
|
860
|
+
|| ['reserve', 'critical', 'exhausted'].includes(this._weeklyRawState(account));
|
|
787
861
|
}
|
|
788
862
|
|
|
789
863
|
_retryInfo(account) {
|
|
790
864
|
const now = Date.now();
|
|
791
865
|
const q = account.quota || {};
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
866
|
+
|
|
867
|
+
// TERMINAL (non-recoverable) states FIRST — before any weekly/short-term
|
|
868
|
+
// bucket. An auth-dead / disabled / exhausted-status account is NOT
|
|
869
|
+
// recoverable-by-definition: it must error-fast (retryAt:null, no weeklyCritical
|
|
870
|
+
// tag → Infinity → 429), and a stale critical/exhausted QUOTA reading must never
|
|
871
|
+
// shadow that into a finite hold that spins the session for up to 7 days.
|
|
872
|
+
if (!account.enabled) return { cause: 'disabled', retryAt: null, queueable: false };
|
|
873
|
+
if (account.status === 'error') return { cause: 'error', retryAt: null, queueable: false };
|
|
874
|
+
if (account.status === 'exhausted') return { cause: 'exhausted', retryAt: null, queueable: false };
|
|
875
|
+
|
|
876
|
+
// RAW weekly state, so the retry oracle agrees with _isAvailable's raw gate.
|
|
877
|
+
// (Pace must NOT classify a raw-healthy account as weekly_critical here, or
|
|
878
|
+
// the queue keys on a far-future reset instead of the account's real
|
|
879
|
+
// short-term availability — the session-kill bug.)
|
|
880
|
+
const weeklyState = this._weeklyRawState(account);
|
|
881
|
+
|
|
882
|
+
// Short-term blockers (rate-limit / cooldown / upstream / 5h session cap /
|
|
883
|
+
// token-request-provider limits) clear on their OWN schedule — usually FAR
|
|
884
|
+
// sooner than a 7d weekly reset. Compute them up front so a weekly-critical
|
|
885
|
+
// account reports its REAL near-term recovery, not the distant weekly reset.
|
|
886
|
+
const shortTerm = this._shortTermRetry(account, now, q);
|
|
796
887
|
|
|
797
888
|
if (weeklyState === 'exhausted') {
|
|
889
|
+
// Hard block: only a weekly reset unblocks it — a sooner short-term clear
|
|
890
|
+
// does not help — so key the hold on the weekly reset.
|
|
798
891
|
return { cause: 'weekly_exhausted', retryAt: q.unified7dReset || null, queueable: false };
|
|
799
892
|
}
|
|
800
893
|
|
|
894
|
+
if (weeklyState === 'critical') {
|
|
895
|
+
// Last-resort USABLE: the account becomes selectable (as last resort) the
|
|
896
|
+
// moment its short-term blocker clears — NOT at the far weekly reset. So
|
|
897
|
+
// report the SOONER real blocker (the 5h cap / rate-limit), not unified7dReset.
|
|
898
|
+
// Tag weeklyCritical so the oracle ALWAYS holds (finite) on it: a critical
|
|
899
|
+
// account is recoverable by definition and must never collapse to an
|
|
900
|
+
// Infinity session-kill, even when no reset time is known.
|
|
901
|
+
if (shortTerm) return { ...shortTerm, weeklyCritical: true };
|
|
902
|
+
// No short-term blocker → the only thing keeping it out of the last-resort
|
|
903
|
+
// pool is a TRANSIENT cap (in-flight/concurrency, admission pause), which
|
|
904
|
+
// clears in seconds when a sibling completes — NOT the 7d weekly reset. Hold
|
|
905
|
+
// a bounded re-poll (retryAt:null → BOUNDED_REPOLL_HOLD_MS), never the far
|
|
906
|
+
// weekly reset, so a non-stream request isn't error-fasted for ~7d.
|
|
907
|
+
return { cause: 'weekly_critical', retryAt: null, queueable: false, weeklyCritical: true };
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
// Healthy / soft / reserve weekly: the ordinary short-term blocker, if any.
|
|
911
|
+
if (shortTerm) return shortTerm;
|
|
912
|
+
|
|
913
|
+
// Otherwise-healthy but at the in-flight / global concurrency cap — a TRANSIENT,
|
|
914
|
+
// self-clearing block (a sibling completing frees a slot in seconds). HOLD a
|
|
915
|
+
// bounded re-poll rather than error-fasting (Infinity): the symmetric case to a
|
|
916
|
+
// concurrency-capped weekly-critical account, which already holds finite above.
|
|
917
|
+
if (account.inFlight >= this.scheduler.safetyMaxActivePerAccount
|
|
918
|
+
|| this.getGlobalInFlight() >= this.scheduler.safetyMaxGlobalActive) {
|
|
919
|
+
return { cause: 'concurrency_cap', retryAt: null, queueable: false, transientCap: true };
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
return { cause: 'unavailable', retryAt: null, queueable: false };
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
// The soonest active short-term (non-weekly) blocker for an account, or null if
|
|
926
|
+
// none is active. Ordered most-specific-first; each entry is a {cause, retryAt,
|
|
927
|
+
// queueable} the retry oracle can hold on. Kept separate from the weekly state
|
|
928
|
+
// so weekly-critical accounts surface their real near-term recovery time.
|
|
929
|
+
_shortTermRetry(account, now, q) {
|
|
801
930
|
if (account.status === 'throttled' && account.rateLimitedUntil && now < account.rateLimitedUntil) {
|
|
802
931
|
return { cause: 'rate_limited', retryAt: account.rateLimitedUntil, queueable: true };
|
|
803
932
|
}
|
|
@@ -834,10 +963,7 @@ export class AccountManager {
|
|
|
834
963
|
return { cause: 'provider_limit', retryAt: q.genericReset || null, queueable: Boolean(q.genericReset) };
|
|
835
964
|
}
|
|
836
965
|
|
|
837
|
-
|
|
838
|
-
if (account.status === 'error') return { cause: 'error', retryAt: null, queueable: false };
|
|
839
|
-
if (account.status === 'exhausted') return { cause: 'exhausted', retryAt: null, queueable: false };
|
|
840
|
-
return { cause: 'unavailable', retryAt: null, queueable: false };
|
|
966
|
+
return null;
|
|
841
967
|
}
|
|
842
968
|
|
|
843
969
|
_selectNext(requestInfo = {}, excludedIndexes = new Set()) {
|
|
@@ -1374,7 +1500,7 @@ export class AccountManager {
|
|
|
1374
1500
|
: account.quota.tokensLimit
|
|
1375
1501
|
? ((1 - account.quota.tokensRemaining / account.quota.tokensLimit) * 100).toFixed(1)
|
|
1376
1502
|
: '?';
|
|
1377
|
-
const reason = this._isSessionQuotaUnavailable(account) ? 'session quota' : `weekly ${this.
|
|
1503
|
+
const reason = this._isSessionQuotaUnavailable(account) ? 'session quota' : `weekly ${this._weeklyRawState(account)}`;
|
|
1378
1504
|
const logKey = `${reason}:${pct}`;
|
|
1379
1505
|
if (account.lastQuotaLogKey !== logKey) {
|
|
1380
1506
|
account.lastQuotaLogKey = logKey;
|
|
@@ -1488,7 +1614,7 @@ export class AccountManager {
|
|
|
1488
1614
|
if (incident.accounts.has(account.index)) continue;
|
|
1489
1615
|
if (account.status === 'exhausted' || account.status === 'error') continue;
|
|
1490
1616
|
if (this._isSessionQuotaUnavailable(account)) continue;
|
|
1491
|
-
if (this.
|
|
1617
|
+
if (this._weeklyRawState(account) === 'exhausted') continue;
|
|
1492
1618
|
return false;
|
|
1493
1619
|
}
|
|
1494
1620
|
return true;
|
package/src/config.js
CHANGED
|
@@ -87,10 +87,16 @@ export function createDefaultConfig() {
|
|
|
87
87
|
// account's REAL reset time, so a generous bound never spins pointlessly).
|
|
88
88
|
queue: {
|
|
89
89
|
enabled: true,
|
|
90
|
-
maxWaitMs: 24 * 60 * 60 * 1000, // hard ceiling for
|
|
90
|
+
maxWaitMs: 24 * 60 * 60 * 1000, // hard ceiling for non-streaming/capacity holds; streaming uses streamHoldMaxMs
|
|
91
91
|
autoMaxWaitMs: null, // 5h/session-cap hold (null = maxWaitMs)
|
|
92
92
|
capacityMaxWaitMs: 15 * 60 * 1000, // upstream 529/overload — stays short, never governed by the others
|
|
93
|
-
weeklyMaxWaitMs: 24 * 60 * 60 * 1000, //
|
|
93
|
+
weeklyMaxWaitMs: 24 * 60 * 60 * 1000, // legacy bound; streaming holds use streamHoldMaxMs
|
|
94
|
+
// Streaming hold ceiling: how long a streaming session is held ALIVE on the
|
|
95
|
+
// heartbeat waiting for any account to free up. 7d so a session is never
|
|
96
|
+
// killed while a real reset is on the way; only permanent failures (all
|
|
97
|
+
// accounts logged out / no eligible route) error fast. Lower if your client
|
|
98
|
+
// uses a wall-clock total-request timeout the heartbeat can't reset.
|
|
99
|
+
streamHoldMaxMs: 7 * 24 * 60 * 60 * 1000,
|
|
94
100
|
nonStreamMaxWaitMs: 5 * 60 * 1000, // non-streaming requests have no keepalive; cap their wait
|
|
95
101
|
maxConcurrentQueued: 64, // backpressure: max requests held at once
|
|
96
102
|
maxQueuedBytes: 1024 * 1024 * 1024, // backpressure: max aggregate buffered body bytes (1 GiB)
|
package/src/index.js
CHANGED
|
@@ -6,7 +6,7 @@ import { loadOrCreateConfig, loadConfig, saveConfig, atomicConfigUpdate, getConf
|
|
|
6
6
|
import { AccountManager } from './account-manager.js';
|
|
7
7
|
import { createProxyServer } from './server.js';
|
|
8
8
|
import { Prober } from './prober.js';
|
|
9
|
-
import {
|
|
9
|
+
import { loginOAuth, fetchProfile, refreshAccessToken, isTokenExpiringSoon } from './oauth.js';
|
|
10
10
|
import { TUI } from './tui.js';
|
|
11
11
|
import { RestartController } from './restart-controller.js';
|
|
12
12
|
import { resolveAccounts } from './account-config.js';
|
|
@@ -24,10 +24,6 @@ switch (command) {
|
|
|
24
24
|
case 'run':
|
|
25
25
|
await runCommand();
|
|
26
26
|
break;
|
|
27
|
-
case 'import':
|
|
28
|
-
await importCommand();
|
|
29
|
-
process.exit(0);
|
|
30
|
-
break;
|
|
31
27
|
case 'login':
|
|
32
28
|
await loginCommand();
|
|
33
29
|
process.exit(0);
|
|
@@ -124,7 +120,6 @@ async function serverWorkerCommand() {
|
|
|
124
120
|
if (config.accounts.length === 0) {
|
|
125
121
|
console.error('No accounts configured.\n');
|
|
126
122
|
console.error('Add an account first:');
|
|
127
|
-
console.error(' maxpool import Import from Claude Code');
|
|
128
123
|
console.error(' maxpool login OAuth login via browser');
|
|
129
124
|
console.error(' maxpool login --api Add an API key');
|
|
130
125
|
process.exit(1);
|
|
@@ -399,47 +394,6 @@ function logPlainServerStart({ host, port, accounts, threshold, config }) {
|
|
|
399
394
|
console.log('');
|
|
400
395
|
}
|
|
401
396
|
|
|
402
|
-
// ── import ──────────────────────────────────────────────────
|
|
403
|
-
|
|
404
|
-
async function importCommand() {
|
|
405
|
-
const config = await loadOrCreateConfig();
|
|
406
|
-
|
|
407
|
-
let name = argValue('--name');
|
|
408
|
-
const jsonStr = argValue('--json');
|
|
409
|
-
|
|
410
|
-
let creds;
|
|
411
|
-
if (jsonStr) {
|
|
412
|
-
// Accept raw JSON: --json '{"claudeAiOauth":{"accessToken":"...","refreshToken":"...","expiresAt":...}}'
|
|
413
|
-
// or flat: --json '{"accessToken":"...","refreshToken":"...","expiresAt":...}'
|
|
414
|
-
try {
|
|
415
|
-
const raw = JSON.parse(jsonStr);
|
|
416
|
-
const data = raw.claudeAiOauth || raw;
|
|
417
|
-
if (!data.accessToken) {
|
|
418
|
-
console.error('JSON must contain "accessToken" (directly or under "claudeAiOauth")');
|
|
419
|
-
process.exit(1);
|
|
420
|
-
}
|
|
421
|
-
creds = {
|
|
422
|
-
accessToken: data.accessToken,
|
|
423
|
-
refreshToken: data.refreshToken,
|
|
424
|
-
expiresAt: data.expiresAt,
|
|
425
|
-
};
|
|
426
|
-
} catch (err) {
|
|
427
|
-
console.error(`Failed to parse --json: ${err.message}`);
|
|
428
|
-
process.exit(1);
|
|
429
|
-
}
|
|
430
|
-
} else {
|
|
431
|
-
const fromPath = argValue('--from') || '~/.claude/.credentials.json';
|
|
432
|
-
try {
|
|
433
|
-
creds = await importCredentials(fromPath);
|
|
434
|
-
} catch (err) {
|
|
435
|
-
console.error(`Failed to import from ${fromPath}: ${err.message}`);
|
|
436
|
-
process.exit(1);
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
await upsertOAuthAccount(config, name, creds, 'import');
|
|
441
|
-
}
|
|
442
|
-
|
|
443
397
|
// ── login ───────────────────────────────────────────────────
|
|
444
398
|
|
|
445
399
|
async function loginCommand() {
|
|
@@ -505,6 +459,9 @@ async function loginOAuthCommand() {
|
|
|
505
459
|
let name = argValue('--name');
|
|
506
460
|
|
|
507
461
|
console.log('Starting OAuth login...');
|
|
462
|
+
console.log('Note: this adds whatever account you are currently signed into at claude.ai —');
|
|
463
|
+
console.log('there is no account picker. To add a DIFFERENT account, sign into THAT account at');
|
|
464
|
+
console.log('claude.ai first (or use a logged-out / incognito browser window), then continue.');
|
|
508
465
|
let creds;
|
|
509
466
|
try {
|
|
510
467
|
creds = await loginOAuth();
|
|
@@ -512,7 +469,6 @@ async function loginOAuthCommand() {
|
|
|
512
469
|
console.error(`OAuth login failed: ${err.message}`);
|
|
513
470
|
console.error('');
|
|
514
471
|
console.error('Alternatives:');
|
|
515
|
-
console.error(' maxpool import Import from existing Claude Code credentials');
|
|
516
472
|
console.error(' maxpool login --api Add an API key instead');
|
|
517
473
|
process.exit(1);
|
|
518
474
|
}
|
|
@@ -636,7 +592,7 @@ async function accountsCommand() {
|
|
|
636
592
|
|
|
637
593
|
if (config.accounts.length === 0) {
|
|
638
594
|
console.log('No accounts configured.');
|
|
639
|
-
console.log('Add one with: maxpool
|
|
595
|
+
console.log('Add one with: maxpool login (browser) or maxpool login --api');
|
|
640
596
|
return;
|
|
641
597
|
}
|
|
642
598
|
|
|
@@ -847,8 +803,7 @@ Usage: maxpool [command] [options]
|
|
|
847
803
|
|
|
848
804
|
Commands:
|
|
849
805
|
server Start the proxy server (default)
|
|
850
|
-
|
|
851
|
-
login OAuth login via browser
|
|
806
|
+
login OAuth login via browser (adds the account you're signed into at claude.ai)
|
|
852
807
|
login --api Add an API key account
|
|
853
808
|
env [--with-key] Print env vars to use with Claude
|
|
854
809
|
run [-- args...] Run Claude Code through the proxy
|
|
@@ -860,10 +815,7 @@ Commands:
|
|
|
860
815
|
help Show this help
|
|
861
816
|
|
|
862
817
|
Options:
|
|
863
|
-
--name NAME Set account name (
|
|
864
|
-
--from PATH Credentials path (import, default: ~/.claude/.credentials.json)
|
|
865
|
-
--json JSON Import from inline JSON (import), e.g.:
|
|
866
|
-
--json '{"accessToken":"...","refreshToken":"...","expiresAt":1234}'
|
|
818
|
+
--name NAME Set account name (login)
|
|
867
819
|
--log-to DIR Log full requests/responses to DIR (server, one file per request)
|
|
868
820
|
--with-key Include proxy API key in maxpool env output
|
|
869
821
|
|
|
@@ -957,14 +909,7 @@ async function syncAccountsFromDisk(diskConfig, memConfig, accountManager) {
|
|
|
957
909
|
|
|
958
910
|
// Existing account — resolve fresh credentials from disk
|
|
959
911
|
let freshCred = null;
|
|
960
|
-
if (diskAcct.type === 'oauth' && diskAcct.
|
|
961
|
-
try {
|
|
962
|
-
const creds = await importCredentials(diskAcct.importFrom);
|
|
963
|
-
freshCred = { accessToken: creds.accessToken, refreshToken: creds.refreshToken, expiresAt: creds.expiresAt };
|
|
964
|
-
} catch (err) {
|
|
965
|
-
console.error(`[Maxpool] Re-import failed for "${diskAcct.name}": ${err.message}`);
|
|
966
|
-
}
|
|
967
|
-
} else if (diskAcct.type === 'oauth' && diskAcct.accessToken) {
|
|
912
|
+
if (diskAcct.type === 'oauth' && diskAcct.accessToken) {
|
|
968
913
|
freshCred = { accessToken: diskAcct.accessToken, refreshToken: diskAcct.refreshToken, expiresAt: diskAcct.expiresAt };
|
|
969
914
|
} else if (diskAcct.type === 'apikey' && diskAcct.apiKey) {
|
|
970
915
|
freshCred = { apiKey: diskAcct.apiKey };
|
package/src/oauth.js
CHANGED
|
@@ -1,74 +1,8 @@
|
|
|
1
|
-
import { readFile } from 'node:fs/promises';
|
|
2
|
-
import { homedir, userInfo } from 'node:os';
|
|
3
1
|
import { randomBytes, createHash } from 'node:crypto';
|
|
4
|
-
import { exec
|
|
5
|
-
import { promisify } from 'node:util';
|
|
2
|
+
import { exec } from 'node:child_process';
|
|
6
3
|
import { createInterface } from 'node:readline';
|
|
7
4
|
import http from 'node:http';
|
|
8
5
|
|
|
9
|
-
const execFileAsync = promisify(execFile);
|
|
10
|
-
|
|
11
|
-
const KEYCHAIN_SERVICE = 'Claude Code-credentials';
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* Read Claude Code credentials from the macOS Keychain.
|
|
15
|
-
* Claude Code (recent versions, macOS) stores OAuth creds in the login
|
|
16
|
-
* Keychain under service "Claude Code-credentials", account = the OS
|
|
17
|
-
* username — NOT in ~/.claude/.credentials.json. Returns the parsed
|
|
18
|
-
* credential object (unwrapped from "claudeAiOauth"), or null if absent.
|
|
19
|
-
*/
|
|
20
|
-
async function readMacKeychainCredentials() {
|
|
21
|
-
if (process.platform !== 'darwin') return null;
|
|
22
|
-
const account = userInfo().username;
|
|
23
|
-
try {
|
|
24
|
-
const { stdout } = await execFileAsync('security', [
|
|
25
|
-
'find-generic-password', '-s', KEYCHAIN_SERVICE, '-a', account, '-w',
|
|
26
|
-
]);
|
|
27
|
-
const raw = JSON.parse(stdout.trim());
|
|
28
|
-
return raw.claudeAiOauth || raw;
|
|
29
|
-
} catch {
|
|
30
|
-
return null;
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* Import OAuth credentials from a Claude Code credentials file, falling back
|
|
36
|
-
* to the macOS Keychain when the file is absent (the default on macOS).
|
|
37
|
-
*/
|
|
38
|
-
export async function importCredentials(filePath = '~/.claude/.credentials.json') {
|
|
39
|
-
const resolvedPath = filePath.replace(/^~/, homedir());
|
|
40
|
-
|
|
41
|
-
let data;
|
|
42
|
-
try {
|
|
43
|
-
const raw = JSON.parse(await readFile(resolvedPath, 'utf-8'));
|
|
44
|
-
// Claude Code stores credentials nested under "claudeAiOauth"
|
|
45
|
-
data = raw.claudeAiOauth || raw;
|
|
46
|
-
} catch (fileErr) {
|
|
47
|
-
// No file → try the macOS Keychain (where Claude Code now stores creds).
|
|
48
|
-
data = await readMacKeychainCredentials();
|
|
49
|
-
if (!data) {
|
|
50
|
-
throw new Error(
|
|
51
|
-
process.platform === 'darwin'
|
|
52
|
-
? `No credentials at ${resolvedPath} and none in the macOS Keychain ` +
|
|
53
|
-
`("${KEYCHAIN_SERVICE}"). Is Claude Code logged in on this machine? ` +
|
|
54
|
-
`Run 'claude' once to log in, or paste a token with 'maxpool import --json ...'.`
|
|
55
|
-
: `Could not read credentials from ${resolvedPath}: ${fileErr.message}`,
|
|
56
|
-
);
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
if (!data.accessToken) {
|
|
61
|
-
throw new Error('Imported credentials have no accessToken');
|
|
62
|
-
}
|
|
63
|
-
return {
|
|
64
|
-
accessToken: data.accessToken,
|
|
65
|
-
refreshToken: data.refreshToken,
|
|
66
|
-
expiresAt: data.expiresAt,
|
|
67
|
-
subscriptionType: data.subscriptionType,
|
|
68
|
-
rateLimitTier: data.rateLimitTier,
|
|
69
|
-
};
|
|
70
|
-
}
|
|
71
|
-
|
|
72
6
|
const PROFILE_URL = 'https://api.anthropic.com/api/oauth/profile';
|
|
73
7
|
const DEFAULT_TOKEN_ENDPOINT = 'https://platform.claude.com/v1/oauth/token';
|
|
74
8
|
const DEFAULT_CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e';
|
package/src/server.js
CHANGED
|
@@ -19,12 +19,17 @@ const DEFAULT_QUEUE = {
|
|
|
19
19
|
maxWaitMs: 24 * 60 * 60 * 1000,
|
|
20
20
|
autoMaxWaitMs: null,
|
|
21
21
|
capacityMaxWaitMs: 15 * 60 * 1000,
|
|
22
|
-
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
|
|
22
|
+
weeklyMaxWaitMs: 24 * 60 * 60 * 1000, // legacy bound; streaming holds use streamHoldMaxMs
|
|
23
|
+
// STREAMING hold ceiling: how long a streaming request may be HELD ALIVE on
|
|
24
|
+
// the SSE heartbeat waiting for any account to free up. Defaults to 7d (the
|
|
25
|
+
// max weekly window) so a session is never killed while a real reset is on the
|
|
26
|
+
// way — it resumes the instant any account frees. The hold is gated by the
|
|
27
|
+
// nextRetryForRequest oracle: it ONLY holds when ≥1 eligible route has a finite
|
|
28
|
+
// reset within this ceiling; permanent failures (all accounts logged out / no
|
|
29
|
+
// eligible route / reset unknown) error fast instead of hanging. The heartbeat
|
|
30
|
+
// resets idle-gap client timeouts; if a client uses a wall-clock total-request
|
|
31
|
+
// deadline, lower this to just under it.
|
|
32
|
+
streamHoldMaxMs: 7 * 24 * 60 * 60 * 1000,
|
|
28
33
|
// Non-streaming requests have no SSE heartbeat to keep them alive, so a long
|
|
29
34
|
// hold would die on the client timeout anyway. Cap their wait conservatively.
|
|
30
35
|
nonStreamMaxWaitMs: 5 * 60 * 1000,
|
|
@@ -282,13 +287,44 @@ async function forwardRequest(
|
|
|
282
287
|
return;
|
|
283
288
|
}
|
|
284
289
|
|
|
290
|
+
// The admission (if this was a resumed queued request) has now been CONSUMED —
|
|
291
|
+
// it got its account. Clear queueAdmitted so any subsequent internal failover
|
|
292
|
+
// recursion (excludedIndexes path) re-enters the fairness gate as a normal
|
|
293
|
+
// waiter instead of preferentially jumping ahead of the FIFO for the rest of
|
|
294
|
+
// this request's failover chain.
|
|
295
|
+
requestInfo.queueAdmitted = false;
|
|
296
|
+
|
|
297
|
+
// Abort the upstream fetch and release the lease if the CLIENT disconnects during
|
|
298
|
+
// the pre-response window (token refresh + connect + waiting for the upstream's
|
|
299
|
+
// first byte). Without this, a client that drops mid-flight leaves account.inFlight
|
|
300
|
+
// pinned until the fetch resolves on its own (~undici body timeout), benching
|
|
301
|
+
// scarce capacity — acute for the hold feature, which targets already-scarce
|
|
302
|
+
// accounts. The listener is removed once the response arrives; mid-stream
|
|
303
|
+
// disconnects are handled by streamResponse's res.destroyed checks.
|
|
304
|
+
const clientGone = new AbortController();
|
|
305
|
+
const onClientClose = () => clientGone.abort();
|
|
306
|
+
res.once('close', onClientClose);
|
|
307
|
+
const releaseOnClientGone = () => {
|
|
308
|
+
res.off('close', onClientClose);
|
|
309
|
+
accountManager.releaseAccount(lease);
|
|
310
|
+
clearQueueHeartbeat(requestInfo);
|
|
311
|
+
accountManager.removeQueuedRequest?.(requestInfo);
|
|
312
|
+
};
|
|
313
|
+
|
|
285
314
|
// Track which account handles this request
|
|
286
315
|
ctx.account = account.name;
|
|
287
316
|
hooks.onRequestRouted?.(reqId, { account: account.name });
|
|
288
317
|
|
|
289
318
|
// Refresh OAuth token if needed
|
|
290
319
|
const tokenReady = await accountManager.ensureTokenFresh(account.index);
|
|
320
|
+
if (clientGone.signal.aborted) { releaseOnClientGone(); return; }
|
|
291
321
|
if (!tokenReady) {
|
|
322
|
+
// Token refresh failed (not a client disconnect). This frame is leaving via
|
|
323
|
+
// recursion / queue / error WITHOUT reaching the post-fetch off() — drop the
|
|
324
|
+
// 'close' listener now so it doesn't accumulate one-per-failover-hop on `res`
|
|
325
|
+
// (MaxListenersExceededWarning + leak); the recursive/resumed frame registers
|
|
326
|
+
// its own.
|
|
327
|
+
res.off('close', onClientClose);
|
|
292
328
|
accountManager.releaseAccount(lease);
|
|
293
329
|
excludedIndexes.add(account.index);
|
|
294
330
|
if (
|
|
@@ -376,7 +412,11 @@ async function forwardRequest(
|
|
|
376
412
|
headers,
|
|
377
413
|
body: ['GET', 'HEAD'].includes(method) ? undefined : upstreamBody,
|
|
378
414
|
redirect: 'manual',
|
|
415
|
+
signal: clientGone.signal,
|
|
379
416
|
});
|
|
417
|
+
// Response arrived — the pre-response leak window is over. Stop guarding for
|
|
418
|
+
// client-disconnect via abort (streamResponse handles mid-stream disconnects).
|
|
419
|
+
res.off('close', onClientClose);
|
|
380
420
|
|
|
381
421
|
// Extract rate limit headers
|
|
382
422
|
const rateLimitHeaders = {};
|
|
@@ -749,7 +789,20 @@ async function forwardRequest(
|
|
|
749
789
|
if (requestInfo.queueHeartbeatActive) {
|
|
750
790
|
clearQueueHeartbeat(requestInfo);
|
|
751
791
|
if (!res.destroyed && !res.writableEnded) {
|
|
752
|
-
|
|
792
|
+
// We already committed `200 text/event-stream` (the queue heartbeat), but
|
|
793
|
+
// the resumed upstream returned a NON-streaming body. Writing the raw JSON
|
|
794
|
+
// as a lone `data:` line corrupts the client's SSE parser (no message_start
|
|
795
|
+
// envelope, no message_stop). Frame it as a proper SSE error event so the
|
|
796
|
+
// client fails cleanly instead of hanging/mis-parsing. (Rare: an upstream
|
|
797
|
+
// honoring stream:true never lands here; reachable on a fallback upstream
|
|
798
|
+
// quirk.)
|
|
799
|
+
res.write(`event: error\ndata: ${JSON.stringify({
|
|
800
|
+
type: 'error',
|
|
801
|
+
error: {
|
|
802
|
+
type: 'api_error',
|
|
803
|
+
message: `Upstream returned a non-streaming ${upstreamRes.status} response for a streaming request`,
|
|
804
|
+
},
|
|
805
|
+
})}\n\n`);
|
|
753
806
|
res.end();
|
|
754
807
|
}
|
|
755
808
|
} else {
|
|
@@ -758,6 +811,14 @@ async function forwardRequest(
|
|
|
758
811
|
}
|
|
759
812
|
}
|
|
760
813
|
} catch (err) {
|
|
814
|
+
res.off('close', onClientClose);
|
|
815
|
+
// Client disconnected mid-flight → we aborted the upstream fetch. Release the
|
|
816
|
+
// lease (free the scarce account) and STOP: no retry (the client is gone), no
|
|
817
|
+
// write (the socket is dead).
|
|
818
|
+
if (clientGone.signal.aborted) {
|
|
819
|
+
releaseOnClientGone();
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
761
822
|
console.error(`[Maxpool] Upstream error (account "${account.name}"):`, err.message);
|
|
762
823
|
|
|
763
824
|
if (logDir) {
|
|
@@ -869,7 +930,7 @@ function unavailableMessage(accountManager, requestInfo = {}, retryAfter, willRe
|
|
|
869
930
|
return `All ${n} accounts exhausted. Retry in ${retryAfter}s.`;
|
|
870
931
|
}
|
|
871
932
|
|
|
872
|
-
export const __serverTest = { unavailableMessage, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile };
|
|
933
|
+
export const __serverTest = { unavailableMessage, isRetriableUpstreamStatus, headerValue, getMaxpoolProfile, ensureQueueHeartbeat, clearQueueHeartbeat };
|
|
873
934
|
|
|
874
935
|
async function readErrorBody(upstreamRes, limitBytes = 64 * 1024) {
|
|
875
936
|
if (!upstreamRes.body) return '';
|
|
@@ -1084,7 +1145,6 @@ async function queueAndRetry(
|
|
|
1084
1145
|
const capacityMaxWaitMs = queueConfig.capacityMaxWaitMs == null
|
|
1085
1146
|
? autoMaxWaitMs
|
|
1086
1147
|
: Math.max(0, Number(queueConfig.capacityMaxWaitMs) || 0);
|
|
1087
|
-
const weeklyMaxWaitMs = Math.max(0, Number(queueConfig.weeklyMaxWaitMs) || 0);
|
|
1088
1148
|
const nonStreamMaxWaitMs = queueConfig.nonStreamMaxWaitMs == null
|
|
1089
1149
|
? 5 * 60_000
|
|
1090
1150
|
: Math.max(0, Number(queueConfig.nonStreamMaxWaitMs) || 0);
|
|
@@ -1106,20 +1166,46 @@ async function queueAndRetry(
|
|
|
1106
1166
|
return finishQueuedStreamIfNeeded(res, requestInfo, honestMessage);
|
|
1107
1167
|
}
|
|
1108
1168
|
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1169
|
+
const streamHoldMaxMs = queueConfig.streamHoldMaxMs == null
|
|
1170
|
+
? 7 * 24 * 60 * 60 * 1000
|
|
1171
|
+
: Math.max(0, Number(queueConfig.streamHoldMaxMs) || 0);
|
|
1172
|
+
// Pick the hold ceiling:
|
|
1173
|
+
// capacity (upstream 529/overload) → its own short cap, never a long hold
|
|
1174
|
+
// non-streaming (no heartbeat) → short cap (would die on client timeout)
|
|
1175
|
+
// streaming → up to streamHoldMaxMs (7d), kept alive
|
|
1176
|
+
// by the heartbeat
|
|
1177
|
+
let queueWindowMs;
|
|
1178
|
+
if (cause === 'capacity') {
|
|
1179
|
+
queueWindowMs = Math.min(maxWaitMs, capacityMaxWaitMs);
|
|
1180
|
+
} else if (!requestInfo.stream) {
|
|
1181
|
+
queueWindowMs = nonStreamMaxWaitMs;
|
|
1182
|
+
} else {
|
|
1183
|
+
queueWindowMs = streamHoldMaxMs;
|
|
1184
|
+
}
|
|
1185
|
+
// A non-streaming request has no heartbeat regardless of cause, so it must
|
|
1186
|
+
// never outlast nonStreamMaxWaitMs even under capacity (it would occupy a
|
|
1187
|
+
// slot 3x its documented cap with nothing to reap it).
|
|
1119
1188
|
if (!requestInfo.stream) queueWindowMs = Math.min(queueWindowMs, nonStreamMaxWaitMs);
|
|
1120
1189
|
|
|
1190
|
+
// A pure concurrency-cap block (every account healthy but all in-flight/global
|
|
1191
|
+
// slots busy — NOT a quota/rate-limit reset) is a LOCAL capacity transient. Bound
|
|
1192
|
+
// it by the short capacity window, never the multi-day streaming hold: a slot
|
|
1193
|
+
// frees as active requests finish (seconds–minutes), and if the fleet stays
|
|
1194
|
+
// saturated past the window the request sheds load (error-fast) instead of
|
|
1195
|
+
// spinning a queue slot for up to streamHoldMaxMs (7d) — the soft-deadlock guard.
|
|
1196
|
+
if (retryPlan.cause === 'concurrency_cap') {
|
|
1197
|
+
queueWindowMs = Math.min(queueWindowMs, capacityMaxWaitMs);
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1121
1200
|
if (queueWindowMs <= 0) return finishQueuedStreamIfNeeded(res, requestInfo, honestMessage);
|
|
1122
1201
|
|
|
1202
|
+
// HOLD-vs-ERROR oracle (from nextRetryForRequest): HOLD only when a TEMPORARY
|
|
1203
|
+
// cause has a finite real reset within the ceiling. ERROR FAST for permanent /
|
|
1204
|
+
// unsatisfiable cases — nextRetryForRequest returns retryAfterMs === Infinity
|
|
1205
|
+
// for no_eligible_route, weekly_reset_unknown, and "all matching routes are
|
|
1206
|
+
// terminal (disabled / error / auth-dead)". This is what stops an indefinite
|
|
1207
|
+
// hold from silently hanging every session when something is actually broken
|
|
1208
|
+
// (all accounts logged out, the only healthy account removed, etc.).
|
|
1123
1209
|
const retryAfterMs = retryPlan.retryAfterMs;
|
|
1124
1210
|
if (!Number.isFinite(retryAfterMs) || retryAfterMs > queueWindowMs) {
|
|
1125
1211
|
return finishQueuedStreamIfNeeded(res, requestInfo, honestMessage);
|
|
@@ -1129,6 +1215,8 @@ async function queueAndRetry(
|
|
|
1129
1215
|
const ticket = accountManager.registerQueuedRequest?.(requestInfo, {
|
|
1130
1216
|
bytes: body?.length || 0,
|
|
1131
1217
|
deadlineAt: requestInfo.queueStartedAt + queueWindowMs,
|
|
1218
|
+
sessionKey: requestInfo.sessionKey,
|
|
1219
|
+
res, // liveness check for ghost-only eviction
|
|
1132
1220
|
maxConcurrentQueued: queueConfig.maxConcurrentQueued,
|
|
1133
1221
|
maxQueuedBytes: queueConfig.maxQueuedBytes,
|
|
1134
1222
|
});
|
|
@@ -1148,7 +1236,7 @@ async function queueAndRetry(
|
|
|
1148
1236
|
ctx.account = '(queued)';
|
|
1149
1237
|
hooks.onRequestRouted?.(reqId, { account: '(queued)' });
|
|
1150
1238
|
console.log(`[Maxpool] ${reason}; queueing request for up to ${Math.ceil(remaining / 1000)}s (cause: ${cause}, retry: ${retryPlan.cause})`);
|
|
1151
|
-
ensureQueueHeartbeat(res, requestInfo, queueConfig);
|
|
1239
|
+
ensureQueueHeartbeat(res, requestInfo, queueConfig, accountManager);
|
|
1152
1240
|
|
|
1153
1241
|
const available = await waitForAvailableRoute(req, res, accountManager, requestInfo, queueConfig, remaining);
|
|
1154
1242
|
if (!available) {
|
|
@@ -1157,13 +1245,23 @@ async function queueAndRetry(
|
|
|
1157
1245
|
return finishQueuedStreamIfNeeded(res, requestInfo, honestMessage);
|
|
1158
1246
|
}
|
|
1159
1247
|
|
|
1248
|
+
// NOTE: the heartbeat is deliberately NOT cleared here. It must stay alive
|
|
1249
|
+
// through the resumed forward's CONNECTION + failover attempts: if the freed
|
|
1250
|
+
// account 529s/throttles on the first resumed request (before any upstream
|
|
1251
|
+
// bytes), forwardRequest re-enters queueAndRetry — whose guard at the top
|
|
1252
|
+
// (`res.headersSent && !queueHeartbeatActive`) would otherwise BAIL on the
|
|
1253
|
+
// committed stream and DROP the held session. Keeping the heartbeat active lets
|
|
1254
|
+
// it re-hold. The heartbeat is instead stopped the instant real upstream bytes
|
|
1255
|
+
// start flowing, inside streamResponse — that prevents the Bug A interleave
|
|
1256
|
+
// (': maxpool queued' comments injected between real SSE events) without losing
|
|
1257
|
+
// re-holdability on a post-resume failover.
|
|
1160
1258
|
return forwardRequest(
|
|
1161
1259
|
req, res, body, accountManager, upstream, 0, hooks, reqId, ctx, logDir,
|
|
1162
1260
|
retryConfig, queueConfig, requestInfo, canRetryBufferedBody, canQueueBufferedBody, new Set(),
|
|
1163
|
-
).then(() => true)
|
|
1261
|
+
).then(() => true);
|
|
1164
1262
|
}
|
|
1165
1263
|
|
|
1166
|
-
function ensureQueueHeartbeat(res, requestInfo, queueConfig) {
|
|
1264
|
+
function ensureQueueHeartbeat(res, requestInfo, queueConfig, accountManager) {
|
|
1167
1265
|
if (!requestInfo.stream || requestInfo.queueHeartbeatActive || res.headersSent) return;
|
|
1168
1266
|
const heartbeatMs = Math.max(1000, Number(queueConfig.heartbeatMs) || 10_000);
|
|
1169
1267
|
res.writeHead(200, {
|
|
@@ -1175,12 +1273,21 @@ function ensureQueueHeartbeat(res, requestInfo, queueConfig) {
|
|
|
1175
1273
|
res.flushHeaders?.();
|
|
1176
1274
|
res.write(': maxpool queued\n\n');
|
|
1177
1275
|
requestInfo.queueHeartbeatActive = true;
|
|
1276
|
+
// The heartbeat is the liveness probe: if the client is gone (socket
|
|
1277
|
+
// destroyed/ended, or the write throws EPIPE/ERR_STREAM_DESTROYED), release
|
|
1278
|
+
// the queue slot + bytes IMMEDIATELY rather than letting a dead ticket occupy
|
|
1279
|
+
// the queue until its (up to 7d) deadline — the ghost-leak guard.
|
|
1280
|
+
const reapDead = () => {
|
|
1281
|
+
clearQueueHeartbeat(requestInfo);
|
|
1282
|
+
accountManager?.removeQueuedRequest?.(requestInfo);
|
|
1283
|
+
};
|
|
1178
1284
|
requestInfo.queueHeartbeatTimer = setInterval(() => {
|
|
1179
|
-
if (res.destroyed || res.writableEnded) {
|
|
1180
|
-
|
|
1181
|
-
|
|
1285
|
+
if (res.destroyed || res.writableEnded) { reapDead(); return; }
|
|
1286
|
+
try {
|
|
1287
|
+
res.write(': maxpool queued\n\n');
|
|
1288
|
+
} catch {
|
|
1289
|
+
reapDead();
|
|
1182
1290
|
}
|
|
1183
|
-
res.write(': maxpool queued\n\n');
|
|
1184
1291
|
}, heartbeatMs);
|
|
1185
1292
|
requestInfo.queueHeartbeatTimer.unref?.();
|
|
1186
1293
|
}
|
|
@@ -1220,6 +1327,14 @@ async function waitForAvailableRoute(req, res, accountManager, requestInfo, queu
|
|
|
1220
1327
|
&& accountManager.canAdmitQueuedRequest?.(requestInfo) !== false
|
|
1221
1328
|
) return true;
|
|
1222
1329
|
|
|
1330
|
+
// Re-classify each tick: if no eligible route can EVER recover (every
|
|
1331
|
+
// matching account went terminal/auth-dead, the only healthy account was
|
|
1332
|
+
// removed, or the reset is unknown → retryAfterMs Infinity), stop holding
|
|
1333
|
+
// and error fast instead of spinning to the 7d ceiling. Hold is valid only
|
|
1334
|
+
// while ≥1 eligible route has a finite, known reset.
|
|
1335
|
+
const plan = accountManager.nextRetryForRequest?.(requestInfo, new Set());
|
|
1336
|
+
if (plan && plan.cause !== 'available' && !Number.isFinite(plan.retryAfterMs)) return false;
|
|
1337
|
+
|
|
1223
1338
|
const remaining = maxWaitMs - (Date.now() - startedAt);
|
|
1224
1339
|
// Jitter the poll so a synchronized weekly-reset event doesn't re-align
|
|
1225
1340
|
// every waiter's poll into the same instant (thundering scan).
|
|
@@ -1384,6 +1499,14 @@ async function streamResponse(webStream, res, status, responseHeaders, accountIn
|
|
|
1384
1499
|
let committed = res.headersSent;
|
|
1385
1500
|
let readFailed = false;
|
|
1386
1501
|
|
|
1502
|
+
// We're now committed to streaming a real upstream response body onto this
|
|
1503
|
+
// response — there is no more failover for this forward. Stop the queue
|
|
1504
|
+
// heartbeat (if this was a resumed held stream) BEFORE the first real byte, so
|
|
1505
|
+
// the setInterval can't inject ': maxpool queued' comments between live SSE
|
|
1506
|
+
// events (Bug A). It is deliberately NOT cleared earlier (on resume), so a
|
|
1507
|
+
// pre-byte failover can still re-hold the session via queueAndRetry.
|
|
1508
|
+
clearQueueHeartbeat(requestInfo);
|
|
1509
|
+
|
|
1387
1510
|
try {
|
|
1388
1511
|
while (true) {
|
|
1389
1512
|
const { done, value } = await reader.read();
|
package/src/tui.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createInterface } from 'node:readline';
|
|
2
|
-
import {
|
|
2
|
+
import { fetchProfile, loginOAuth } from './oauth.js';
|
|
3
3
|
|
|
4
4
|
// ── ANSI helpers ─────────────────────────────────────────────
|
|
5
5
|
|
|
@@ -93,16 +93,37 @@ function statusColor(status) {
|
|
|
93
93
|
return String(status);
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
/** Short live countdown to a timestamp, SINGLE-unit so it stays ≤3 chars
|
|
97
|
+
* ("41s"/"5m"/"23h"/"2d") and never overflows the status column: seconds under a
|
|
98
|
+
* minute (the common throttle cooldown), then minute/hour/day. '' once elapsed.
|
|
99
|
+
* Accepts a numeric ms timestamp (live account field) or an ISO string (snapshot). */
|
|
100
|
+
function countdown(ts) {
|
|
101
|
+
if (!ts) return '';
|
|
102
|
+
const target = typeof ts === 'string' ? Date.parse(ts) : ts;
|
|
103
|
+
const ms = target - Date.now();
|
|
104
|
+
if (!Number.isFinite(ms) || ms <= 0) return '';
|
|
105
|
+
if (ms < 60_000) return `${Math.ceil(ms / 1000)}s`;
|
|
106
|
+
if (ms < 3_600_000) return `${Math.ceil(ms / 60_000)}m`;
|
|
107
|
+
if (ms < 86_400_000) return `${Math.ceil(ms / 3_600_000)}h`;
|
|
108
|
+
return `${Math.ceil(ms / 86_400_000)}d`;
|
|
109
|
+
}
|
|
110
|
+
|
|
96
111
|
function loadText(load) {
|
|
97
112
|
const cur = load?.current || {};
|
|
98
113
|
const m15 = load?.last15m || {};
|
|
99
114
|
const h1 = load?.last1h || {};
|
|
100
|
-
|
|
115
|
+
// "Now" = what this account is handling right THIS moment: N in-flight requests
|
|
116
|
+
// (and their combined weight ~ payload size, the scheduler's load input). Kept
|
|
117
|
+
// distinct from the "15m"/"1h" THROUGHPUT counts that follow, which the old
|
|
118
|
+
// "Load X/Y" label collided with.
|
|
119
|
+
const inflight = cur.inFlight || 0;
|
|
120
|
+
const weight = cur.activeWeight || 0;
|
|
121
|
+
const now = weight > 0 ? `Now ${inflight} (${weight}w)` : `Now ${inflight}`;
|
|
101
122
|
const recent = `${m15.requests || 0}r`;
|
|
102
123
|
const recentAvg = m15.avgMs != null ? ` ${formatMs(m15.avgMs)}` : '';
|
|
103
124
|
const hour = `${h1.requests || 0}r`;
|
|
104
125
|
const fails = (m15.failed || 0) > 0 ? ` ${red(`${m15.failed}f`)}` : '';
|
|
105
|
-
return
|
|
126
|
+
return `${now} 15m ${recent}${recentAvg}${fails} 1h ${hour}`;
|
|
106
127
|
}
|
|
107
128
|
|
|
108
129
|
function weeklyPolicyText(am, account) {
|
|
@@ -161,7 +182,7 @@ function bar(ratio, w = 10, resetTs) {
|
|
|
161
182
|
return out;
|
|
162
183
|
}
|
|
163
184
|
|
|
164
|
-
export const __tuiTest = { formatReset, quotaLabel, bar, strip };
|
|
185
|
+
export const __tuiTest = { formatReset, quotaLabel, bar, strip, loadText, countdown };
|
|
165
186
|
|
|
166
187
|
function timestamp() {
|
|
167
188
|
return new Date().toLocaleTimeString('en-US', { hour12: false });
|
|
@@ -351,13 +372,7 @@ export class TUI {
|
|
|
351
372
|
}
|
|
352
373
|
|
|
353
374
|
_keyAccounts(k) {
|
|
354
|
-
if (k === '
|
|
355
|
-
this._confirm(
|
|
356
|
-
'Import current Claude login?',
|
|
357
|
-
'Add or update the account currently logged into Claude Code.',
|
|
358
|
-
() => this._doImport(),
|
|
359
|
-
);
|
|
360
|
-
} else if (k === 'k') {
|
|
375
|
+
if (k === 'k') {
|
|
361
376
|
this.mode = 'input';
|
|
362
377
|
this.inputPrompt = 'Anthropic API key';
|
|
363
378
|
this.inputBuf = '';
|
|
@@ -533,26 +548,6 @@ export class TUI {
|
|
|
533
548
|
}
|
|
534
549
|
}
|
|
535
550
|
|
|
536
|
-
async _doImport() {
|
|
537
|
-
try {
|
|
538
|
-
this._addLog('Importing credentials...');
|
|
539
|
-
const creds = await importCredentials(); // file, then macOS Keychain fallback
|
|
540
|
-
const profile = await fetchProfile(creds.accessToken);
|
|
541
|
-
if (!profile || profile.error) {
|
|
542
|
-
this._addLog(`Warning: could not fetch profile — ${profile?.error || 'no token'}`);
|
|
543
|
-
}
|
|
544
|
-
let name;
|
|
545
|
-
if (profile?.email) {
|
|
546
|
-
name = profile.email;
|
|
547
|
-
const tier = profile.hasClaudeMax ? 'Max' : profile.hasClaudePro ? 'Pro' : null;
|
|
548
|
-
if (tier) this._addLog(`Detected Claude ${tier}: ${name}`);
|
|
549
|
-
}
|
|
550
|
-
await this._upsertOAuthAccount({ creds, profile, name, source: 'import', verb: 'Imported' });
|
|
551
|
-
} catch (e) {
|
|
552
|
-
this._addLog(`Import failed: ${e.message}`);
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
|
-
|
|
556
551
|
// Browser OAuth login: any Claude account, named afterward. Suspends the TUI
|
|
557
552
|
// around the interactive flow (browser + name prompt), then resumes.
|
|
558
553
|
async _doLogin() {
|
|
@@ -932,12 +927,19 @@ export class TUI {
|
|
|
932
927
|
case 'waiting': status = yellow('waiting'); break;
|
|
933
928
|
case 'paused': status = yellow('paused'); break;
|
|
934
929
|
case 'disabled': status = gray('disabled'); break;
|
|
935
|
-
case 'throttled':
|
|
930
|
+
case 'throttled': {
|
|
931
|
+
// A transient auto-recovering cooldown — show the remaining time (from
|
|
932
|
+
// rateLimitedUntil) so it reads as "recovering in Ns", not stuck.
|
|
933
|
+
const cd = countdown(a.rateLimitedUntil);
|
|
934
|
+
status = yellow(cd ? `throttled ${cd}` : 'throttled');
|
|
935
|
+
break;
|
|
936
|
+
}
|
|
936
937
|
case 'exhausted': status = red('exhausted'); break;
|
|
937
938
|
case 'error': status = red('error'); break;
|
|
938
939
|
default: status = a.status || 'ready';
|
|
939
940
|
}
|
|
940
|
-
|
|
941
|
+
// Widened from 10 to fit "throttled 59s" so the quota bars stay column-aligned.
|
|
942
|
+
status = rpad(status, 13);
|
|
941
943
|
|
|
942
944
|
if (a.type === 'provider') {
|
|
943
945
|
return this._renderProviderAcct(sel, cur, name, type, status, a);
|
|
@@ -1008,7 +1010,7 @@ export class TUI {
|
|
|
1008
1010
|
case 'normal':
|
|
1009
1011
|
return ` ${bold('a')} Accounts ${bold('m')} Routing ${bold('s')} Sync ${bold('r')} Restart ${bold('q')} Stop`;
|
|
1010
1012
|
case 'accounts':
|
|
1011
|
-
return ` ${bold('
|
|
1013
|
+
return ` ${bold('l')} Login (browser) ${bold('k')} API key ${bold('n')} Rename ${bold('t')} Enable/disable ${bold('d')} Delete ${bold('Esc')} Back`;
|
|
1012
1014
|
case 'routing':
|
|
1013
1015
|
return ` ${bold('a')} Automatic ${bold('p')} Manual preference ${bold('Esc')} Back`;
|
|
1014
1016
|
case 'select': {
|