@rikcodes/teamclaude 1.1.13-rik.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +122 -0
- package/package.json +43 -0
- package/src/account-manager.js +1459 -0
- package/src/account-uuid-rewrite.js +115 -0
- package/src/alias.js +125 -0
- package/src/claude-env.js +65 -0
- package/src/config.js +146 -0
- package/src/crash-log.js +27 -0
- package/src/egress-guard.js +132 -0
- package/src/identity.js +96 -0
- package/src/index.js +1873 -0
- package/src/json-format-stream.js +63 -0
- package/src/mitm.js +336 -0
- package/src/model.js +276 -0
- package/src/oauth.js +459 -0
- package/src/prober.js +158 -0
- package/src/request-log.js +32 -0
- package/src/resolve-accounts.js +43 -0
- package/src/server.js +1319 -0
- package/src/service.js +241 -0
- package/src/session-tracker.js +133 -0
- package/src/status-renderer.js +316 -0
- package/src/sx.js +218 -0
- package/src/terminal-title.js +31 -0
- package/src/tool-pair-sanitize.js +193 -0
- package/src/tui-remote.js +274 -0
- package/src/tui.js +1634 -0
- package/src/updater.js +177 -0
- package/src/upstream-fetch.js +267 -0
- package/src/upstream-proxy.js +214 -0
- package/src/warmer.js +237 -0
- package/src/x509.js +166 -0
|
@@ -0,0 +1,1459 @@
|
|
|
1
|
+
import { refreshAccessToken, isTokenExpiringSoon, isTokenExpired } from './oauth.js';
|
|
2
|
+
import { sameIdentity } from './identity.js';
|
|
3
|
+
import { weeklyBucketForModel, modelGlobMatches } from './model.js';
|
|
4
|
+
import { SessionTracker } from './session-tracker.js';
|
|
5
|
+
|
|
6
|
+
// Re-exported for callers that import these model helpers from here.
|
|
7
|
+
export { isFableModel, parseRequestModel, parseAdvisorModel } from './model.js';
|
|
8
|
+
|
|
9
|
+
// How long after a successful token refresh a forced (post-401) refresh is
|
|
10
|
+
// suppressed. Long enough to cover the 401s from requests already in flight
|
|
11
|
+
// when the token turned over, short enough that a genuinely bad new token
|
|
12
|
+
// recovers on the next request rather than staying stuck.
|
|
13
|
+
const FORCED_REFRESH_FLOOR_MS = 10_000;
|
|
14
|
+
|
|
15
|
+
// Quota fields that survive a restart: utilization levels and their reset
|
|
16
|
+
// windows, learned passively from upstream responses. Transient/derived state
|
|
17
|
+
// (probing, requalify, rateLimitedUntil) is intentionally excluded.
|
|
18
|
+
const PERSISTED_QUOTA_FIELDS = [
|
|
19
|
+
'unified5h', 'unified7d', 'unified7dSonnet', 'unified7dFable',
|
|
20
|
+
'unified5hReset', 'unified7dReset', 'unified7dSonnetReset', 'unified7dFableReset', 'unifiedStatus',
|
|
21
|
+
'tokensLimit', 'tokensRemaining', 'requestsLimit', 'requestsRemaining', 'resetsAt',
|
|
22
|
+
];
|
|
23
|
+
|
|
24
|
+
function emptyQuota() {
|
|
25
|
+
return {
|
|
26
|
+
// Standard API rate limits (API key accounts)
|
|
27
|
+
tokensLimit: null,
|
|
28
|
+
tokensRemaining: null,
|
|
29
|
+
requestsLimit: null,
|
|
30
|
+
requestsRemaining: null,
|
|
31
|
+
// Unified rate limits (Claude Max accounts)
|
|
32
|
+
unified5h: null, // utilization 0-1
|
|
33
|
+
unified7d: null, // utilization 0-1
|
|
34
|
+
unified7dSonnet: null, // utilization 0-1 (Sonnet-specific weekly bucket)
|
|
35
|
+
unified7dFable: null, // utilization 0-1 (Fable-specific weekly bucket)
|
|
36
|
+
unified5hReset: null, // ms timestamp
|
|
37
|
+
unified7dReset: null, // ms timestamp
|
|
38
|
+
unified7dSonnetReset: null, // ms timestamp
|
|
39
|
+
unified7dFableReset: null, // ms timestamp
|
|
40
|
+
unifiedStatus: null, // allowed | allowed_warning | rejected
|
|
41
|
+
resetsAt: null,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Build a fresh in-memory account record from a config/disk account object.
|
|
46
|
+
// Shared by the constructor and addAccount() so the field set can never drift
|
|
47
|
+
// between startup accounts and runtime-added ones (a divergence here once left
|
|
48
|
+
// runtime-added accounts without `inFlight`, hanging every request in admit()).
|
|
49
|
+
function makeAccount(acct, index) {
|
|
50
|
+
return {
|
|
51
|
+
index,
|
|
52
|
+
name: acct.name,
|
|
53
|
+
type: acct.type,
|
|
54
|
+
accountUuid: acct.accountUuid || null,
|
|
55
|
+
orgUuid: acct.orgUuid || null,
|
|
56
|
+
orgName: acct.orgName || null,
|
|
57
|
+
priority: acct.priority || 0,
|
|
58
|
+
disabled: acct.disabled || false,
|
|
59
|
+
upstream: acct.upstream || null,
|
|
60
|
+
modelMap: acct.modelMap || null,
|
|
61
|
+
models: acct.models || null,
|
|
62
|
+
credential: acct.accessToken || acct.apiKey,
|
|
63
|
+
refreshToken: acct.refreshToken || null,
|
|
64
|
+
expiresAt: acct.expiresAt || null,
|
|
65
|
+
status: 'active',
|
|
66
|
+
// No quota is known at startup, so start probing: the first response for
|
|
67
|
+
// an account reveals its weekly limit and triggers re-evaluation.
|
|
68
|
+
probing: true,
|
|
69
|
+
quota: emptyQuota(),
|
|
70
|
+
usage: {
|
|
71
|
+
totalInputTokens: 0,
|
|
72
|
+
totalOutputTokens: 0,
|
|
73
|
+
totalRequests: 0,
|
|
74
|
+
lastUsed: null,
|
|
75
|
+
},
|
|
76
|
+
rateLimitedUntil: null,
|
|
77
|
+
throttledAt: null,
|
|
78
|
+
// Storm control (see admit/release): in-flight upstream requests and the
|
|
79
|
+
// time this account last became the current one (starts a ramp window).
|
|
80
|
+
inFlight: 0,
|
|
81
|
+
rampStartedAt: null,
|
|
82
|
+
// Rate-limit pause (see pauseAccount): a short window during which new
|
|
83
|
+
// requests wait in admit() rather than flooding — set from a 429's
|
|
84
|
+
// retry-after. Distinct from `throttled`/rateLimitedUntil: it does NOT
|
|
85
|
+
// make the account unavailable, so selection never rotates away from it.
|
|
86
|
+
pausedUntil: null,
|
|
87
|
+
// When this account's token was last successfully refreshed. Gates forced
|
|
88
|
+
// (post-401) refreshes so a burst of stale in-flight requests can't rotate
|
|
89
|
+
// the refresh-token family once per request — see ensureTokenFresh.
|
|
90
|
+
_lastRefreshAt: null,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Does a declared `models` entry name `model`? The declared side may carry a
|
|
95
|
+
// trailing [Nm] context-length suffix (e.g. "deepseek-v4-pro[1m]"); we match it
|
|
96
|
+
// against a bare request too. Shared by _accountOwnsModel's two lookups so the
|
|
97
|
+
// predicate can't drift.
|
|
98
|
+
function modelMatches(declared, model) {
|
|
99
|
+
return declared === model || declared.replace(/\[\d+m\]$/, '') === model;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// A representative model for a route's own globs, used to report what that route
|
|
103
|
+
// does right now (which accounts may serve it, and which one it would pick).
|
|
104
|
+
// Taken from the route object rather than looked up by name, so two routes
|
|
105
|
+
// sharing a name are still each described by their own globs.
|
|
106
|
+
function sampleModelFor(route) {
|
|
107
|
+
return route.match[0].replace(/\*/g, '') || 'model';
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export class AccountManager {
|
|
111
|
+
constructor(accounts, switchThreshold = 0.98, { refreshFn = refreshAccessToken, throttleProbeFloorMs, forcedRefreshFloorMs = FORCED_REFRESH_FLOOR_MS, routes, ramp, distributeSessions = false, soonestWeekly, sessionTracker } = {}) {
|
|
112
|
+
// How long a just-minted token is trusted against a forced refresh.
|
|
113
|
+
this._forcedRefreshFloorMs = forcedRefreshFloorMs;
|
|
114
|
+
// Injectable for tests (mirrors Prober's probeFn); defaults to the real
|
|
115
|
+
// OAuth token refresh.
|
|
116
|
+
this._refreshFn = refreshFn;
|
|
117
|
+
this.accounts = accounts.map((acct, index) => makeAccount(acct, index));
|
|
118
|
+
this.currentIndex = 0;
|
|
119
|
+
// Session awareness (issue #109). The tracker is always on (passive — it just
|
|
120
|
+
// observes the x-claude-code-session-id header for the status readout).
|
|
121
|
+
// `distributeSessions` gates the behavioural change: keep each session on its
|
|
122
|
+
// account for cache reuse, but spread NEW sessions across equal-priority
|
|
123
|
+
// accounts by load instead of funnelling them all onto the current one.
|
|
124
|
+
this.sessionTracker = sessionTracker || new SessionTracker();
|
|
125
|
+
this.distributeSessions = !!distributeSessions;
|
|
126
|
+
// Ephemeral per-route manual pins (routeName → account index). Not persisted:
|
|
127
|
+
// like the global manual switch (currentIndex) these are runtime overrides that
|
|
128
|
+
// bias selection for a route's models and reset on restart. A pinned account
|
|
129
|
+
// that becomes ineligible is skipped — routing falls back to best-available.
|
|
130
|
+
this.routePins = new Map();
|
|
131
|
+
this.switchThreshold = switchThreshold;
|
|
132
|
+
this.setRoutes(routes);
|
|
133
|
+
this.setSoonestWeekly(soonestWeekly);
|
|
134
|
+
// Storm control: when rotation switches to a fresh account, a burst of
|
|
135
|
+
// in-flight requests (e.g. dozens of agents failing over together) would all
|
|
136
|
+
// hit it at once and instantly throttle it — cascading down the fleet
|
|
137
|
+
// (issue #84). admit() caps concurrent requests to a just-switched account
|
|
138
|
+
// and ramps the cap up over a short window, so the first few reveal whether
|
|
139
|
+
// it's also near-exhausted before the whole herd commits.
|
|
140
|
+
this.ramp = {
|
|
141
|
+
enabled: true,
|
|
142
|
+
startConc: 1, // concurrent requests allowed at the instant of a switch
|
|
143
|
+
stepConc: 1, // cap increase per stepMs
|
|
144
|
+
stepMs: 250, // → +stepConc every 250ms (default ramps ~4 req/s)
|
|
145
|
+
windowMs: 30_000, // after this, pacing stops entirely (cap = Infinity)
|
|
146
|
+
pollMs: 50, // how often a waiting request re-checks the cap
|
|
147
|
+
...ramp,
|
|
148
|
+
};
|
|
149
|
+
// When every account reads as over-quota we would otherwise refuse locally
|
|
150
|
+
// forever (a stale cached utilization is never re-validated because no
|
|
151
|
+
// request is ever sent). Instead, allow one real upstream probe at most this
|
|
152
|
+
// often to refresh the cached quota. See _selectProbe.
|
|
153
|
+
this.probeIntervalMs = 60_000;
|
|
154
|
+
this._nextProbeAt = 0;
|
|
155
|
+
// Minimum time a 429 hold is respected verbatim before a throttled account
|
|
156
|
+
// becomes probe-eligible (see _isProbeable). Long enough to honor a genuine
|
|
157
|
+
// retry-after, short enough that a stale hold cannot pin the fleet.
|
|
158
|
+
this.throttleProbeFloorMs = throttleProbeFloorMs
|
|
159
|
+
?? (Number(process.env.TEAMCLAUDE_THROTTLE_PROBE_FLOOR_MS) || 60_000);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Start (or restart) the ramp window for an account that just became current,
|
|
163
|
+
* so a failover burst is paced onto it rather than all landing at once. */
|
|
164
|
+
_beginRamp(account) {
|
|
165
|
+
if (account && this.ramp.enabled) account.rampStartedAt = Date.now();
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Max concurrent upstream requests allowed to `account` right now. Infinity
|
|
169
|
+
* once the ramp window has elapsed (or ramping is off / never started). */
|
|
170
|
+
_rampCap(account, now = Date.now()) {
|
|
171
|
+
if (!this.ramp.enabled || account.rampStartedAt == null) return Infinity;
|
|
172
|
+
// Clamp to 0: pauseAccount arms rampStartedAt in the FUTURE (pause-end), so a
|
|
173
|
+
// call during the pause would otherwise yield a negative elapsed → negative
|
|
174
|
+
// cap. admit()'s pause branch already guards this, but keep _rampCap sound on
|
|
175
|
+
// its own — a future start simply means "cap is at its floor (startConc)".
|
|
176
|
+
const elapsed = Math.max(0, now - account.rampStartedAt);
|
|
177
|
+
if (elapsed >= this.ramp.windowMs) { account.rampStartedAt = null; return Infinity; }
|
|
178
|
+
return this.ramp.startConc + Math.floor(elapsed / this.ramp.stepMs) * this.ramp.stepConc;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Reserve a concurrency slot on `account` before sending upstream. Waits while
|
|
183
|
+
* the account is in a rate-limit pause (a 429's retry-after window) and while
|
|
184
|
+
* it is over its current ramp cap. Fail-open: returns true once a slot is taken
|
|
185
|
+
* (always eventually — the pause ends and the ramp cap grows), or false if
|
|
186
|
+
* `isAborted()` reports the client went away while waiting. Pair every `true`
|
|
187
|
+
* with a `release(index)`.
|
|
188
|
+
*/
|
|
189
|
+
async admit(index, isAborted) {
|
|
190
|
+
const account = this.accounts[index];
|
|
191
|
+
if (!account) return true;
|
|
192
|
+
while (true) {
|
|
193
|
+
if (isAborted?.()) return false;
|
|
194
|
+
const now = Date.now();
|
|
195
|
+
// Rate-limit pause: hold new requests off this account until the window
|
|
196
|
+
// passes instead of flooding it (which would deepen the 429). Not a
|
|
197
|
+
// rotation trigger — the account stays selectable the whole time.
|
|
198
|
+
if (account.pausedUntil && now < account.pausedUntil) {
|
|
199
|
+
await new Promise(r => setTimeout(r, Math.min(account.pausedUntil - now, this.ramp.pollMs * 4)));
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
const cap = this.ramp.enabled ? this._rampCap(account, now) : Infinity;
|
|
203
|
+
if (account.inFlight < cap) { account.inFlight++; return true; }
|
|
204
|
+
await new Promise(r => setTimeout(r, this.ramp.pollMs));
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Release a slot taken by admit(). Safe to call once per successful admit. */
|
|
209
|
+
release(index) {
|
|
210
|
+
const account = this.accounts[index];
|
|
211
|
+
if (account && account.inFlight > 0) account.inFlight--;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Pause an account after a rate-limit (non-quota) 429 so concurrent requests
|
|
216
|
+
* wait in admit() instead of piling on. Unlike markRateLimited this does NOT
|
|
217
|
+
* set `throttled`/rateLimitedUntil, so _isAvailable still returns true and
|
|
218
|
+
* selection never rotates away — rotation is reserved for quota exhaustion.
|
|
219
|
+
* When the pause lifts, the held requests are released through a fresh ramp
|
|
220
|
+
* window (storm control) so they trickle out rather than flood. Extends an
|
|
221
|
+
* existing pause rather than shortening it.
|
|
222
|
+
*/
|
|
223
|
+
pauseAccount(index, seconds) {
|
|
224
|
+
const account = this.accounts[index];
|
|
225
|
+
if (!account) return;
|
|
226
|
+
const until = Date.now() + Math.max(0, seconds) * 1000;
|
|
227
|
+
account.pausedUntil = Math.max(account.pausedUntil || 0, until);
|
|
228
|
+
// Arm the ramp to begin when the pause ends: while paused, admit() holds on
|
|
229
|
+
// the pause branch; once it lifts, _rampCap counts from here and releases the
|
|
230
|
+
// backlog gradually (startConc, then +stepConc per step).
|
|
231
|
+
if (this.ramp.enabled) account.rampStartedAt = account.pausedUntil;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Get the best available account, rotating if the current one is near quota.
|
|
236
|
+
* Returns null if all accounts are exhausted.
|
|
237
|
+
*
|
|
238
|
+
* `advisorModel` is the second model an advisor request carries (Claude Code's
|
|
239
|
+
* advisor tool, nested in tools[] — see parseAdvisorModel): the advisor
|
|
240
|
+
* sub-inference runs on the SAME account and spends that model's family
|
|
241
|
+
* bucket, so the account must be eligible for both models. When no account
|
|
242
|
+
* satisfies both, selection degrades to executor-only routing so the main
|
|
243
|
+
* request keeps flowing (upstream then fails just the advisor call).
|
|
244
|
+
*/
|
|
245
|
+
getActiveAccount(exclude = null, model = null, advisorModel = null, sessionId = null) {
|
|
246
|
+
// Clear expired quotas across all accounts and switch proactively if a
|
|
247
|
+
// session reset made a sooner-expiring account the better choice. This runs
|
|
248
|
+
// on every request so the behaviour holds without the TUI render loop.
|
|
249
|
+
this.refreshExpiredQuotas();
|
|
250
|
+
// Session-affinity distribution (opt-in): keep a session on its pinned
|
|
251
|
+
// account for cache reuse, and route a new session to the least-loaded
|
|
252
|
+
// account. Only when enabled, only for a real session, and only outside a
|
|
253
|
+
// manual route pin (which must still win). Falls through to the normal walk
|
|
254
|
+
// if nothing session-eligible is found (e.g. the whole tier is exhausted).
|
|
255
|
+
if (this.distributeSessions && sessionId && !this._pinnedAccountForModel(model, advisorModel)) {
|
|
256
|
+
const acc = this._selectForSession(sessionId, exclude, model, advisorModel);
|
|
257
|
+
if (acc) return acc;
|
|
258
|
+
}
|
|
259
|
+
if (advisorModel) {
|
|
260
|
+
const account = this._select(exclude, model, advisorModel, false);
|
|
261
|
+
if (account) return account;
|
|
262
|
+
// Throttled so a busy advisor session doesn't flood the activity log.
|
|
263
|
+
if (Date.now() >= (this._advisorDegradeLogAt || 0)) {
|
|
264
|
+
this._advisorDegradeLogAt = Date.now() + 60_000;
|
|
265
|
+
console.log(`[TeamClaude] No account eligible for advisor model "${advisorModel}" — routing by request model only`);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return this._select(exclude, model, null, true);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** The selection walk getActiveAccount runs: manual pin → current account →
|
|
272
|
+
* best-available. `allowProbe` gates the exhausted-fleet probe fallback so the
|
|
273
|
+
* advisor-constrained pass can fail soft (degrade to executor-only) instead of
|
|
274
|
+
* burning the throttled probe slot on the stricter constraint. */
|
|
275
|
+
_select(exclude, model, advisorModel, allowProbe) {
|
|
276
|
+
// A manual per-route pin biases selection for that route's models (independent
|
|
277
|
+
// of the global currentIndex). Honored only while eligible — otherwise we fall
|
|
278
|
+
// through to normal best-available selection so requests keep flowing.
|
|
279
|
+
const pinned = this._pinnedAccountForModel(model, advisorModel);
|
|
280
|
+
if (pinned && this._isAvailable(pinned, model, advisorModel) && !exclude?.has(pinned.index)) return pinned;
|
|
281
|
+
const current = this.accounts[this.currentIndex];
|
|
282
|
+
// `model` scopes availability: an account whose Fable weekly bucket is spent
|
|
283
|
+
// is still fully usable for other models, so it is only excluded when THIS
|
|
284
|
+
// request targets Fable (see _isAvailable).
|
|
285
|
+
// `exclude` is a per-request set of indices already tried this request (e.g.
|
|
286
|
+
// an account that just threw a transport error). It is never a persistent
|
|
287
|
+
// status change — the account stays healthy for the next request.
|
|
288
|
+
// We just learned a probed account's weekly quota — re-evaluate which
|
|
289
|
+
// account is best now that its limit is known.
|
|
290
|
+
if (current && current.requalify) {
|
|
291
|
+
// Consume the flag on the final pass; the advisor-constrained pass leaves
|
|
292
|
+
// it set unless it actually switches, so the requalification isn't lost
|
|
293
|
+
// when that pass comes up empty and selection degrades.
|
|
294
|
+
if (allowProbe) current.requalify = false;
|
|
295
|
+
const next = this._selectNext(exclude, model, advisorModel);
|
|
296
|
+
if (next) { current.requalify = false; return next; }
|
|
297
|
+
}
|
|
298
|
+
if (this._isAvailable(current, model, advisorModel) && !exclude?.has(current.index)) {
|
|
299
|
+
const betterExists = this._preemptedBy(current, model, advisorModel, exclude);
|
|
300
|
+
return betterExists ? this._selectNext(exclude, model, advisorModel) : current;
|
|
301
|
+
}
|
|
302
|
+
const next = this._selectNext(exclude, model, advisorModel);
|
|
303
|
+
if (next) return next;
|
|
304
|
+
// No account is under the switch threshold. Before refusing locally, allow a
|
|
305
|
+
// throttled probe so a stale/poisoned cached quota can't pin us in a
|
|
306
|
+
// permanent "all exhausted" state — the probe's real response refreshes the
|
|
307
|
+
// quota (or upstream's own 429 converts soft exhaustion into a hard
|
|
308
|
+
// rate-limit hold). null here means the caller emits the synthetic 429.
|
|
309
|
+
return allowProbe ? this._selectProbe(exclude, model) : null;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** Session-affinity selection (opt-in, issue #109). Honor a known session's
|
|
313
|
+
* pin when that account is still eligible and not preempted by a
|
|
314
|
+
* higher-priority one; otherwise route the session to the least-loaded
|
|
315
|
+
* eligible account. Returns null if nothing is eligible, so the caller falls
|
|
316
|
+
* back to the normal quota-driven walk. Does NOT record the pin — that happens
|
|
317
|
+
* on the actual route (recordSession), so retries/failover re-pin naturally. */
|
|
318
|
+
_selectForSession(sessionId, exclude, model, advisorModel) {
|
|
319
|
+
const pinIdx = this.sessionTracker.pinnedAccount(sessionId);
|
|
320
|
+
if (pinIdx != null) {
|
|
321
|
+
const pinned = this.accounts[pinIdx];
|
|
322
|
+
if (pinned && this._isAvailable(pinned, model, advisorModel) && !exclude?.has(pinIdx)) {
|
|
323
|
+
// Mirror _select's preemption (priority, and soonest-weekly when
|
|
324
|
+
// enabled) so an operator's priority order — and a strictly sooner
|
|
325
|
+
// weekly pool — still win over a session's stickiness.
|
|
326
|
+
if (!this._preemptedBy(pinned, model, advisorModel, exclude)) return pinned;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
return this._pickLeastLoaded(exclude, model, advisorModel);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** Best-available biased toward the fewest active sessions, so new sessions
|
|
333
|
+
* spread across equal-priority accounts instead of funnelling onto one. Order:
|
|
334
|
+
* priority → fewest active sessions → fewest in-flight → soonest weekly reset
|
|
335
|
+
* (the existing tiebreak). */
|
|
336
|
+
_pickLeastLoaded(exclude = null, model = null, advisorModel = null) {
|
|
337
|
+
const now = Date.now();
|
|
338
|
+
const candidates = [];
|
|
339
|
+
for (const account of this.accounts) {
|
|
340
|
+
if (exclude?.has(account.index)) continue;
|
|
341
|
+
if (!this._isAvailable(account, model, advisorModel)) continue;
|
|
342
|
+
candidates.push(account);
|
|
343
|
+
}
|
|
344
|
+
// Soonest-weekly pool: within the winning priority tier, only accounts
|
|
345
|
+
// whose governing weekly reset is within poolHours of the soonest known
|
|
346
|
+
// reset receive new sessions. An unknown reset counts as in-pool so a
|
|
347
|
+
// request still reaches it and learns its quota (the same probe-first
|
|
348
|
+
// convention as _pickBestAvailable).
|
|
349
|
+
const sw = this.soonestWeekly;
|
|
350
|
+
let poolEdge = Infinity;
|
|
351
|
+
if (sw.enabled && candidates.length) {
|
|
352
|
+
const tier = Math.min(...candidates.map(a => a.priority || 0));
|
|
353
|
+
for (const a of candidates) {
|
|
354
|
+
if ((a.priority || 0) !== tier) continue;
|
|
355
|
+
const reset = this._governingWeeklyReset(a, model);
|
|
356
|
+
if (reset != null && reset < poolEdge) poolEdge = reset;
|
|
357
|
+
}
|
|
358
|
+
poolEdge += sw.poolHours * 3600_000;
|
|
359
|
+
}
|
|
360
|
+
let best = null;
|
|
361
|
+
let bestPriority = Infinity;
|
|
362
|
+
let bestInPool = false;
|
|
363
|
+
let bestSessions = Infinity;
|
|
364
|
+
let bestInFlight = Infinity;
|
|
365
|
+
let bestReset = Infinity;
|
|
366
|
+
for (const account of candidates) {
|
|
367
|
+
const priority = account.priority || 0;
|
|
368
|
+
const reset = this._governingWeeklyReset(account, model) || -Infinity;
|
|
369
|
+
const inPool = reset <= poolEdge;
|
|
370
|
+
const sessions = this.sessionTracker.activeCountFor(account.index, now);
|
|
371
|
+
const inFlight = account.inFlight || 0;
|
|
372
|
+
if (priority < bestPriority
|
|
373
|
+
|| (priority === bestPriority && inPool > bestInPool)
|
|
374
|
+
|| (priority === bestPriority && inPool === bestInPool && sessions < bestSessions)
|
|
375
|
+
|| (priority === bestPriority && inPool === bestInPool && sessions === bestSessions && inFlight < bestInFlight)
|
|
376
|
+
|| (priority === bestPriority && inPool === bestInPool && sessions === bestSessions && inFlight === bestInFlight && reset < bestReset)) {
|
|
377
|
+
best = account;
|
|
378
|
+
bestPriority = priority;
|
|
379
|
+
bestInPool = inPool;
|
|
380
|
+
bestSessions = sessions;
|
|
381
|
+
bestInFlight = inFlight;
|
|
382
|
+
bestReset = reset;
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
return best;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/** Record that a session's request was served by an account (always on, even
|
|
389
|
+
* when distribution is off — the readout is passive). This is what pins a
|
|
390
|
+
* session for future affinity. */
|
|
391
|
+
recordSession(sessionId, accountIndex) {
|
|
392
|
+
if (sessionId) this.sessionTracker.touch(sessionId, accountIndex);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/** Mark a session request as in flight / finished. Paired around the whole
|
|
396
|
+
* client request (including retries) so a long streaming completion keeps the
|
|
397
|
+
* session counted as active for its full duration. */
|
|
398
|
+
beginSession(sessionId) {
|
|
399
|
+
if (sessionId) this.sessionTracker.beginRequest(sessionId);
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
endSession(sessionId) {
|
|
403
|
+
if (sessionId) this.sessionTracker.endRequest(sessionId);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** { known, active, perAccount } session counts for status/TUI. */
|
|
407
|
+
sessionStats() {
|
|
408
|
+
return this.sessionTracker.stats();
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* Like getActiveAccount, but if the selected account's OAuth token has ALREADY
|
|
413
|
+
* expired it blocks on a refresh before returning — so a caller that injects
|
|
414
|
+
* the token immediately (the MITM relay) never sends a dead token and eats a
|
|
415
|
+
* 401. A token that is merely expiring soon (still valid) is left to the
|
|
416
|
+
* caller's opportunistic background refresh; only a hard-expired one blocks.
|
|
417
|
+
*/
|
|
418
|
+
async getActiveAccountFresh(exclude = null, model = null, advisorModel = null, sessionId = null) {
|
|
419
|
+
const account = this.getActiveAccount(exclude, model, advisorModel, sessionId);
|
|
420
|
+
if (account && account.type === 'oauth' && account.refreshToken
|
|
421
|
+
&& isTokenExpired(account.expiresAt)) {
|
|
422
|
+
await this.ensureTokenFresh(account.index); // coalesces with any in-flight refresh
|
|
423
|
+
}
|
|
424
|
+
return account;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/**
|
|
428
|
+
* Read-only: the index of the account a request for `model` would be served by
|
|
429
|
+
* right now — the same decision getActiveAccount makes (manual pin → the global
|
|
430
|
+
* current account if it can serve the model → best-available), but WITHOUT
|
|
431
|
+
* mutating currentIndex and without the exhausted-fleet probe fallback. Returns
|
|
432
|
+
* null when nothing can serve `model` at the moment. The TUI uses this to mark
|
|
433
|
+
* the single account each secondary bucket (Fable/Sonnet) currently routes to —
|
|
434
|
+
* the F7/S7 analogue of the ► that marks the default route's current account.
|
|
435
|
+
*/
|
|
436
|
+
previewRouteIndex(model) {
|
|
437
|
+
const pinned = this._pinnedAccountForModel(model);
|
|
438
|
+
if (pinned && this._isAvailable(pinned, model)) return pinned.index;
|
|
439
|
+
const current = this.accounts[this.currentIndex];
|
|
440
|
+
if (current && this._isAvailable(current, model)) {
|
|
441
|
+
// Mirror getActiveAccount's priority preemption: a strictly higher-priority
|
|
442
|
+
// available account wins over a healthy current one; same tier stays put.
|
|
443
|
+
const better = this.accounts.some(a =>
|
|
444
|
+
this._isAvailable(a, model) && (a.priority || 0) < (current.priority || 0));
|
|
445
|
+
if (!better) return current.index;
|
|
446
|
+
}
|
|
447
|
+
const best = this._pickBestAvailable(null, model);
|
|
448
|
+
return best ? best.index : null;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
_isProbeable(account) {
|
|
452
|
+
if (!account) return false;
|
|
453
|
+
// Never probe an account the operator has taken out of rotation or one
|
|
454
|
+
// whose token is broken — those are hard states, not stale guesses.
|
|
455
|
+
if (account.disabled) return false;
|
|
456
|
+
if (account.status === 'error' || account.status === 'exhausted') return false;
|
|
457
|
+
// A 429 hold is respected verbatim at first, but a hold is a snapshot: the
|
|
458
|
+
// 429 that armed it may itself have been transient (e.g. the retry burst
|
|
459
|
+
// after a network flap), and while it lasts NOTHING revalidates it — so a
|
|
460
|
+
// stale hold pins the fleet in synthetic 429s for up to an hour and only a
|
|
461
|
+
// restart (which wipes the in-memory hold) recovers. After the floor, let
|
|
462
|
+
// the account be probed: the probe's real response either clears the hold
|
|
463
|
+
// (any non-429 → clearRateLimited) or re-arms it with a fresh retry-after.
|
|
464
|
+
if (account.status === 'throttled' && account.rateLimitedUntil
|
|
465
|
+
&& Date.now() < account.rateLimitedUntil) {
|
|
466
|
+
return Date.now() >= (account.throttledAt || 0) + this.throttleProbeFloorMs;
|
|
467
|
+
}
|
|
468
|
+
return true;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/** Highest utilization across the quota dimensions that govern `model` (0-1),
|
|
472
|
+
* used to pick the least-exhausted probe target. Mirrors _isNearQuota: the
|
|
473
|
+
* shared 5-hour bucket plus the model's governing weekly bucket. With no model
|
|
474
|
+
* it falls back to the shared weekly. */
|
|
475
|
+
_maxUtilization(account, model = null) {
|
|
476
|
+
const q = account.quota;
|
|
477
|
+
let max = 0;
|
|
478
|
+
if (q.unified5h != null) max = Math.max(max, q.unified5h);
|
|
479
|
+
const weeklyVal = this._governingWeekly(account, model);
|
|
480
|
+
if (weeklyVal != null) max = Math.max(max, weeklyVal);
|
|
481
|
+
if (q.tokensLimit != null && q.tokensRemaining != null) {
|
|
482
|
+
max = Math.max(max, 1 - q.tokensRemaining / q.tokensLimit);
|
|
483
|
+
}
|
|
484
|
+
if (q.requestsLimit != null && q.requestsRemaining != null) {
|
|
485
|
+
max = Math.max(max, 1 - q.requestsRemaining / q.requestsLimit);
|
|
486
|
+
}
|
|
487
|
+
return max;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/** Utilization (0-1) of the weekly bucket that governs `model` on this account:
|
|
491
|
+
* unified7dFable for Fable, unified7dSonnet for Sonnet, unified7d otherwise.
|
|
492
|
+
* Falls back to the shared unified7d when a family-specific bucket isn't
|
|
493
|
+
* reported. Returns null when nothing is known. */
|
|
494
|
+
_governingWeekly(account, model) {
|
|
495
|
+
const q = account.quota;
|
|
496
|
+
const key = this._weeklyBucketFor(model);
|
|
497
|
+
if (q[key] != null) return q[key];
|
|
498
|
+
return key !== 'unified7d' ? q.unified7d : null;
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/** Reset timestamp (ms) of the weekly bucket that governs `model`, falling back
|
|
502
|
+
* to the shared weekly reset. Used to spend the soonest-expiring quota first. */
|
|
503
|
+
_governingWeeklyReset(account, model) {
|
|
504
|
+
const q = account.quota;
|
|
505
|
+
const key = this._weeklyBucketFor(model);
|
|
506
|
+
return q[`${key}Reset`] || q.unified7dReset || null;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/** True when the family-specific weekly bucket that governs `model` is spent.
|
|
510
|
+
* Unlike _isNearQuota this ignores the shared 5h/weekly caps — it is only used
|
|
511
|
+
* to skip an account for a probe of a model it definitely can't serve. Returns
|
|
512
|
+
* false for families without a dedicated bucket (they share unified7d, already
|
|
513
|
+
* covered by _isNearQuota). */
|
|
514
|
+
_modelWeeklyExhausted(account, model) {
|
|
515
|
+
const q = account.quota;
|
|
516
|
+
const key = this._weeklyBucketFor(model);
|
|
517
|
+
if (key === 'unified7d') return false;
|
|
518
|
+
return q[key] != null && q[key] >= this.switchThreshold;
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
/**
|
|
522
|
+
* Pick an account to send a single revalidation probe upstream when every
|
|
523
|
+
* account reads as over the switch threshold. Throttled to one probe per
|
|
524
|
+
* probeIntervalMs so a genuinely-exhausted fleet isn't hammered — between
|
|
525
|
+
* probes this returns null and the caller falls back to the synthetic 429.
|
|
526
|
+
* The chosen account is the least-utilized probeable one (most likely to have
|
|
527
|
+
* stale headroom), so the refreshed quota corrects the cache fastest.
|
|
528
|
+
*/
|
|
529
|
+
_selectProbe(exclude = null, model = null) {
|
|
530
|
+
const now = Date.now();
|
|
531
|
+
if (now < this._nextProbeAt) return null;
|
|
532
|
+
|
|
533
|
+
let best = null;
|
|
534
|
+
let bestPriority = Infinity;
|
|
535
|
+
let bestUsage = Infinity;
|
|
536
|
+
for (const account of this.accounts) {
|
|
537
|
+
if (exclude?.has(account.index)) continue;
|
|
538
|
+
if (!this._isProbeable(account)) continue;
|
|
539
|
+
// A family-exhausted account can't serve that family even as a probe — it
|
|
540
|
+
// would just 429 again — so skip it (Fable/Sonnet) and let the caller emit
|
|
541
|
+
// the synthetic 429 when no other account is available.
|
|
542
|
+
if (model && this._modelWeeklyExhausted(account, model)) continue;
|
|
543
|
+
// Same for routing/ownership: a probe for a routed or owned model must not
|
|
544
|
+
// land on an ineligible account (it would just reject the unknown model id).
|
|
545
|
+
if (model && !this._routeAllows(account, model)) continue;
|
|
546
|
+
const priority = account.priority || 0;
|
|
547
|
+
const usage = this._maxUtilization(account, model);
|
|
548
|
+
if (priority < bestPriority ||
|
|
549
|
+
(priority === bestPriority && usage < bestUsage)) {
|
|
550
|
+
bestPriority = priority;
|
|
551
|
+
bestUsage = usage;
|
|
552
|
+
best = account;
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
if (!best) return null;
|
|
556
|
+
|
|
557
|
+
this._nextProbeAt = now + this.probeIntervalMs;
|
|
558
|
+
this.currentIndex = best.index;
|
|
559
|
+
this._beginRamp(best);
|
|
560
|
+
if (best.status === 'throttled') {
|
|
561
|
+
console.log(`[TeamClaude] All accounts unavailable — revalidating throttled "${best.name}" with a live request`);
|
|
562
|
+
} else {
|
|
563
|
+
console.log(`[TeamClaude] All accounts over threshold — probing "${best.name}" to refresh quota`);
|
|
564
|
+
}
|
|
565
|
+
return best;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
_isAvailable(account, model = null, advisorModel = null) {
|
|
569
|
+
if (!account) return false;
|
|
570
|
+
|
|
571
|
+
// Manually disabled accounts are skipped entirely until re-enabled.
|
|
572
|
+
if (account.disabled) return false;
|
|
573
|
+
|
|
574
|
+
// Check rate limit expiry
|
|
575
|
+
if (account.status === 'throttled' && account.rateLimitedUntil) {
|
|
576
|
+
if (Date.now() < account.rateLimitedUntil) return false;
|
|
577
|
+
account.status = 'active';
|
|
578
|
+
account.rateLimitedUntil = null;
|
|
579
|
+
account.throttledAt = null;
|
|
580
|
+
console.log(`[TeamClaude] Account "${account.name}" rate limit expired, marking active`);
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
if (account.status === 'exhausted' || account.status === 'error') return false;
|
|
584
|
+
// Model-scoped: _isNearQuota checks the shared 5h bucket plus only the weekly
|
|
585
|
+
// bucket that governs this model, so a spent Fable/Sonnet bucket bars just
|
|
586
|
+
// that family — the account still serves every other model normally.
|
|
587
|
+
if (this._isNearQuota(account, model)) return false;
|
|
588
|
+
|
|
589
|
+
// Route/ownership restriction: a configured route can pin a model pattern to
|
|
590
|
+
// an exclusive set of accounts; failing that, a per-account `models` claim
|
|
591
|
+
// restricts an owned model to its owners. Either way an account not eligible
|
|
592
|
+
// for this model is skipped so the request never lands somewhere it can't run.
|
|
593
|
+
if (model && !this._routeAllows(account, model)) return false;
|
|
594
|
+
|
|
595
|
+
// An advisor request additionally needs the account to serve the ADVISOR's
|
|
596
|
+
// model: its family bucket must have headroom (the shared buckets were
|
|
597
|
+
// already checked above for the executor) and any route/ownership rule for
|
|
598
|
+
// it must allow this account.
|
|
599
|
+
if (advisorModel) {
|
|
600
|
+
if (this._modelWeeklyExhausted(account, advisorModel)) return false;
|
|
601
|
+
if (!this._routeAllows(account, advisorModel)) return false;
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
return true;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/**
|
|
608
|
+
* The available account that would preempt `account` under the priority rule,
|
|
609
|
+
* or null. A strictly lower priority value wins; within the same tier we stay
|
|
610
|
+
* put, so the common case (every account at the default priority 0) never
|
|
611
|
+
* thrashes. Shared by _select, which enforces it, and eligibility(), which
|
|
612
|
+
* reports it — one predicate so the answer cannot drift from the behaviour.
|
|
613
|
+
*/
|
|
614
|
+
_preemptedBy(account, model = null, advisorModel = null, exclude = null) {
|
|
615
|
+
const pri = account.priority || 0;
|
|
616
|
+
const sw = this.soonestWeekly;
|
|
617
|
+
// Reset-preemption needs both windows known: an unknown candidate must not
|
|
618
|
+
// preempt (it sorts first in _pickBestAvailable purely so a request probes
|
|
619
|
+
// it), and an unknown current account is itself still being probed, so
|
|
620
|
+
// yanking traffic off it would prevent learning its quota (mirrors
|
|
621
|
+
// _switchOnSessionReset's guard).
|
|
622
|
+
const currentReset = sw.enabled ? this._governingWeeklyReset(account, model) : null;
|
|
623
|
+
const poolMs = sw.poolHours * 3600_000;
|
|
624
|
+
return this.accounts.find(a => {
|
|
625
|
+
if (a.index === account.index) return false;
|
|
626
|
+
if (exclude?.has(a.index)) return false;
|
|
627
|
+
if (!this._isAvailable(a, model, advisorModel)) return false;
|
|
628
|
+
const p = a.priority || 0;
|
|
629
|
+
if (p < pri) return true;
|
|
630
|
+
if (p !== pri || currentReset == null) return false;
|
|
631
|
+
const reset = this._governingWeeklyReset(a, model);
|
|
632
|
+
return reset != null && reset < currentReset - poolMs;
|
|
633
|
+
}) || null;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/**
|
|
637
|
+
* Whether a request right now would actually route to an account, with a short
|
|
638
|
+
* reason when it would not. A caller that records a manual choice (the control
|
|
639
|
+
* plane's switch endpoint) needs to report whether that choice will take
|
|
640
|
+
* effect, not merely that it was stored: selection drops the choice on the very
|
|
641
|
+
* next request both when the account cannot serve traffic and when another
|
|
642
|
+
* available account outranks it on priority. Both are asked here through the
|
|
643
|
+
* same helpers _select uses, so the flag cannot promise more than the selector
|
|
644
|
+
* delivers.
|
|
645
|
+
* @returns {{eligible: boolean, reason?: string}}
|
|
646
|
+
*/
|
|
647
|
+
eligibility(accountIndex) {
|
|
648
|
+
const account = this.accounts[accountIndex];
|
|
649
|
+
if (!account) return { eligible: false, reason: 'no such account' };
|
|
650
|
+
// _isAvailable also clears an expired throttle, so the specific reasons below
|
|
651
|
+
// are only consulted once it has actually said no.
|
|
652
|
+
if (!this._isAvailable(account)) {
|
|
653
|
+
if (account.disabled) return { eligible: false, reason: 'disabled' };
|
|
654
|
+
if (account.status === 'error') return { eligible: false, reason: 'in an error state and needs a re-login' };
|
|
655
|
+
if (account.status === 'exhausted') return { eligible: false, reason: 'out of quota' };
|
|
656
|
+
if (account.status === 'throttled') return { eligible: false, reason: 'rate-limited' };
|
|
657
|
+
return { eligible: false, reason: 'at or above the switch threshold' };
|
|
658
|
+
}
|
|
659
|
+
// Healthy, but a higher-priority account preempts it on the next selection.
|
|
660
|
+
// Phrased to read correctly after "<name> is ..." in the caller's message.
|
|
661
|
+
const preemptor = this._preemptedBy(account);
|
|
662
|
+
if (preemptor) {
|
|
663
|
+
const reason = (preemptor.priority || 0) < (account.priority || 0)
|
|
664
|
+
? `outranked by higher-priority account "${preemptor.name}"`
|
|
665
|
+
: `account "${preemptor.name}"'s weekly window resets sooner`;
|
|
666
|
+
return { eligible: false, reason };
|
|
667
|
+
}
|
|
668
|
+
return { eligible: true };
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/** Session-distribution toggle (issue #109), applied live on config reload.
|
|
672
|
+
* Existing session pins survive a toggle: the flag gates only how NEW
|
|
673
|
+
* sessions are routed. */
|
|
674
|
+
setDistributeSessions(enabled) {
|
|
675
|
+
this.distributeSessions = !!enabled;
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
/**
|
|
679
|
+
* Soonest-weekly preference: treat the governing weekly reset as a dynamic
|
|
680
|
+
* priority tier, so the account whose window refreshes soonest is spent
|
|
681
|
+
* first even while the current account is still healthy. Accounts within
|
|
682
|
+
* `poolHours` of the soonest known reset form a pool: selection prefers and
|
|
683
|
+
* (with distributeSessions) balances within it, and the current account is
|
|
684
|
+
* preempted only by one that resets more than `poolHours` sooner — the pool
|
|
685
|
+
* width doubles as the anti-flip-flop epsilon. Called from the constructor
|
|
686
|
+
* and on config reload; passing undefined disables it.
|
|
687
|
+
*/
|
|
688
|
+
setSoonestWeekly(cfg) {
|
|
689
|
+
const c = cfg || {};
|
|
690
|
+
this.soonestWeekly = {
|
|
691
|
+
enabled: !!c.enabled,
|
|
692
|
+
poolHours: Math.max(0, c.poolHours ?? 12),
|
|
693
|
+
};
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* Normalize and store the configurable routing table. A route pins a set of
|
|
698
|
+
* model globs to an exclusive set of accounts (and may override the governing
|
|
699
|
+
* quota bucket). Called from the constructor and on config reload.
|
|
700
|
+
* { name, match: string|string[], accounts?: (name|index)[], bucket? }
|
|
701
|
+
*/
|
|
702
|
+
setRoutes(routes) {
|
|
703
|
+
this.routes = (Array.isArray(routes) ? routes : []).map((r, i) => ({
|
|
704
|
+
name: r.name || `route-${i + 1}`,
|
|
705
|
+
match: (Array.isArray(r.match) ? r.match : [r.match]).filter(g => typeof g === 'string' && g),
|
|
706
|
+
accounts: Array.isArray(r.accounts) ? r.accounts.map(String) : [],
|
|
707
|
+
bucket: r.bucket || null,
|
|
708
|
+
color: r.color || null, // display-only accent for the route's inline marker
|
|
709
|
+
})).filter(r => r.match.length);
|
|
710
|
+
// Drop pins for routes that no longer exist after a reload.
|
|
711
|
+
if (this.routePins?.size) {
|
|
712
|
+
const names = new Set(this.routes.map(r => r.name));
|
|
713
|
+
for (const name of [...this.routePins.keys()]) {
|
|
714
|
+
if (name !== 'fable' && name !== 'sonnet' && !names.has(name)) this.routePins.delete(name);
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
/** The first configured route whose globs match `model`, or null. */
|
|
720
|
+
_routeForModel(model) {
|
|
721
|
+
if (!model || !this.routes?.length) return null;
|
|
722
|
+
return this.routes.find(r => r.match.some(g => modelGlobMatches(g, model))) || null;
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
/** The weekly quota bucket that governs `model` — a matching route's `bucket`
|
|
726
|
+
* override wins, otherwise the model family's default bucket. */
|
|
727
|
+
_weeklyBucketFor(model) {
|
|
728
|
+
const route = this._routeForModel(model);
|
|
729
|
+
return route?.bucket || weeklyBucketForModel(model);
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
/** Whether `account` may serve `model`. A matching route with an `accounts`
|
|
733
|
+
* list is exclusive (only listed accounts, by name or index). With no matching
|
|
734
|
+
* route — or a route that lists no accounts — it falls back to the per-account
|
|
735
|
+
* `models` ownership claim (deprecated — use `routes` instead). */
|
|
736
|
+
_routeAllows(account, model) {
|
|
737
|
+
const route = this._routeForModel(model);
|
|
738
|
+
if (route && route.accounts.length) {
|
|
739
|
+
return route.accounts.includes(account.name) || route.accounts.includes(String(account.index));
|
|
740
|
+
}
|
|
741
|
+
return this._accountOwnsModel(account, model);
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
/** @deprecated Use `routes` with an `accounts` list instead.
|
|
745
|
+
* Returns true if no account claims model ownership, or this account does. */
|
|
746
|
+
_accountOwnsModel(account, model) {
|
|
747
|
+
for (const a of this.accounts) {
|
|
748
|
+
if (a.models && a.models.some(m => modelMatches(m, model))) {
|
|
749
|
+
// Some other account owns this model — this account must own it too.
|
|
750
|
+
return !!(account.models && account.models.some(m => modelMatches(m, model)));
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
return true; // no one claims ownership → any account is fine
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
/**
|
|
757
|
+
* The routing table for display: every configured route plus an ephemeral,
|
|
758
|
+
* auto-created route for each model family that some account meters with its
|
|
759
|
+
* own weekly bucket but no configured route already covers. Auto-created routes
|
|
760
|
+
* carry `autocreated: true` and are never persisted — they simply surface the
|
|
761
|
+
* per-model quota the server already respects. Each route lists the accounts it
|
|
762
|
+
* can use with a live eligibility flag, plus `target`: the one account it would
|
|
763
|
+
* pick right now. Everything here is derived for display and thrown away — the
|
|
764
|
+
* entries are fresh objects, never the stored (persisted) route definitions.
|
|
765
|
+
*/
|
|
766
|
+
getRoutes() {
|
|
767
|
+
const out = this.routes.map(r => ({
|
|
768
|
+
name: r.name, match: r.match, bucket: r.bucket, color: r.color || null, autocreated: false,
|
|
769
|
+
pinned: this._pinnedName(r.name),
|
|
770
|
+
accounts: this._routeAccountsView(r),
|
|
771
|
+
target: this._routeTarget(sampleModelFor(r)),
|
|
772
|
+
}));
|
|
773
|
+
|
|
774
|
+
const detected = [];
|
|
775
|
+
if (this.accounts.some(a => a.quota.unified7dFable != null)) {
|
|
776
|
+
detected.push({ name: 'fable', match: ['*fable*'], sample: 'claude-fable-5' });
|
|
777
|
+
}
|
|
778
|
+
if (this.accounts.some(a => a.quota.unified7dSonnet != null)) {
|
|
779
|
+
detected.push({ name: 'sonnet', match: ['*sonnet*'], sample: 'claude-sonnet-4-6' });
|
|
780
|
+
}
|
|
781
|
+
for (const d of detected) {
|
|
782
|
+
if (this._routeForModel(d.sample)) continue; // already covered by a configured route
|
|
783
|
+
out.push({
|
|
784
|
+
name: d.name, match: d.match, bucket: null, color: null, autocreated: true,
|
|
785
|
+
pinned: this._pinnedName(d.name),
|
|
786
|
+
accounts: this.accounts.map(a => ({ name: a.name, eligible: this._isAvailable(a, d.sample) })),
|
|
787
|
+
target: this._routeTarget(d.sample),
|
|
788
|
+
});
|
|
789
|
+
}
|
|
790
|
+
return out;
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
/** The name of the account a request for `model` would land on right now, or
|
|
794
|
+
* null when nothing can serve it (every candidate disabled, spent or excluded). */
|
|
795
|
+
_routeTarget(model) {
|
|
796
|
+
const idx = this.previewRouteIndex(model);
|
|
797
|
+
return idx == null ? null : (this.accounts[idx]?.name ?? null);
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
/** The name of the account this route is manually pinned to, or null. */
|
|
801
|
+
_pinnedName(routeName) {
|
|
802
|
+
const idx = this.routePins.get(routeName);
|
|
803
|
+
return idx == null ? null : (this.accounts[idx]?.name ?? null);
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
/** Accounts a configured route can use (all accounts when it lists none), each
|
|
807
|
+
* with a live eligibility flag for a representative model of the route. */
|
|
808
|
+
_routeAccountsView(route) {
|
|
809
|
+
const sample = sampleModelFor(route);
|
|
810
|
+
const inRoute = a => !route.accounts.length
|
|
811
|
+
|| route.accounts.includes(a.name) || route.accounts.includes(String(a.index));
|
|
812
|
+
return this.accounts.filter(inRoute).map(a => ({ name: a.name, eligible: this._isAvailable(a, sample) }));
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
/** A representative model id for a route name (configured or auto fable/sonnet),
|
|
816
|
+
* used to test route-allowance when pinning. Null for an unknown route. */
|
|
817
|
+
_routeSample(routeName) {
|
|
818
|
+
const r = this.routes.find(x => x.name === routeName);
|
|
819
|
+
if (r) return r.match[0]?.replace(/\*/g, '') || 'model';
|
|
820
|
+
if (routeName === 'fable') return 'claude-fable-5';
|
|
821
|
+
if (routeName === 'sonnet') return 'claude-sonnet-4-6';
|
|
822
|
+
return null;
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
/**
|
|
826
|
+
* Manually pin a route to an account (ephemeral runtime override). Rejects an
|
|
827
|
+
* account the route's exclusivity/ownership rules disallow. Pinning an account
|
|
828
|
+
* that is merely near-quota/throttled is allowed — it acts as a preference and
|
|
829
|
+
* routing falls back to best-available until the pinned account is eligible.
|
|
830
|
+
* Returns { ok, reason? }.
|
|
831
|
+
*/
|
|
832
|
+
setRoutePin(routeName, accountIndex) {
|
|
833
|
+
const account = this.accounts[accountIndex];
|
|
834
|
+
if (!account) return { ok: false, reason: 'no such account' };
|
|
835
|
+
const sample = this._routeSample(routeName);
|
|
836
|
+
if (sample && !this._routeAllows(account, sample)) {
|
|
837
|
+
return { ok: false, reason: `route "${routeName}" does not allow "${account.name}"` };
|
|
838
|
+
}
|
|
839
|
+
this.routePins.set(routeName, accountIndex);
|
|
840
|
+
return { ok: true };
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
clearRoutePin(routeName) { this.routePins.delete(routeName); }
|
|
844
|
+
|
|
845
|
+
/** The account a route is pinned to, or null. */
|
|
846
|
+
getRoutePin(routeName) {
|
|
847
|
+
const idx = this.routePins.get(routeName);
|
|
848
|
+
return idx == null ? null : (this.accounts[idx] || null);
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/** The manually-pinned account governing `model`, if any: a configured route's
|
|
852
|
+
* pin wins, else an auto fable/sonnet family pin (only when no configured route
|
|
853
|
+
* covers the model). For an advisor request the executor's pin wins (it is the
|
|
854
|
+
* bulk of the spend); the advisor model's pin applies only when nothing pins
|
|
855
|
+
* the executor. Returns null when nothing is pinned for this model. */
|
|
856
|
+
_pinnedAccountForModel(model, advisorModel = null) {
|
|
857
|
+
return this._pinnedFor(model)
|
|
858
|
+
|| (advisorModel ? this._pinnedFor(advisorModel) : null);
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
_pinnedFor(model) {
|
|
862
|
+
if (!model || !this.routePins.size) return null;
|
|
863
|
+
const route = this._routeForModel(model);
|
|
864
|
+
if (route) {
|
|
865
|
+
const idx = this.routePins.get(route.name);
|
|
866
|
+
return idx == null ? null : (this.accounts[idx] || null);
|
|
867
|
+
}
|
|
868
|
+
for (const name of ['fable', 'sonnet']) {
|
|
869
|
+
if (this.routePins.has(name) && modelGlobMatches(`*${name}*`, model)) {
|
|
870
|
+
return this.accounts[this.routePins.get(name)] || null;
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
return null;
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
/**
|
|
877
|
+
* Clear any quota counters whose reset time has passed. Cheap and safe to
|
|
878
|
+
* call frequently (e.g. from the TUI render loop) — once a counter is cleared
|
|
879
|
+
* it stays null until the next upstream response repopulates it, so the
|
|
880
|
+
* "reset" log fires at most once per window.
|
|
881
|
+
* @returns {{changed: boolean, session: boolean}} what was cleared.
|
|
882
|
+
*/
|
|
883
|
+
_clearExpiredQuotas(account) {
|
|
884
|
+
const q = account.quota;
|
|
885
|
+
const now = Date.now();
|
|
886
|
+
let changed = false;
|
|
887
|
+
let session = false;
|
|
888
|
+
|
|
889
|
+
// Clear expired unified quotas
|
|
890
|
+
if (q.unified5h != null && q.unified5hReset && now >= q.unified5hReset) {
|
|
891
|
+
console.log(`[TeamClaude] Account "${account.name}" session quota reset`);
|
|
892
|
+
q.unified5h = null;
|
|
893
|
+
q.unified5hReset = null;
|
|
894
|
+
changed = true;
|
|
895
|
+
session = true;
|
|
896
|
+
}
|
|
897
|
+
if (q.unified7d != null && q.unified7dReset && now >= q.unified7dReset) {
|
|
898
|
+
console.log(`[TeamClaude] Account "${account.name}" weekly quota reset`);
|
|
899
|
+
q.unified7d = null;
|
|
900
|
+
q.unified7dReset = null;
|
|
901
|
+
q.unifiedStatus = null;
|
|
902
|
+
changed = true;
|
|
903
|
+
}
|
|
904
|
+
if (q.unified7dSonnet != null && q.unified7dSonnetReset && now >= q.unified7dSonnetReset) {
|
|
905
|
+
q.unified7dSonnet = null;
|
|
906
|
+
q.unified7dSonnetReset = null;
|
|
907
|
+
changed = true;
|
|
908
|
+
}
|
|
909
|
+
if (q.unified7dFable != null && q.unified7dFableReset && now >= q.unified7dFableReset) {
|
|
910
|
+
q.unified7dFable = null;
|
|
911
|
+
q.unified7dFableReset = null;
|
|
912
|
+
changed = true;
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
// Clear expired standard quotas
|
|
916
|
+
if (q.resetsAt && now >= new Date(q.resetsAt).getTime()) {
|
|
917
|
+
q.tokensRemaining = null;
|
|
918
|
+
q.tokensLimit = null;
|
|
919
|
+
q.requestsRemaining = null;
|
|
920
|
+
q.requestsLimit = null;
|
|
921
|
+
q.resetsAt = null;
|
|
922
|
+
changed = true;
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
return { changed, session };
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
/**
|
|
929
|
+
* Clear expired quotas across all accounts. Called from the display loop and
|
|
930
|
+
* the request path so a window expiry (e.g. the 5-hour session quota) resets
|
|
931
|
+
* the view instantly rather than waiting for the next request.
|
|
932
|
+
*
|
|
933
|
+
* When an account's session quota resets, it may have become the better
|
|
934
|
+
* choice — switch to it if its weekly limit expires sooner than the current
|
|
935
|
+
* account's (and it still has weekly quota), so we spend the quota closest to
|
|
936
|
+
* refreshing first.
|
|
937
|
+
*/
|
|
938
|
+
refreshExpiredQuotas() {
|
|
939
|
+
let changed = false;
|
|
940
|
+
const sessionReset = [];
|
|
941
|
+
for (const account of this.accounts) {
|
|
942
|
+
const r = this._clearExpiredQuotas(account);
|
|
943
|
+
if (r.changed) changed = true;
|
|
944
|
+
if (r.session) sessionReset.push(account);
|
|
945
|
+
}
|
|
946
|
+
if (sessionReset.length) this._switchOnSessionReset(sessionReset);
|
|
947
|
+
return changed;
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
/**
|
|
951
|
+
* Given accounts whose session quota just reset, switch to the one whose
|
|
952
|
+
* weekly limit expires soonest — but only if that is sooner than the current
|
|
953
|
+
* account's weekly limit and the account still has weekly quota to spend.
|
|
954
|
+
*/
|
|
955
|
+
_switchOnSessionReset(candidates) {
|
|
956
|
+
const current = this.accounts[this.currentIndex];
|
|
957
|
+
// Need a known weekly reset on the current account to compare against;
|
|
958
|
+
// if it is unknown we are still probing it, so leave it alone.
|
|
959
|
+
if (!current || current.quota.unified7dReset == null) return;
|
|
960
|
+
|
|
961
|
+
let best = null;
|
|
962
|
+
let bestWeekly = current.quota.unified7dReset;
|
|
963
|
+
for (const acc of candidates) {
|
|
964
|
+
if (acc.index === this.currentIndex) continue;
|
|
965
|
+
if (!this._isAvailable(acc)) continue; // enough session & weekly quota left
|
|
966
|
+
// Don't demote to a lower-priority (higher value) account on a reset.
|
|
967
|
+
if ((acc.priority || 0) > (current.priority || 0)) continue;
|
|
968
|
+
const weekly = acc.quota.unified7dReset;
|
|
969
|
+
if (weekly == null) continue; // need a known weekly to compare
|
|
970
|
+
if (weekly < bestWeekly) {
|
|
971
|
+
bestWeekly = weekly;
|
|
972
|
+
best = acc;
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
if (best) {
|
|
977
|
+
this.currentIndex = best.index;
|
|
978
|
+
this._beginRamp(best);
|
|
979
|
+
console.log(`[TeamClaude] Account "${best.name}" session quota reset and weekly expires sooner — switching to it`);
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
_isNearQuota(account, model = null) {
|
|
984
|
+
const q = account.quota;
|
|
985
|
+
this._clearExpiredQuotas(account);
|
|
986
|
+
|
|
987
|
+
// Shared 5-hour bucket gates every request regardless of model.
|
|
988
|
+
if (q.unified5h != null && q.unified5h >= this.switchThreshold) return true;
|
|
989
|
+
|
|
990
|
+
// Only the weekly bucket that GOVERNS this model is checked: Fable and Sonnet
|
|
991
|
+
// meter their own weekly quota, so a spent Fable bucket must not bar an Opus
|
|
992
|
+
// or Sonnet request (and vice versa). When the family bucket isn't reported
|
|
993
|
+
// (e.g. the plan doesn't expose it), fall back to the shared weekly so an
|
|
994
|
+
// account over its overall cap is still treated as near-quota.
|
|
995
|
+
const weeklyVal = this._governingWeekly(account, model);
|
|
996
|
+
if (weeklyVal != null && weeklyVal >= this.switchThreshold) return true;
|
|
997
|
+
|
|
998
|
+
// Standard quotas (API key accounts)
|
|
999
|
+
if (q.tokensLimit != null && q.tokensRemaining != null) {
|
|
1000
|
+
const used = 1 - (q.tokensRemaining / q.tokensLimit);
|
|
1001
|
+
if (used >= this.switchThreshold) return true;
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
if (q.requestsLimit != null && q.requestsRemaining != null) {
|
|
1005
|
+
const used = 1 - (q.requestsRemaining / q.requestsLimit);
|
|
1006
|
+
if (used >= this.switchThreshold) return true;
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
return false;
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
/**
|
|
1013
|
+
* Pick the best available account by selection order, WITHOUT mutating state:
|
|
1014
|
+
* 1. lowest `priority` value (operator-controlled; default 0, lower = preferred)
|
|
1015
|
+
* 2. then the account with no known weekly limit — using it lets us
|
|
1016
|
+
* discover its quota
|
|
1017
|
+
* 3. then the account whose weekly limit expires soonest: that quota is
|
|
1018
|
+
* closest to refreshing, so spending it first preserves accounts whose
|
|
1019
|
+
* weekly window resets further out.
|
|
1020
|
+
* With all priorities at the default 0, this reduces to the weekly-reset
|
|
1021
|
+
* heuristic. Returns the account or null if none are available.
|
|
1022
|
+
*/
|
|
1023
|
+
_pickBestAvailable(exclude = null, model = null, advisorModel = null) {
|
|
1024
|
+
let best = null;
|
|
1025
|
+
let bestPriority = Infinity;
|
|
1026
|
+
let bestReset = Infinity;
|
|
1027
|
+
|
|
1028
|
+
for (let i = 0; i < this.accounts.length; i++) {
|
|
1029
|
+
const account = this.accounts[i];
|
|
1030
|
+
if (exclude?.has(account.index)) continue;
|
|
1031
|
+
// _isAvailable filters out accounts at/above the switch threshold, so the
|
|
1032
|
+
// soonest-expiring pick only ever lands on an account whose 5-hour quota
|
|
1033
|
+
// is still below 98%.
|
|
1034
|
+
if (!this._isAvailable(account, model, advisorModel)) continue;
|
|
1035
|
+
|
|
1036
|
+
const priority = account.priority || 0;
|
|
1037
|
+
// Rank by the reset of the weekly bucket that governs THIS model (Fable and
|
|
1038
|
+
// Sonnet have their own), so a Fable request spends the account whose Fable
|
|
1039
|
+
// window refreshes soonest while preserving accounts that reset later for
|
|
1040
|
+
// Opus/Sonnet. Unknown reset sorts first so we probe and fill it in.
|
|
1041
|
+
const weeklyReset = this._governingWeeklyReset(account, model) || -Infinity;
|
|
1042
|
+
if (priority < bestPriority ||
|
|
1043
|
+
(priority === bestPriority && weeklyReset < bestReset)) {
|
|
1044
|
+
bestPriority = priority;
|
|
1045
|
+
bestReset = weeklyReset;
|
|
1046
|
+
best = account;
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
return best;
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
/**
|
|
1053
|
+
* Select the active account up front (e.g. on daemon launch, once persisted
|
|
1054
|
+
* quota has been restored) so we start on the highest-priority / soonest-
|
|
1055
|
+
* resetting account instead of blindly on index 0. Mirrors rotation order.
|
|
1056
|
+
* Returns the chosen account, or the existing current one if none are
|
|
1057
|
+
* available (the server still starts; requests 429 until a window resets).
|
|
1058
|
+
*/
|
|
1059
|
+
selectActiveAccount() {
|
|
1060
|
+
this.refreshExpiredQuotas(); // drop any restored windows that already expired
|
|
1061
|
+
const best = this._pickBestAvailable();
|
|
1062
|
+
if (!best) return this.accounts[this.currentIndex] || null;
|
|
1063
|
+
this.currentIndex = best.index;
|
|
1064
|
+
this._beginRamp(best);
|
|
1065
|
+
best.probing = best.quota.unified7dReset == null;
|
|
1066
|
+
const wk = best.quota.unified7d != null
|
|
1067
|
+
? `${(best.quota.unified7d * 100).toFixed(1)}% weekly used`
|
|
1068
|
+
: 'weekly quota unknown';
|
|
1069
|
+
console.log(`[TeamClaude] Starting on account "${best.name}" (priority ${best.priority || 0}, ${wk})`);
|
|
1070
|
+
return best;
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
_selectNext(exclude = null, model = null, advisorModel = null) {
|
|
1074
|
+
const best = this._pickBestAvailable(exclude, model, advisorModel);
|
|
1075
|
+
if (best) {
|
|
1076
|
+
const switched = best.index !== this.currentIndex;
|
|
1077
|
+
this.currentIndex = best.index;
|
|
1078
|
+
// If we switched to an account whose weekly quota is still unknown, flag
|
|
1079
|
+
// it so we re-evaluate once that quota is learned (see updateQuota).
|
|
1080
|
+
best.probing = best.quota.unified7dReset == null;
|
|
1081
|
+
if (switched) {
|
|
1082
|
+
this._beginRamp(best);
|
|
1083
|
+
console.log(`[TeamClaude] Switched to account "${best.name}"`);
|
|
1084
|
+
}
|
|
1085
|
+
return best;
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
// All accounts unavailable — find the one that resets soonest
|
|
1089
|
+
let soonestAccount = null;
|
|
1090
|
+
let soonestTime = Infinity;
|
|
1091
|
+
|
|
1092
|
+
for (const account of this.accounts) {
|
|
1093
|
+
if (exclude?.has(account.index)) continue;
|
|
1094
|
+
// Never resurrect a hard-state account: `disabled` is an operator decision
|
|
1095
|
+
// and `error` means the token is broken (needs re-login). Selecting either
|
|
1096
|
+
// here would send a live request on an account that must not be used and,
|
|
1097
|
+
// below, silently clear its throttle/error state. (Mirrors _isAvailable.)
|
|
1098
|
+
if (account.disabled || account.status === 'error') continue;
|
|
1099
|
+
// A routed/owned model must not fall back to an ineligible account —
|
|
1100
|
+
// neither the executor's nor an advisor's.
|
|
1101
|
+
if (model && !this._routeAllows(account, model)) continue;
|
|
1102
|
+
if (advisorModel && !this._routeAllows(account, advisorModel)) continue;
|
|
1103
|
+
const resetTime = account.rateLimitedUntil
|
|
1104
|
+
|| account.quota.unified5hReset
|
|
1105
|
+
|| account.quota.unified7dReset
|
|
1106
|
+
|| (account.quota.resetsAt ? new Date(account.quota.resetsAt).getTime() : null);
|
|
1107
|
+
|
|
1108
|
+
if (resetTime && resetTime < soonestTime) {
|
|
1109
|
+
soonestTime = resetTime;
|
|
1110
|
+
soonestAccount = account;
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
if (soonestAccount && soonestTime <= Date.now()) {
|
|
1115
|
+
soonestAccount.status = 'active';
|
|
1116
|
+
soonestAccount.rateLimitedUntil = null;
|
|
1117
|
+
this.currentIndex = soonestAccount.index;
|
|
1118
|
+
this._beginRamp(soonestAccount);
|
|
1119
|
+
console.log(`[TeamClaude] Account "${soonestAccount.name}" reset, switching to it`);
|
|
1120
|
+
return soonestAccount;
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
return null;
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
/**
|
|
1127
|
+
* Update an account's quota tracking from upstream response headers.
|
|
1128
|
+
*/
|
|
1129
|
+
updateQuota(accountIndex, headers) {
|
|
1130
|
+
const account = this.accounts[accountIndex];
|
|
1131
|
+
if (!account) return;
|
|
1132
|
+
|
|
1133
|
+
// Unified rate limits (Claude Max)
|
|
1134
|
+
const u5h = parseFloat(headers['anthropic-ratelimit-unified-5h-utilization']);
|
|
1135
|
+
const u7d = parseFloat(headers['anthropic-ratelimit-unified-7d-utilization']);
|
|
1136
|
+
if (!isNaN(u5h)) account.quota.unified5h = u5h;
|
|
1137
|
+
if (!isNaN(u7d)) account.quota.unified7d = u7d;
|
|
1138
|
+
|
|
1139
|
+
const r5h = headers['anthropic-ratelimit-unified-5h-reset'];
|
|
1140
|
+
const r7d = headers['anthropic-ratelimit-unified-7d-reset'];
|
|
1141
|
+
if (r5h) account.quota.unified5hReset = parseInt(r5h, 10) * 1000;
|
|
1142
|
+
if (r7d) account.quota.unified7dReset = parseInt(r7d, 10) * 1000;
|
|
1143
|
+
|
|
1144
|
+
// Model-scoped weekly bucket — surfaced in headers as `7d_oi` ("7-day,
|
|
1145
|
+
// overage included"). On current subscription plans this is the Fable weekly
|
|
1146
|
+
// limit (it correlates with the usage endpoint's Fable-scoped weekly bucket).
|
|
1147
|
+
// Utilization here is already a 0-1 fraction (can exceed 1 when in overage).
|
|
1148
|
+
const u7dOi = parseFloat(headers['anthropic-ratelimit-unified-7d_oi-utilization']);
|
|
1149
|
+
if (!isNaN(u7dOi)) account.quota.unified7dFable = u7dOi;
|
|
1150
|
+
const r7dOi = headers['anthropic-ratelimit-unified-7d_oi-reset'];
|
|
1151
|
+
if (r7dOi) account.quota.unified7dFableReset = parseInt(r7dOi, 10) * 1000;
|
|
1152
|
+
|
|
1153
|
+
// We switched to this account to discover its weekly quota; now that we
|
|
1154
|
+
// know it, flag for re-evaluation so selection can pick the best account.
|
|
1155
|
+
if (account.probing && account.quota.unified7dReset != null) {
|
|
1156
|
+
account.probing = false;
|
|
1157
|
+
account.requalify = true;
|
|
1158
|
+
console.log(`[TeamClaude] Learned weekly quota for "${account.name}", re-evaluating selection`);
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
const uStatus = headers['anthropic-ratelimit-unified-status'];
|
|
1162
|
+
if (uStatus) account.quota.unifiedStatus = uStatus;
|
|
1163
|
+
|
|
1164
|
+
// Standard rate limits (API key accounts)
|
|
1165
|
+
const tokensLimit = parseInt(headers['anthropic-ratelimit-tokens-limit'], 10);
|
|
1166
|
+
const tokensRemaining = parseInt(headers['anthropic-ratelimit-tokens-remaining'], 10);
|
|
1167
|
+
const tokensReset = headers['anthropic-ratelimit-tokens-reset'];
|
|
1168
|
+
const requestsLimit = parseInt(headers['anthropic-ratelimit-requests-limit'], 10);
|
|
1169
|
+
const requestsRemaining = parseInt(headers['anthropic-ratelimit-requests-remaining'], 10);
|
|
1170
|
+
const requestsReset = headers['anthropic-ratelimit-requests-reset'];
|
|
1171
|
+
|
|
1172
|
+
if (!isNaN(tokensLimit)) account.quota.tokensLimit = tokensLimit;
|
|
1173
|
+
if (!isNaN(tokensRemaining)) account.quota.tokensRemaining = tokensRemaining;
|
|
1174
|
+
if (!isNaN(requestsLimit)) account.quota.requestsLimit = requestsLimit;
|
|
1175
|
+
if (!isNaN(requestsRemaining)) account.quota.requestsRemaining = requestsRemaining;
|
|
1176
|
+
|
|
1177
|
+
if (tokensReset) account.quota.resetsAt = tokensReset;
|
|
1178
|
+
else if (requestsReset) account.quota.resetsAt = requestsReset;
|
|
1179
|
+
|
|
1180
|
+
account.usage.totalRequests++;
|
|
1181
|
+
account.usage.lastUsed = new Date().toISOString();
|
|
1182
|
+
|
|
1183
|
+
// Log when approaching quota
|
|
1184
|
+
if (this._isNearQuota(account)) {
|
|
1185
|
+
const pct = account.quota.unified7d != null
|
|
1186
|
+
? (account.quota.unified7d * 100).toFixed(1)
|
|
1187
|
+
: account.quota.tokensLimit
|
|
1188
|
+
? ((1 - account.quota.tokensRemaining / account.quota.tokensLimit) * 100).toFixed(1)
|
|
1189
|
+
: '?';
|
|
1190
|
+
console.log(`[TeamClaude] Account "${account.name}" at ${pct}% usage — will switch on next request`);
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
/**
|
|
1195
|
+
* Update cumulative token usage from response body data.
|
|
1196
|
+
*/
|
|
1197
|
+
updateUsage(accountIndex, inputTokens, outputTokens) {
|
|
1198
|
+
const account = this.accounts[accountIndex];
|
|
1199
|
+
if (!account) return;
|
|
1200
|
+
if (inputTokens) account.usage.totalInputTokens += inputTokens;
|
|
1201
|
+
if (outputTokens) account.usage.totalOutputTokens += outputTokens;
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
/**
|
|
1205
|
+
* Enable or disable an account. A disabled account is skipped by rotation
|
|
1206
|
+
* until re-enabled. Re-enabling also clears a stuck 'error' state (and any
|
|
1207
|
+
* lingering rate-limit hold) so the account is retried immediately.
|
|
1208
|
+
*/
|
|
1209
|
+
setDisabled(accountIndex, disabled) {
|
|
1210
|
+
const account = this.accounts[accountIndex];
|
|
1211
|
+
if (!account) return;
|
|
1212
|
+
account.disabled = disabled;
|
|
1213
|
+
if (!disabled && account.status === 'error') {
|
|
1214
|
+
account.status = 'active';
|
|
1215
|
+
account.rateLimitedUntil = null;
|
|
1216
|
+
console.log(`[TeamClaude] Account "${account.name}" re-enabled — clearing error state`);
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
/**
|
|
1221
|
+
* Apply quota learned from the OAuth usage endpoint (the background probe).
|
|
1222
|
+
* Updates utilization/reset for the 5h, 7d, Sonnet-7d, and Fable-7d buckets WITHOUT
|
|
1223
|
+
* touching usage counters — a probe is not real client traffic.
|
|
1224
|
+
*/
|
|
1225
|
+
applyUsageData(accountIndex, usage) {
|
|
1226
|
+
const account = this.accounts[accountIndex];
|
|
1227
|
+
if (!account || !usage) return;
|
|
1228
|
+
const q = account.quota;
|
|
1229
|
+
|
|
1230
|
+
if (usage.fiveHour) {
|
|
1231
|
+
if (usage.fiveHour.utilization != null) q.unified5h = usage.fiveHour.utilization;
|
|
1232
|
+
if (usage.fiveHour.resetAt != null) q.unified5hReset = usage.fiveHour.resetAt;
|
|
1233
|
+
}
|
|
1234
|
+
if (usage.sevenDay) {
|
|
1235
|
+
if (usage.sevenDay.utilization != null) q.unified7d = usage.sevenDay.utilization;
|
|
1236
|
+
if (usage.sevenDay.resetAt != null) q.unified7dReset = usage.sevenDay.resetAt;
|
|
1237
|
+
}
|
|
1238
|
+
if (usage.sevenDaySonnet) {
|
|
1239
|
+
if (usage.sevenDaySonnet.utilization != null) q.unified7dSonnet = usage.sevenDaySonnet.utilization;
|
|
1240
|
+
if (usage.sevenDaySonnet.resetAt != null) q.unified7dSonnetReset = usage.sevenDaySonnet.resetAt;
|
|
1241
|
+
}
|
|
1242
|
+
if (usage.sevenDayFable) {
|
|
1243
|
+
if (usage.sevenDayFable.utilization != null) q.unified7dFable = usage.sevenDayFable.utilization;
|
|
1244
|
+
if (usage.sevenDayFable.resetAt != null) q.unified7dFableReset = usage.sevenDayFable.resetAt;
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
// If we just learned this account's weekly window while probing, re-evaluate
|
|
1248
|
+
// selection (same path as learning it from a live response).
|
|
1249
|
+
if (account.probing && q.unified7dReset != null) {
|
|
1250
|
+
account.probing = false;
|
|
1251
|
+
account.requalify = true;
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
/**
|
|
1256
|
+
* Mark an account as rate-limited for a given duration.
|
|
1257
|
+
*/
|
|
1258
|
+
markRateLimited(accountIndex, retryAfterSeconds) {
|
|
1259
|
+
const account = this.accounts[accountIndex];
|
|
1260
|
+
if (!account) return;
|
|
1261
|
+
account.status = 'throttled';
|
|
1262
|
+
account.rateLimitedUntil = Date.now() + (retryAfterSeconds * 1000);
|
|
1263
|
+
// Marks when the hold was (re-)armed: a revalidation probe is allowed only
|
|
1264
|
+
// after throttleProbeFloorMs from here, so a probe that 429s again pushes
|
|
1265
|
+
// the next probe out by a full floor rather than hammering upstream.
|
|
1266
|
+
account.throttledAt = Date.now();
|
|
1267
|
+
console.log(`[TeamClaude] Account "${account.name}" rate limited for ${retryAfterSeconds}s`);
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
/**
|
|
1271
|
+
* Clear a rate-limit hold after live proof it no longer binds: any non-429
|
|
1272
|
+
* upstream response on a throttled account (a revalidation probe reaching
|
|
1273
|
+
* here, or a hold armed moments before traffic resumed). No-op otherwise.
|
|
1274
|
+
*/
|
|
1275
|
+
clearRateLimited(accountIndex) {
|
|
1276
|
+
const account = this.accounts[accountIndex];
|
|
1277
|
+
if (!account || account.status !== 'throttled') return;
|
|
1278
|
+
account.status = 'active';
|
|
1279
|
+
account.rateLimitedUntil = null;
|
|
1280
|
+
account.throttledAt = null;
|
|
1281
|
+
console.log(`[TeamClaude] Account "${account.name}" revalidated — rate limit no longer applies, back in rotation`);
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
/**
|
|
1285
|
+
* Ensure an OAuth account's token is fresh, refreshing if needed.
|
|
1286
|
+
* Pass force=true to refresh regardless of expiry (e.g. after a 401).
|
|
1287
|
+
* Concurrent calls for the same account coalesce into a single refresh.
|
|
1288
|
+
*/
|
|
1289
|
+
async ensureTokenFresh(accountIndex, force = false) {
|
|
1290
|
+
const account = this.accounts[accountIndex];
|
|
1291
|
+
if (!account || account.type !== 'oauth' || !account.refreshToken) return;
|
|
1292
|
+
|
|
1293
|
+
if (!force && !isTokenExpiringSoon(account.expiresAt)) return;
|
|
1294
|
+
|
|
1295
|
+
// A forced refresh answers a 401, but 401s arrive in bursts: every request
|
|
1296
|
+
// already in flight when the token went bad comes back rejected, and each
|
|
1297
|
+
// one would force its own refresh. Coalescing only covers refreshes that
|
|
1298
|
+
// OVERLAP — these arrive staggered, so they would rotate the refresh-token
|
|
1299
|
+
// family once per request and make the proxy the very "other holder
|
|
1300
|
+
// rotating the family" that causes this failure in the first place. A 401
|
|
1301
|
+
// for a token minted moments ago is stale news from a request sent before
|
|
1302
|
+
// the refresh landed, so trust the new token and let the caller retry with
|
|
1303
|
+
// it. Only an expiry-driven refresh (force=false) bypasses this — it isn't
|
|
1304
|
+
// reacting to a response and can't stampede.
|
|
1305
|
+
if (force && account._lastRefreshAt !== null
|
|
1306
|
+
&& Date.now() - account._lastRefreshAt < this._forcedRefreshFloorMs) {
|
|
1307
|
+
return;
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
// Coalesce concurrent refreshes
|
|
1311
|
+
if (account._refreshPromise) return account._refreshPromise;
|
|
1312
|
+
|
|
1313
|
+
account._refreshPromise = (async () => {
|
|
1314
|
+
console.log(`[TeamClaude] Refreshing token for account "${account.name}"...`);
|
|
1315
|
+
try {
|
|
1316
|
+
const newTokens = await this._refreshFn(account.refreshToken);
|
|
1317
|
+
account.credential = newTokens.accessToken;
|
|
1318
|
+
account.refreshToken = newTokens.refreshToken;
|
|
1319
|
+
account.expiresAt = newTokens.expiresAt;
|
|
1320
|
+
account._lastRefreshAt = Date.now();
|
|
1321
|
+
console.log(`[TeamClaude] Token refreshed for account "${account.name}"`);
|
|
1322
|
+
this._onTokenRefresh?.(accountIndex, newTokens);
|
|
1323
|
+
} catch (err) {
|
|
1324
|
+
console.error(`[TeamClaude] Token refresh failed for "${account.name}": ${err.message}`);
|
|
1325
|
+
// Reserve 'error' (which drops the account from rotation until re-login)
|
|
1326
|
+
// for a GENUINE auth rejection: the refresh token itself is no longer
|
|
1327
|
+
// valid — revoked, or invalidated by an account/plan migration. A
|
|
1328
|
+
// transient failure (network, 5xx, timeout) must NOT sideline a healthy
|
|
1329
|
+
// account: keep its current token and retry on the next request. This is
|
|
1330
|
+
// what kept accounts wrongly "errored" after a momentary refresh blip.
|
|
1331
|
+
const isAuthRejection = err.status === 400 || err.status === 401 || err.status === 403;
|
|
1332
|
+
if (isAuthRejection) {
|
|
1333
|
+
account.status = 'error';
|
|
1334
|
+
console.error(`[TeamClaude] Account "${account.name}" needs re-login (refresh token rejected) — run: teamclaude login`);
|
|
1335
|
+
}
|
|
1336
|
+
} finally {
|
|
1337
|
+
account._refreshPromise = null;
|
|
1338
|
+
}
|
|
1339
|
+
})();
|
|
1340
|
+
|
|
1341
|
+
return account._refreshPromise;
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
/**
|
|
1345
|
+
* Set a callback to persist refreshed tokens to config.
|
|
1346
|
+
*/
|
|
1347
|
+
onTokenRefresh(callback) {
|
|
1348
|
+
this._onTokenRefresh = callback;
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
/**
|
|
1352
|
+
* Update a specific account's OAuth tokens (e.g. after intercepting a token refresh).
|
|
1353
|
+
*/
|
|
1354
|
+
updateAccountTokens(accountIndex, { accessToken, refreshToken, expiresAt }) {
|
|
1355
|
+
const account = this.accounts[accountIndex];
|
|
1356
|
+
if (!account || account.type !== 'oauth') return;
|
|
1357
|
+
|
|
1358
|
+
account.credential = accessToken;
|
|
1359
|
+
if (refreshToken) account.refreshToken = refreshToken;
|
|
1360
|
+
account.expiresAt = expiresAt;
|
|
1361
|
+
if (account.status === 'error') account.status = 'active';
|
|
1362
|
+
console.log(`[TeamClaude] Updated tokens for account "${account.name}"`);
|
|
1363
|
+
this._onTokenRefresh?.(accountIndex, {
|
|
1364
|
+
accessToken,
|
|
1365
|
+
refreshToken: account.refreshToken,
|
|
1366
|
+
expiresAt: account.expiresAt,
|
|
1367
|
+
});
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
/**
|
|
1371
|
+
* Add a new account at runtime.
|
|
1372
|
+
*/
|
|
1373
|
+
addAccount(acctData) {
|
|
1374
|
+
const index = this.accounts.length;
|
|
1375
|
+
this.accounts.push(makeAccount(acctData, index));
|
|
1376
|
+
return index;
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
/**
|
|
1380
|
+
* Remove an account by index.
|
|
1381
|
+
*/
|
|
1382
|
+
removeAccount(index) {
|
|
1383
|
+
if (index < 0 || index >= this.accounts.length) return;
|
|
1384
|
+
this.accounts.splice(index, 1);
|
|
1385
|
+
this.accounts.forEach((a, i) => a.index = i);
|
|
1386
|
+
if (this.currentIndex >= this.accounts.length) {
|
|
1387
|
+
this.currentIndex = Math.max(0, this.accounts.length - 1);
|
|
1388
|
+
} else if (this.currentIndex > index) {
|
|
1389
|
+
this.currentIndex--;
|
|
1390
|
+
}
|
|
1391
|
+
// Keep route pins pointing at the right account after the index shift: drop a
|
|
1392
|
+
// pin on the removed account, decrement pins that sat above it.
|
|
1393
|
+
for (const [name, idx] of [...this.routePins.entries()]) {
|
|
1394
|
+
if (idx === index) this.routePins.delete(name);
|
|
1395
|
+
else if (idx > index) this.routePins.set(name, idx - 1);
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
/**
|
|
1400
|
+
* Serialize persistable quota state for all accounts (no credentials), keyed
|
|
1401
|
+
* by account identity so it can be matched back after a restart.
|
|
1402
|
+
*/
|
|
1403
|
+
exportQuotaState() {
|
|
1404
|
+
return this.accounts.map(a => {
|
|
1405
|
+
const quota = {};
|
|
1406
|
+
for (const f of PERSISTED_QUOTA_FIELDS) quota[f] = a.quota[f];
|
|
1407
|
+
return { accountUuid: a.accountUuid, orgUuid: a.orgUuid, orgName: a.orgName, name: a.name, quota };
|
|
1408
|
+
});
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
/**
|
|
1412
|
+
* Restore quota learned in a previous run. Matches saved entries to accounts
|
|
1413
|
+
* by identity. Stale windows are not special-cased here — _clearExpiredQuotas
|
|
1414
|
+
* wipes any restored window whose reset time has already passed on first use.
|
|
1415
|
+
*/
|
|
1416
|
+
restoreQuotaState(saved) {
|
|
1417
|
+
if (!Array.isArray(saved)) return;
|
|
1418
|
+
for (const account of this.accounts) {
|
|
1419
|
+
const match = saved.find(s => sameIdentity(s, account));
|
|
1420
|
+
if (!match || !match.quota) continue;
|
|
1421
|
+
for (const f of PERSISTED_QUOTA_FIELDS) {
|
|
1422
|
+
if (match.quota[f] != null) account.quota[f] = match.quota[f];
|
|
1423
|
+
}
|
|
1424
|
+
// We already know this account's weekly window, so it isn't "probing".
|
|
1425
|
+
if (account.quota.unified7dReset != null) account.probing = false;
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
/**
|
|
1430
|
+
* Return a status summary of all accounts (safe to expose, no credentials).
|
|
1431
|
+
*/
|
|
1432
|
+
getStatus() {
|
|
1433
|
+
const sessions = this.sessionTracker.stats();
|
|
1434
|
+
return {
|
|
1435
|
+
currentAccount: this.accounts[this.currentIndex]?.name,
|
|
1436
|
+
switchThreshold: this.switchThreshold,
|
|
1437
|
+
routes: this.getRoutes(),
|
|
1438
|
+
sessions: { ...sessions, distribute: this.distributeSessions },
|
|
1439
|
+
soonestWeekly: { ...this.soonestWeekly },
|
|
1440
|
+
accounts: this.accounts.map(a => ({
|
|
1441
|
+
name: a.name,
|
|
1442
|
+
type: a.type,
|
|
1443
|
+
orgName: a.orgName || null,
|
|
1444
|
+
priority: a.priority || 0,
|
|
1445
|
+
disabled: a.disabled || false,
|
|
1446
|
+
status: a.status,
|
|
1447
|
+
sessions: sessions.perAccount[a.index] || 0,
|
|
1448
|
+
quota: { ...a.quota },
|
|
1449
|
+
usage: { ...a.usage },
|
|
1450
|
+
rateLimitedUntil: a.rateLimitedUntil
|
|
1451
|
+
? new Date(a.rateLimitedUntil).toISOString()
|
|
1452
|
+
: null,
|
|
1453
|
+
pausedUntil: a.pausedUntil && a.pausedUntil > Date.now()
|
|
1454
|
+
? new Date(a.pausedUntil).toISOString()
|
|
1455
|
+
: null,
|
|
1456
|
+
})),
|
|
1457
|
+
};
|
|
1458
|
+
}
|
|
1459
|
+
}
|