@timo972/cc-router 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -0
- package/Dockerfile +42 -0
- package/LICENSE +21 -0
- package/README.md +716 -0
- package/accounts.example.json +25 -0
- package/dist/cli/cmd-accounts.js +248 -0
- package/dist/cli/cmd-client.js +612 -0
- package/dist/cli/cmd-configure.js +145 -0
- package/dist/cli/cmd-docker.js +140 -0
- package/dist/cli/cmd-logs.js +85 -0
- package/dist/cli/cmd-models.js +125 -0
- package/dist/cli/cmd-service.js +193 -0
- package/dist/cli/cmd-setup.js +501 -0
- package/dist/cli/cmd-start.js +318 -0
- package/dist/cli/cmd-status.js +177 -0
- package/dist/cli/cmd-stop.js +100 -0
- package/dist/cli/cmd-telemetry.js +58 -0
- package/dist/cli/cmd-update.js +37 -0
- package/dist/cli/index.js +59 -0
- package/dist/config/manager.js +262 -0
- package/dist/config/paths.js +21 -0
- package/dist/config/telemetry.js +64 -0
- package/dist/daemon/launcher.js +163 -0
- package/dist/daemon/pid.js +98 -0
- package/dist/daemon/service.js +260 -0
- package/dist/interceptor/mitmproxy-manager.js +616 -0
- package/dist/protocol/anthropic-to-openai.js +51 -0
- package/dist/protocol/anthropic-types.js +1 -0
- package/dist/protocol/model-ref.js +36 -0
- package/dist/protocol/model-routing-config.js +30 -0
- package/dist/protocol/openai-response-to-anthropic.js +20 -0
- package/dist/protocol/openai-responses-types.js +1 -0
- package/dist/protocol/openai-stream-to-anthropic.js +75 -0
- package/dist/protocol/openai-to-anthropic.js +61 -0
- package/dist/protocol/sse.js +17 -0
- package/dist/providers/model-discovery.js +71 -0
- package/dist/providers/openai/account-pool.js +11 -0
- package/dist/providers/openai/account-record.js +33 -0
- package/dist/providers/openai/codex-transport.js +36 -0
- package/dist/providers/openai/device-oauth.js +116 -0
- package/dist/providers/openai/token-refresher.js +56 -0
- package/dist/providers/route-selector.js +8 -0
- package/dist/providers/types.js +1 -0
- package/dist/proxy/account-deletion.js +44 -0
- package/dist/proxy/anthropic-proxy.js +26 -0
- package/dist/proxy/anthropic-routing.js +90 -0
- package/dist/proxy/lease-lifecycle.js +68 -0
- package/dist/proxy/logger.js +39 -0
- package/dist/proxy/messages-cross-route.js +179 -0
- package/dist/proxy/models-server.js +150 -0
- package/dist/proxy/provider-routing.js +14 -0
- package/dist/proxy/responses-server.js +91 -0
- package/dist/proxy/server.js +875 -0
- package/dist/proxy/session-router.js +171 -0
- package/dist/proxy/stats.js +25 -0
- package/dist/proxy/stream-lifecycle.js +83 -0
- package/dist/proxy/token-pool.js +407 -0
- package/dist/proxy/token-refresher.js +209 -0
- package/dist/proxy/types.js +29 -0
- package/dist/ui/Dashboard.js +640 -0
- package/dist/ui/accountsApi.js +48 -0
- package/dist/ui/modelsApi.js +47 -0
- package/dist/utils/claude-config.js +185 -0
- package/dist/utils/codex-config.js +62 -0
- package/dist/utils/network.js +16 -0
- package/dist/utils/platform.js +13 -0
- package/dist/utils/self-update.js +239 -0
- package/dist/utils/telemetry.js +88 -0
- package/dist/utils/token-extractor.js +95 -0
- package/dist/utils/token-validator.js +26 -0
- package/docker-compose.yml +63 -0
- package/litellm-config.yaml +44 -0
- package/package.json +69 -0
- package/src/interceptor/addon.py +78 -0
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
const DEFAULT_TTL_MS = 60 * 60 * 1000;
|
|
2
|
+
const DEFAULT_MAX_ENTRIES = 10_000;
|
|
3
|
+
const MAX_SESSION_ID_BYTES = 256;
|
|
4
|
+
/**
|
|
5
|
+
* Normalize Claude Code's session header without retaining malformed or
|
|
6
|
+
* unexpectedly large values. Arrays are rejected rather than choosing one
|
|
7
|
+
* value because a session must have exactly one unambiguous identity.
|
|
8
|
+
*/
|
|
9
|
+
export function normalizeSessionId(value) {
|
|
10
|
+
if (typeof value !== "string")
|
|
11
|
+
return undefined;
|
|
12
|
+
const normalized = value.trim();
|
|
13
|
+
if (normalized.length === 0)
|
|
14
|
+
return undefined;
|
|
15
|
+
if (Buffer.byteLength(normalized, "utf8") > MAX_SESSION_ID_BYTES)
|
|
16
|
+
return undefined;
|
|
17
|
+
return normalized;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* In-memory session affinity for Claude requests. Bindings and their counts
|
|
21
|
+
* are intentionally process-local and are never exposed for persistence or
|
|
22
|
+
* diagnostics.
|
|
23
|
+
*/
|
|
24
|
+
export class SessionRouter {
|
|
25
|
+
pool;
|
|
26
|
+
bindings = new Map();
|
|
27
|
+
activeSessionCounts = new Map();
|
|
28
|
+
now;
|
|
29
|
+
ttlMs;
|
|
30
|
+
maxEntries;
|
|
31
|
+
nextBindingGeneration = 1;
|
|
32
|
+
constructor(pool, options = {}) {
|
|
33
|
+
this.pool = pool;
|
|
34
|
+
this.now = options.now ?? Date.now;
|
|
35
|
+
this.ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
|
|
36
|
+
this.maxEntries = options.maxEntries ?? DEFAULT_MAX_ENTRIES;
|
|
37
|
+
if (!Number.isFinite(this.ttlMs) || this.ttlMs <= 0) {
|
|
38
|
+
throw new RangeError("session binding TTL must be a positive finite number");
|
|
39
|
+
}
|
|
40
|
+
if (!Number.isInteger(this.maxEntries) || this.maxEntries <= 0) {
|
|
41
|
+
throw new RangeError("session binding capacity must be a positive integer");
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
acquire(sessionHeader) {
|
|
45
|
+
const sessionId = normalizeSessionId(sessionHeader);
|
|
46
|
+
const now = this.now();
|
|
47
|
+
this.sweepExpiredBindings(now);
|
|
48
|
+
if (!sessionId) {
|
|
49
|
+
return this.wrapUnscoped(this.pool.acquireBest(this.activeSessionCounts));
|
|
50
|
+
}
|
|
51
|
+
const existing = this.bindings.get(sessionId);
|
|
52
|
+
if (existing) {
|
|
53
|
+
const stickyLease = this.pool.tryAcquire(existing.accountId);
|
|
54
|
+
if (stickyLease) {
|
|
55
|
+
existing.lastSeen = now;
|
|
56
|
+
return this.wrapScoped(stickyLease, "sticky", sessionId, existing.generation);
|
|
57
|
+
}
|
|
58
|
+
this.removeBinding(sessionId);
|
|
59
|
+
}
|
|
60
|
+
const lease = this.pool.acquireBest(this.activeSessionCounts);
|
|
61
|
+
const binding = this.insertBinding(sessionId, lease.account.id, now);
|
|
62
|
+
return this.wrapScoped(lease, existing ? "failover" : "new-session", sessionId, binding.generation);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Remove a binding only if it still belongs to the expected account. This
|
|
66
|
+
* protects a new failover binding from a late response on the old account.
|
|
67
|
+
*/
|
|
68
|
+
invalidate(sessionHeader, expectedAccountId, expectedGeneration) {
|
|
69
|
+
const sessionId = normalizeSessionId(sessionHeader);
|
|
70
|
+
if (!sessionId)
|
|
71
|
+
return false;
|
|
72
|
+
const binding = this.bindings.get(sessionId);
|
|
73
|
+
if (!binding)
|
|
74
|
+
return false;
|
|
75
|
+
if (expectedAccountId !== undefined && binding.accountId !== expectedAccountId) {
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
if (expectedGeneration !== undefined && binding.generation !== expectedGeneration) {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
return this.removeBinding(sessionId);
|
|
82
|
+
}
|
|
83
|
+
invalidateAccount(accountId) {
|
|
84
|
+
let removed = 0;
|
|
85
|
+
for (const [sessionId, binding] of this.bindings) {
|
|
86
|
+
if (binding.accountId !== accountId)
|
|
87
|
+
continue;
|
|
88
|
+
if (this.removeBinding(sessionId))
|
|
89
|
+
removed++;
|
|
90
|
+
}
|
|
91
|
+
return removed;
|
|
92
|
+
}
|
|
93
|
+
getActiveSessionCount(accountId) {
|
|
94
|
+
this.sweepExpiredBindings(this.now());
|
|
95
|
+
return this.getRawActiveSessionCount(accountId);
|
|
96
|
+
}
|
|
97
|
+
getBindingCount() {
|
|
98
|
+
this.sweepExpiredBindings(this.now());
|
|
99
|
+
return this.bindings.size;
|
|
100
|
+
}
|
|
101
|
+
/** Sweep once and expose only aggregate account IDs/counts. */
|
|
102
|
+
getActiveSessionCountsSnapshot() {
|
|
103
|
+
this.sweepExpiredBindings(this.now());
|
|
104
|
+
return new Map(this.activeSessionCounts);
|
|
105
|
+
}
|
|
106
|
+
wrapUnscoped(lease) {
|
|
107
|
+
return {
|
|
108
|
+
account: lease.account,
|
|
109
|
+
fallback: lease.fallback,
|
|
110
|
+
release: lease.release,
|
|
111
|
+
reason: "unscoped",
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
wrapScoped(lease, reason, sessionId, bindingGeneration) {
|
|
115
|
+
return {
|
|
116
|
+
account: lease.account,
|
|
117
|
+
fallback: lease.fallback,
|
|
118
|
+
release: lease.release,
|
|
119
|
+
reason,
|
|
120
|
+
sessionId,
|
|
121
|
+
bindingGeneration,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
insertBinding(sessionId, accountId, lastSeen) {
|
|
125
|
+
if (this.bindings.size >= this.maxEntries)
|
|
126
|
+
this.evictLeastRecentlyUsed();
|
|
127
|
+
const binding = {
|
|
128
|
+
accountId,
|
|
129
|
+
lastSeen,
|
|
130
|
+
generation: this.nextBindingGeneration++,
|
|
131
|
+
};
|
|
132
|
+
this.bindings.set(sessionId, binding);
|
|
133
|
+
this.activeSessionCounts.set(accountId, this.getRawActiveSessionCount(accountId) + 1);
|
|
134
|
+
return binding;
|
|
135
|
+
}
|
|
136
|
+
removeBinding(sessionId) {
|
|
137
|
+
const binding = this.bindings.get(sessionId);
|
|
138
|
+
if (!binding)
|
|
139
|
+
return false;
|
|
140
|
+
this.bindings.delete(sessionId);
|
|
141
|
+
const remaining = this.getRawActiveSessionCount(binding.accountId) - 1;
|
|
142
|
+
if (remaining <= 0)
|
|
143
|
+
this.activeSessionCounts.delete(binding.accountId);
|
|
144
|
+
else
|
|
145
|
+
this.activeSessionCounts.set(binding.accountId, remaining);
|
|
146
|
+
return true;
|
|
147
|
+
}
|
|
148
|
+
getRawActiveSessionCount(accountId) {
|
|
149
|
+
return this.activeSessionCounts.get(accountId) ?? 0;
|
|
150
|
+
}
|
|
151
|
+
sweepExpiredBindings(now) {
|
|
152
|
+
for (const [sessionId, binding] of this.bindings) {
|
|
153
|
+
if (now - binding.lastSeen >= this.ttlMs)
|
|
154
|
+
this.removeBinding(sessionId);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
evictLeastRecentlyUsed() {
|
|
158
|
+
let oldestSessionId;
|
|
159
|
+
let oldestLastSeen = Infinity;
|
|
160
|
+
// Map iteration order is insertion order, so equal timestamps retain the
|
|
161
|
+
// first entry as the deterministic eviction candidate.
|
|
162
|
+
for (const [sessionId, binding] of this.bindings) {
|
|
163
|
+
if (binding.lastSeen < oldestLastSeen) {
|
|
164
|
+
oldestSessionId = sessionId;
|
|
165
|
+
oldestLastSeen = binding.lastSeen;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (oldestSessionId !== undefined)
|
|
169
|
+
this.removeBinding(oldestSessionId);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
const MAX_LOG_ENTRIES = 100;
|
|
2
|
+
class ProxyStats {
|
|
3
|
+
totalRequests = 0;
|
|
4
|
+
totalErrors = 0;
|
|
5
|
+
totalRefreshes = 0;
|
|
6
|
+
totalCacheReadTokens = 0;
|
|
7
|
+
totalCacheCreationTokens = 0;
|
|
8
|
+
totalInputTokens = 0;
|
|
9
|
+
totalOutputTokens = 0;
|
|
10
|
+
startTime = Date.now();
|
|
11
|
+
logs = [];
|
|
12
|
+
addLog(entry) {
|
|
13
|
+
this.logs.push(entry);
|
|
14
|
+
if (this.logs.length > MAX_LOG_ENTRIES)
|
|
15
|
+
this.logs.shift();
|
|
16
|
+
}
|
|
17
|
+
getRecentLogs(n = 20) {
|
|
18
|
+
return [...this.logs].reverse().slice(0, n);
|
|
19
|
+
}
|
|
20
|
+
getUptimeSeconds() {
|
|
21
|
+
return Math.round((Date.now() - this.startTime) / 1000);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
// Singleton — shared across server and health endpoint
|
|
25
|
+
export const stats = new ProxyStats();
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
export const MAX_RETAINED_SSE_LINE_BYTES = 64 * 1024;
|
|
2
|
+
export function createStreamLifecycleTracker(startedAt, inspectSse, now = Date.now) {
|
|
3
|
+
const state = {
|
|
4
|
+
sawMessageStop: false,
|
|
5
|
+
upstreamEnd: false,
|
|
6
|
+
upstreamAborted: false,
|
|
7
|
+
upstreamClose: false,
|
|
8
|
+
downstreamFinish: false,
|
|
9
|
+
downstreamClose: false,
|
|
10
|
+
};
|
|
11
|
+
let lineBuffer = Buffer.alloc(0);
|
|
12
|
+
let discardingOversizedLine = false;
|
|
13
|
+
const clearParserState = () => {
|
|
14
|
+
lineBuffer = Buffer.alloc(0);
|
|
15
|
+
discardingOversizedLine = false;
|
|
16
|
+
};
|
|
17
|
+
const inspectLine = (line) => {
|
|
18
|
+
const text = line.toString("utf8");
|
|
19
|
+
if (!text.startsWith("data: "))
|
|
20
|
+
return;
|
|
21
|
+
try {
|
|
22
|
+
const event = JSON.parse(text.slice(6));
|
|
23
|
+
if (event.type === "message_stop") {
|
|
24
|
+
state.sawMessageStop = true;
|
|
25
|
+
clearParserState();
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
// Complete non-JSON data lines are irrelevant to terminal tracking.
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
const terminal = () => {
|
|
33
|
+
clearParserState();
|
|
34
|
+
state.bodyDurationMs = Math.max(0, now() - startedAt);
|
|
35
|
+
};
|
|
36
|
+
return {
|
|
37
|
+
state,
|
|
38
|
+
observeChunk(chunk) {
|
|
39
|
+
state.lastByteAt = now();
|
|
40
|
+
if (!inspectSse || state.sawMessageStop)
|
|
41
|
+
return;
|
|
42
|
+
let offset = 0;
|
|
43
|
+
while (offset < chunk.length) {
|
|
44
|
+
const newlineAt = chunk.indexOf(0x0a, offset);
|
|
45
|
+
if (discardingOversizedLine) {
|
|
46
|
+
if (newlineAt === -1)
|
|
47
|
+
return;
|
|
48
|
+
discardingOversizedLine = false;
|
|
49
|
+
offset = newlineAt + 1;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
const fragmentEnd = newlineAt === -1 ? chunk.length : newlineAt;
|
|
53
|
+
const fragment = chunk.subarray(offset, fragmentEnd);
|
|
54
|
+
const retainedBytes = lineBuffer.length + fragment.length;
|
|
55
|
+
if (retainedBytes > MAX_RETAINED_SSE_LINE_BYTES) {
|
|
56
|
+
lineBuffer = Buffer.alloc(0);
|
|
57
|
+
discardingOversizedLine = newlineAt === -1;
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
lineBuffer = lineBuffer.length === 0
|
|
61
|
+
? Buffer.from(fragment)
|
|
62
|
+
: Buffer.concat([lineBuffer, fragment], retainedBytes);
|
|
63
|
+
if (newlineAt !== -1) {
|
|
64
|
+
inspectLine(lineBuffer);
|
|
65
|
+
lineBuffer = Buffer.alloc(0);
|
|
66
|
+
if (state.sawMessageStop)
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (newlineAt === -1)
|
|
71
|
+
return;
|
|
72
|
+
offset = newlineAt + 1;
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
attach(upstream, downstream) {
|
|
76
|
+
upstream.once("end", () => { state.upstreamEnd = true; terminal(); });
|
|
77
|
+
upstream.once("aborted", () => { state.upstreamAborted = true; terminal(); });
|
|
78
|
+
upstream.once("close", () => { state.upstreamClose = true; terminal(); });
|
|
79
|
+
downstream.once("finish", () => { state.downstreamFinish = true; terminal(); });
|
|
80
|
+
downstream.once("close", () => { state.downstreamClose = true; terminal(); });
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
}
|
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
import { DEFAULT_RATE_LIMITS, ACCOUNT_USER_DEFAULTS, clampPercent } from "./types.js";
|
|
2
|
+
export class EmptyPoolError extends Error {
|
|
3
|
+
constructor(message) {
|
|
4
|
+
super(message);
|
|
5
|
+
this.name = "EmptyPoolError";
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
/** Returns the earliest non-zero reset timestamp (seconds) for an account. */
|
|
9
|
+
function earliestReset(a) {
|
|
10
|
+
const r = a.rateLimits;
|
|
11
|
+
if (r.fiveHourReset && r.sevenDayReset)
|
|
12
|
+
return Math.min(r.fiveHourReset, r.sevenDayReset);
|
|
13
|
+
return r.fiveHourReset || r.sevenDayReset || Infinity;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Returns the reset timestamp (seconds) that must pass before the account
|
|
17
|
+
* stops being rate_limited. Prefers the `claim` window (the one Anthropic
|
|
18
|
+
* said was actually limiting); falls back to the earliest non-zero reset.
|
|
19
|
+
* Returns 0 when no reset is known.
|
|
20
|
+
*/
|
|
21
|
+
function limitingReset(a) {
|
|
22
|
+
const r = a.rateLimits;
|
|
23
|
+
if (r.claim === "five_hour" && r.fiveHourReset)
|
|
24
|
+
return r.fiveHourReset;
|
|
25
|
+
if (r.claim === "seven_day" && r.sevenDayReset)
|
|
26
|
+
return r.sevenDayReset;
|
|
27
|
+
const earliest = earliestReset(a);
|
|
28
|
+
return Number.isFinite(earliest) ? earliest : 0;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Roll over any rate-limit window whose reset timestamp has passed.
|
|
32
|
+
*
|
|
33
|
+
* `rateLimits` is a snapshot of Anthropic's response headers — utilization
|
|
34
|
+
* values only refresh when a new response arrives. That creates a stuck
|
|
35
|
+
* state in two scenarios:
|
|
36
|
+
*
|
|
37
|
+
* 1. `status: "rate_limited"` — the pool refuses to route to the account,
|
|
38
|
+
* so no new response ever updates the status.
|
|
39
|
+
* 2. `fiveHourUtil` / `sevenDayUtil` at or above the user cap — `overUserCap`
|
|
40
|
+
* evicts the account from rotation, so the util stays stale at its last
|
|
41
|
+
* recorded value instead of dropping to ~0 when Anthropic's window resets.
|
|
42
|
+
*
|
|
43
|
+
* This sweep resolves both: when `now >= reset` for a window, the util is
|
|
44
|
+
* zeroed and the window's reset timestamp is cleared. When the limiting
|
|
45
|
+
* window expires, `status` flips back to `"allowed"`. The callback fires
|
|
46
|
+
* once per recovery so the dashboard can surface it.
|
|
47
|
+
*/
|
|
48
|
+
function clearExpiredRateLimitWindows(a, nowMs, onExpired) {
|
|
49
|
+
const nowSec = Math.floor(nowMs / 1000);
|
|
50
|
+
const r = a.rateLimits;
|
|
51
|
+
let changed = false;
|
|
52
|
+
let recovered = false;
|
|
53
|
+
if (r.fiveHourReset > 0 && nowSec >= r.fiveHourReset) {
|
|
54
|
+
r.fiveHourUtil = 0;
|
|
55
|
+
r.fiveHourReset = 0;
|
|
56
|
+
changed = true;
|
|
57
|
+
}
|
|
58
|
+
if (r.sevenDayReset > 0 && nowSec >= r.sevenDayReset) {
|
|
59
|
+
r.sevenDayUtil = 0;
|
|
60
|
+
r.sevenDayReset = 0;
|
|
61
|
+
changed = true;
|
|
62
|
+
}
|
|
63
|
+
// If the account was rate_limited and its claimed window just reset,
|
|
64
|
+
// return it to rotation. If we can't tell which window was limiting
|
|
65
|
+
// (empty claim) but all known windows have rolled over, clear the flag.
|
|
66
|
+
if (r.status === "rate_limited") {
|
|
67
|
+
const stillBlocked = (r.claim === "five_hour" && r.fiveHourReset > 0) ||
|
|
68
|
+
(r.claim === "seven_day" && r.sevenDayReset > 0) ||
|
|
69
|
+
(r.claim === "" && (r.fiveHourReset > 0 || r.sevenDayReset > 0));
|
|
70
|
+
if (!stillBlocked) {
|
|
71
|
+
r.status = "allowed";
|
|
72
|
+
recovered = true;
|
|
73
|
+
changed = true;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
if (changed)
|
|
77
|
+
r.lastUpdated = nowMs;
|
|
78
|
+
if (recovered && onExpired)
|
|
79
|
+
onExpired(a);
|
|
80
|
+
}
|
|
81
|
+
/** True when the account's user-defined caps have been reached. */
|
|
82
|
+
function overUserCap(a) {
|
|
83
|
+
return (a.rateLimits.fiveHourUtil * 100 >= a.sessionLimitPercent ||
|
|
84
|
+
a.rateLimits.sevenDayUtil * 100 >= a.weeklyLimitPercent);
|
|
85
|
+
}
|
|
86
|
+
/** Filter out accounts the user has taken out of the rotation. */
|
|
87
|
+
function isUsable(a) {
|
|
88
|
+
return a.enabled && !overUserCap(a);
|
|
89
|
+
}
|
|
90
|
+
export class TokenPool {
|
|
91
|
+
accounts;
|
|
92
|
+
inFlight = new Map();
|
|
93
|
+
cooldownUntil = new Map();
|
|
94
|
+
now;
|
|
95
|
+
currentIndex = 0;
|
|
96
|
+
constructor(accounts, options = {}) {
|
|
97
|
+
this.accounts = accounts;
|
|
98
|
+
this.now = options.now ?? Date.now;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Compatibility wrapper for request sites that do not yet retain leases.
|
|
102
|
+
* Selection considers accounts that are:
|
|
103
|
+
* • healthy
|
|
104
|
+
* • not busy
|
|
105
|
+
* • not rate-limited by Anthropic
|
|
106
|
+
* • enabled (user toggle)
|
|
107
|
+
* • under the user-configured 5h/7d caps
|
|
108
|
+
*
|
|
109
|
+
* Fallback chain when nothing is available:
|
|
110
|
+
* 1. Any healthy+usable (enabled & under caps) account — pick earliest reset.
|
|
111
|
+
* 2. Any healthy account — pick earliest reset. This intentionally ignores
|
|
112
|
+
* user caps when every option is capped; limits are advisory, not a hard
|
|
113
|
+
* ban that would leave Claude Code with no working account. The fallback
|
|
114
|
+
* is logged via the optional onCapBypass callback so the dashboard can
|
|
115
|
+
* surface it instead of silently exceeding the cap.
|
|
116
|
+
* 3. Any account as a last resort (only if every account is unhealthy).
|
|
117
|
+
*
|
|
118
|
+
* Fallback sets prefer the lowest in-flight load, then the earliest reset.
|
|
119
|
+
*
|
|
120
|
+
* Throws `EmptyPoolError` when there are no accounts at all — callers in
|
|
121
|
+
* the request path should map this to a 503. The DELETE endpoint guards
|
|
122
|
+
* against this state by refusing to remove the last account.
|
|
123
|
+
*/
|
|
124
|
+
getNext() {
|
|
125
|
+
const lease = this.acquireBest(new Map());
|
|
126
|
+
lease.release();
|
|
127
|
+
return lease.account;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Acquire the best eligible account using load, session affinity pressure,
|
|
131
|
+
* rate-limit headroom, and a rotating tie-break, in that order.
|
|
132
|
+
*/
|
|
133
|
+
acquireBest(activeSessions) {
|
|
134
|
+
if (this.accounts.length === 0) {
|
|
135
|
+
throw new EmptyPoolError("token pool is empty — add an account first");
|
|
136
|
+
}
|
|
137
|
+
this.sweepExpiredCooldowns();
|
|
138
|
+
const eligible = this.accounts.filter(account => this.isEligibleWithoutSweep(account));
|
|
139
|
+
if (eligible.length > 0) {
|
|
140
|
+
const account = this.selectEligible(eligible, activeSessions);
|
|
141
|
+
this.advanceCursor(account);
|
|
142
|
+
return this.createLease(account, false);
|
|
143
|
+
}
|
|
144
|
+
const healthyUsable = this.accounts.filter(account => account.healthy && isUsable(account));
|
|
145
|
+
const healthy = this.accounts.filter(account => account.healthy);
|
|
146
|
+
const fallbackCandidates = healthyUsable.length > 0
|
|
147
|
+
? healthyUsable
|
|
148
|
+
: healthy.length > 0
|
|
149
|
+
? healthy
|
|
150
|
+
: this.accounts;
|
|
151
|
+
const account = this.selectFallback(fallbackCandidates);
|
|
152
|
+
this.advanceCursor(account);
|
|
153
|
+
if (overUserCap(account))
|
|
154
|
+
this.onCapBypass?.(account);
|
|
155
|
+
return this.createLease(account, true);
|
|
156
|
+
}
|
|
157
|
+
/** Acquire a specific account for an existing sticky session. */
|
|
158
|
+
tryAcquire(accountId) {
|
|
159
|
+
const account = this.findById(accountId);
|
|
160
|
+
if (!account)
|
|
161
|
+
return null;
|
|
162
|
+
clearExpiredRateLimitWindows(account, this.now(), this.onCooldownExpired);
|
|
163
|
+
if (!this.isEligibleWithoutSweep(account))
|
|
164
|
+
return null;
|
|
165
|
+
return this.createLease(account, false);
|
|
166
|
+
}
|
|
167
|
+
isEligible(accountId) {
|
|
168
|
+
const account = this.findById(accountId);
|
|
169
|
+
if (!account)
|
|
170
|
+
return false;
|
|
171
|
+
clearExpiredRateLimitWindows(account, this.now(), this.onCooldownExpired);
|
|
172
|
+
return this.isEligibleWithoutSweep(account);
|
|
173
|
+
}
|
|
174
|
+
getInFlight(accountId) {
|
|
175
|
+
return this.inFlight.get(accountId) ?? 0;
|
|
176
|
+
}
|
|
177
|
+
setCooldown(accountId, durationMs) {
|
|
178
|
+
const account = this.findById(accountId);
|
|
179
|
+
if (!account)
|
|
180
|
+
return;
|
|
181
|
+
this.setCooldownForAccount(account, durationMs);
|
|
182
|
+
}
|
|
183
|
+
/** Apply cooldown only to the exact account incarnation that was routed. */
|
|
184
|
+
setCooldownForAccount(account, durationMs) {
|
|
185
|
+
if (this.findById(account.id) !== account)
|
|
186
|
+
return;
|
|
187
|
+
if (!Number.isFinite(durationMs) || durationMs <= 0)
|
|
188
|
+
return;
|
|
189
|
+
const proposedExpiry = this.now() + durationMs;
|
|
190
|
+
const existingExpiry = this.cooldownUntil.get(account.id) ?? 0;
|
|
191
|
+
this.cooldownUntil.set(account.id, Math.max(existingExpiry, proposedExpiry));
|
|
192
|
+
}
|
|
193
|
+
isCoolingDown(accountId) {
|
|
194
|
+
const until = this.cooldownUntil.get(accountId);
|
|
195
|
+
if (until === undefined)
|
|
196
|
+
return false;
|
|
197
|
+
if (this.now() < until)
|
|
198
|
+
return true;
|
|
199
|
+
this.cooldownUntil.delete(accountId);
|
|
200
|
+
return false;
|
|
201
|
+
}
|
|
202
|
+
isEligibleWithoutSweep(account) {
|
|
203
|
+
return account.healthy &&
|
|
204
|
+
!account.busy &&
|
|
205
|
+
!this.isCoolingDown(account.id) &&
|
|
206
|
+
account.rateLimits.status !== "rate_limited" &&
|
|
207
|
+
isUsable(account);
|
|
208
|
+
}
|
|
209
|
+
selectEligible(candidates, activeSessions) {
|
|
210
|
+
return candidates.reduce((best, account) => {
|
|
211
|
+
const comparison = this.compareTuple([
|
|
212
|
+
this.getInFlight(account.id),
|
|
213
|
+
activeSessions.get(account.id) ?? 0,
|
|
214
|
+
this.headroomScore(account),
|
|
215
|
+
this.circularDistance(account),
|
|
216
|
+
], [
|
|
217
|
+
this.getInFlight(best.id),
|
|
218
|
+
activeSessions.get(best.id) ?? 0,
|
|
219
|
+
this.headroomScore(best),
|
|
220
|
+
this.circularDistance(best),
|
|
221
|
+
]);
|
|
222
|
+
return comparison < 0 ? account : best;
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
selectFallback(candidates) {
|
|
226
|
+
return candidates.reduce((best, account) => {
|
|
227
|
+
const comparison = this.compareTuple([this.getInFlight(account.id), earliestReset(account), this.circularDistance(account)], [this.getInFlight(best.id), earliestReset(best), this.circularDistance(best)]);
|
|
228
|
+
return comparison < 0 ? account : best;
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
headroomScore(account) {
|
|
232
|
+
const fiveHourCap = account.sessionLimitPercent / 100;
|
|
233
|
+
const sevenDayCap = account.weeklyLimitPercent / 100;
|
|
234
|
+
const fiveHourUtil = Number.isFinite(account.rateLimits.fiveHourUtil)
|
|
235
|
+
? Math.max(0, account.rateLimits.fiveHourUtil)
|
|
236
|
+
: 0;
|
|
237
|
+
const sevenDayUtil = Number.isFinite(account.rateLimits.sevenDayUtil)
|
|
238
|
+
? Math.max(0, account.rateLimits.sevenDayUtil)
|
|
239
|
+
: 0;
|
|
240
|
+
return Math.max(fiveHourUtil / fiveHourCap, sevenDayUtil / sevenDayCap);
|
|
241
|
+
}
|
|
242
|
+
circularDistance(account) {
|
|
243
|
+
const index = this.accounts.indexOf(account);
|
|
244
|
+
return (index - this.currentIndex + this.accounts.length) % this.accounts.length;
|
|
245
|
+
}
|
|
246
|
+
compareTuple(left, right) {
|
|
247
|
+
for (let i = 0; i < left.length; i++) {
|
|
248
|
+
if (left[i] !== right[i])
|
|
249
|
+
return left[i] - right[i];
|
|
250
|
+
}
|
|
251
|
+
return 0;
|
|
252
|
+
}
|
|
253
|
+
advanceCursor(account) {
|
|
254
|
+
const index = this.accounts.indexOf(account);
|
|
255
|
+
this.currentIndex = (index + 1) % this.accounts.length;
|
|
256
|
+
}
|
|
257
|
+
createLease(account, fallback) {
|
|
258
|
+
this.inFlight.set(account.id, this.getInFlight(account.id) + 1);
|
|
259
|
+
account.requestCount++;
|
|
260
|
+
account.lastUsed = this.now();
|
|
261
|
+
let released = false;
|
|
262
|
+
return {
|
|
263
|
+
account,
|
|
264
|
+
fallback,
|
|
265
|
+
release: () => {
|
|
266
|
+
if (released)
|
|
267
|
+
return;
|
|
268
|
+
released = true;
|
|
269
|
+
// IDs may be reused after an account is removed. A lease belongs to
|
|
270
|
+
// the exact Account instance it acquired and must never decrement a
|
|
271
|
+
// replacement account's load counter.
|
|
272
|
+
if (this.findById(account.id) !== account)
|
|
273
|
+
return;
|
|
274
|
+
const remaining = Math.max(0, this.getInFlight(account.id) - 1);
|
|
275
|
+
if (remaining === 0)
|
|
276
|
+
this.inFlight.delete(account.id);
|
|
277
|
+
else
|
|
278
|
+
this.inFlight.set(account.id, remaining);
|
|
279
|
+
},
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
/** Optional listener fired when a request is routed to a capped account
|
|
283
|
+
* because every account in the pool was over its user-configured cap. */
|
|
284
|
+
onCapBypass;
|
|
285
|
+
/** Optional listener fired when a rate-limited account's cooldown expires
|
|
286
|
+
* and it is automatically returned to the rotation. */
|
|
287
|
+
onCooldownExpired;
|
|
288
|
+
/**
|
|
289
|
+
* Sweep the pool for accounts whose rate_limited cooldown has passed and
|
|
290
|
+
* clear the flag in place. Intended for periodic calls from the dashboard
|
|
291
|
+
* poll loop so the UI reflects recovery without waiting for a new request.
|
|
292
|
+
*/
|
|
293
|
+
sweepExpiredCooldowns() {
|
|
294
|
+
const now = this.now();
|
|
295
|
+
for (const a of this.accounts) {
|
|
296
|
+
clearExpiredRateLimitWindows(a, now, this.onCooldownExpired);
|
|
297
|
+
this.isCoolingDown(a.id);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
getAll() {
|
|
301
|
+
return this.accounts;
|
|
302
|
+
}
|
|
303
|
+
getHealthy() {
|
|
304
|
+
return this.accounts.filter(a => a.healthy);
|
|
305
|
+
}
|
|
306
|
+
getStats() {
|
|
307
|
+
return this.accounts.map(a => ({
|
|
308
|
+
id: a.id,
|
|
309
|
+
healthy: a.healthy,
|
|
310
|
+
busy: a.busy,
|
|
311
|
+
inFlightRequests: this.getInFlight(a.id),
|
|
312
|
+
coolingDown: this.isCoolingDown(a.id),
|
|
313
|
+
requestCount: a.requestCount,
|
|
314
|
+
errorCount: a.errorCount,
|
|
315
|
+
expiresInMs: a.tokens.expiresAt - Date.now(),
|
|
316
|
+
lastUsedMs: a.lastUsed,
|
|
317
|
+
lastRefreshMs: a.lastRefresh,
|
|
318
|
+
rateLimits: a.rateLimits,
|
|
319
|
+
enabled: a.enabled,
|
|
320
|
+
sessionLimitPercent: a.sessionLimitPercent,
|
|
321
|
+
weeklyLimitPercent: a.weeklyLimitPercent,
|
|
322
|
+
}));
|
|
323
|
+
}
|
|
324
|
+
// ─── Mutation API (used by the authenticated HTTP endpoints) ───────────────
|
|
325
|
+
findById(id) {
|
|
326
|
+
return this.accounts.find(a => a.id === id) ?? null;
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Apply a partial update to an account's user-controlled fields.
|
|
330
|
+
* Only `enabled`, `sessionLimitPercent`, and `weeklyLimitPercent` are
|
|
331
|
+
* touched — token fields are never accepted via this API.
|
|
332
|
+
* Returns the updated account, or null if the id was not found.
|
|
333
|
+
*/
|
|
334
|
+
updateAccount(id, patch) {
|
|
335
|
+
const a = this.findById(id);
|
|
336
|
+
if (!a)
|
|
337
|
+
return null;
|
|
338
|
+
if (patch.enabled !== undefined)
|
|
339
|
+
a.enabled = !!patch.enabled;
|
|
340
|
+
if (patch.sessionLimitPercent !== undefined) {
|
|
341
|
+
a.sessionLimitPercent = clampPercent(patch.sessionLimitPercent);
|
|
342
|
+
}
|
|
343
|
+
if (patch.weeklyLimitPercent !== undefined) {
|
|
344
|
+
a.weeklyLimitPercent = clampPercent(patch.weeklyLimitPercent);
|
|
345
|
+
}
|
|
346
|
+
return a;
|
|
347
|
+
}
|
|
348
|
+
/**
|
|
349
|
+
* Append a new account built from a persisted AccountRecord.
|
|
350
|
+
* Rejects duplicates by id — callers should pre-check with findById().
|
|
351
|
+
*/
|
|
352
|
+
addAccount(record) {
|
|
353
|
+
if (this.findById(record.id)) {
|
|
354
|
+
throw new Error(`Account "${record.id}" already exists`);
|
|
355
|
+
}
|
|
356
|
+
const account = {
|
|
357
|
+
id: record.id,
|
|
358
|
+
tokens: {
|
|
359
|
+
accessToken: record.accessToken,
|
|
360
|
+
refreshToken: record.refreshToken,
|
|
361
|
+
expiresAt: record.expiresAt,
|
|
362
|
+
scopes: record.scopes ?? ["user:inference", "user:profile"],
|
|
363
|
+
},
|
|
364
|
+
healthy: true,
|
|
365
|
+
busy: false,
|
|
366
|
+
requestCount: 0,
|
|
367
|
+
errorCount: 0,
|
|
368
|
+
lastUsed: 0,
|
|
369
|
+
lastRefresh: 0,
|
|
370
|
+
consecutiveErrors: 0,
|
|
371
|
+
rateLimits: { ...DEFAULT_RATE_LIMITS },
|
|
372
|
+
enabled: record.enabled !== false,
|
|
373
|
+
sessionLimitPercent: record.sessionLimitPercent !== undefined
|
|
374
|
+
? clampPercent(record.sessionLimitPercent)
|
|
375
|
+
: ACCOUNT_USER_DEFAULTS.sessionLimitPercent,
|
|
376
|
+
weeklyLimitPercent: record.weeklyLimitPercent !== undefined
|
|
377
|
+
? clampPercent(record.weeklyLimitPercent)
|
|
378
|
+
: ACCOUNT_USER_DEFAULTS.weeklyLimitPercent,
|
|
379
|
+
};
|
|
380
|
+
this.accounts.push(account);
|
|
381
|
+
return account;
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* Remove an account by id. Returns true if something was removed.
|
|
385
|
+
*
|
|
386
|
+
* CRITICAL: mutates `this.accounts` IN PLACE via splice() rather than
|
|
387
|
+
* reassigning it. The server passes the same array reference to
|
|
388
|
+
* `startRefreshLoop()` at startup; reassigning would desynchronize the
|
|
389
|
+
* refresh loop from the pool, and the loop's `saveAccounts(accounts)` call
|
|
390
|
+
* would later resurrect the deleted account on disk.
|
|
391
|
+
*/
|
|
392
|
+
removeAccount(id) {
|
|
393
|
+
const idx = this.accounts.findIndex(a => a.id === id);
|
|
394
|
+
if (idx === -1)
|
|
395
|
+
return false;
|
|
396
|
+
this.accounts.splice(idx, 1);
|
|
397
|
+
this.inFlight.delete(id);
|
|
398
|
+
this.cooldownUntil.delete(id);
|
|
399
|
+
if (this.accounts.length > 0) {
|
|
400
|
+
this.currentIndex = this.currentIndex % this.accounts.length;
|
|
401
|
+
}
|
|
402
|
+
else {
|
|
403
|
+
this.currentIndex = 0;
|
|
404
|
+
}
|
|
405
|
+
return true;
|
|
406
|
+
}
|
|
407
|
+
}
|