ai-cc-router 0.5.1 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/proxy/server.js +9 -0
- package/dist/proxy/token-pool.js +82 -0
- package/package.json +1 -1
package/dist/proxy/server.js
CHANGED
|
@@ -83,6 +83,12 @@ export async function startServer(opts = {}) {
|
|
|
83
83
|
logError(a.id, 0, msg);
|
|
84
84
|
stats.addLog({ ts: Date.now(), accountId: a.id, model: "-", type: "error", details: msg });
|
|
85
85
|
};
|
|
86
|
+
// Surface rate-limit recovery in the dashboard so users see the account
|
|
87
|
+
// rejoin the rotation instead of wondering why it stayed red.
|
|
88
|
+
pool.onCooldownExpired = (a) => {
|
|
89
|
+
const msg = `${a.id} cooldown expired — rate limit cleared`;
|
|
90
|
+
stats.addLog({ ts: Date.now(), accountId: a.id, model: "-", type: "route", details: msg });
|
|
91
|
+
};
|
|
86
92
|
startRefreshLoop(accounts);
|
|
87
93
|
const app = express();
|
|
88
94
|
// ─── Proxy auth middleware ─────────────────────────────────────────────────
|
|
@@ -115,6 +121,9 @@ export async function startServer(opts = {}) {
|
|
|
115
121
|
}
|
|
116
122
|
// ─── Health endpoint (cc-router internal, NOT proxied) ────────────────────
|
|
117
123
|
app.get("/cc-router/health", (_req, res) => {
|
|
124
|
+
// Sweep expired cooldowns on each poll so the dashboard reflects recovery
|
|
125
|
+
// even during idle periods when no /v1 request would trigger getNext().
|
|
126
|
+
pool.sweepExpiredCooldowns();
|
|
118
127
|
res.json({
|
|
119
128
|
status: pool.getHealthy().length > 0 ? "ok" : "degraded",
|
|
120
129
|
mode,
|
package/dist/proxy/token-pool.js
CHANGED
|
@@ -12,6 +12,72 @@ function earliestReset(a) {
|
|
|
12
12
|
return Math.min(r.fiveHourReset, r.sevenDayReset);
|
|
13
13
|
return r.fiveHourReset || r.sevenDayReset || Infinity;
|
|
14
14
|
}
|
|
15
|
+
/**
|
|
16
|
+
* Returns the reset timestamp (seconds) that must pass before the account
|
|
17
|
+
* stops being rate_limited. Prefers the `claim` window (the one Anthropic
|
|
18
|
+
* said was actually limiting); falls back to the earliest non-zero reset.
|
|
19
|
+
* Returns 0 when no reset is known.
|
|
20
|
+
*/
|
|
21
|
+
function limitingReset(a) {
|
|
22
|
+
const r = a.rateLimits;
|
|
23
|
+
if (r.claim === "five_hour" && r.fiveHourReset)
|
|
24
|
+
return r.fiveHourReset;
|
|
25
|
+
if (r.claim === "seven_day" && r.sevenDayReset)
|
|
26
|
+
return r.sevenDayReset;
|
|
27
|
+
const earliest = earliestReset(a);
|
|
28
|
+
return Number.isFinite(earliest) ? earliest : 0;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Roll over any rate-limit window whose reset timestamp has passed.
|
|
32
|
+
*
|
|
33
|
+
* `rateLimits` is a snapshot of Anthropic's response headers — utilization
|
|
34
|
+
* values only refresh when a new response arrives. That creates a stuck
|
|
35
|
+
* state in two scenarios:
|
|
36
|
+
*
|
|
37
|
+
* 1. `status: "rate_limited"` — the pool refuses to route to the account,
|
|
38
|
+
* so no new response ever updates the status.
|
|
39
|
+
* 2. `fiveHourUtil` / `sevenDayUtil` at or above the user cap — `overUserCap`
|
|
40
|
+
* evicts the account from rotation, so the util stays stale at its last
|
|
41
|
+
* recorded value instead of dropping to ~0 when Anthropic's window resets.
|
|
42
|
+
*
|
|
43
|
+
* This sweep resolves both: when `now >= reset` for a window, the util is
|
|
44
|
+
* zeroed and the window's reset timestamp is cleared. When the limiting
|
|
45
|
+
* window expires, `status` flips back to `"allowed"`. The callback fires
|
|
46
|
+
* once per recovery so the dashboard can surface it.
|
|
47
|
+
*/
|
|
48
|
+
function clearExpiredCooldown(a, onExpired) {
|
|
49
|
+
const nowSec = Math.floor(Date.now() / 1000);
|
|
50
|
+
const r = a.rateLimits;
|
|
51
|
+
let changed = false;
|
|
52
|
+
let recovered = false;
|
|
53
|
+
if (r.fiveHourReset > 0 && nowSec >= r.fiveHourReset) {
|
|
54
|
+
r.fiveHourUtil = 0;
|
|
55
|
+
r.fiveHourReset = 0;
|
|
56
|
+
changed = true;
|
|
57
|
+
}
|
|
58
|
+
if (r.sevenDayReset > 0 && nowSec >= r.sevenDayReset) {
|
|
59
|
+
r.sevenDayUtil = 0;
|
|
60
|
+
r.sevenDayReset = 0;
|
|
61
|
+
changed = true;
|
|
62
|
+
}
|
|
63
|
+
// If the account was rate_limited and its claimed window just reset,
|
|
64
|
+
// return it to rotation. If we can't tell which window was limiting
|
|
65
|
+
// (empty claim) but all known windows have rolled over, clear the flag.
|
|
66
|
+
if (r.status === "rate_limited") {
|
|
67
|
+
const stillBlocked = (r.claim === "five_hour" && r.fiveHourReset > 0) ||
|
|
68
|
+
(r.claim === "seven_day" && r.sevenDayReset > 0) ||
|
|
69
|
+
(r.claim === "" && (r.fiveHourReset > 0 || r.sevenDayReset > 0));
|
|
70
|
+
if (!stillBlocked) {
|
|
71
|
+
r.status = "allowed";
|
|
72
|
+
recovered = true;
|
|
73
|
+
changed = true;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
if (changed)
|
|
77
|
+
r.lastUpdated = Date.now();
|
|
78
|
+
if (recovered && onExpired)
|
|
79
|
+
onExpired(a);
|
|
80
|
+
}
|
|
15
81
|
/** True when the account's user-defined caps have been reached. */
|
|
16
82
|
function overUserCap(a) {
|
|
17
83
|
return (a.rateLimits.fiveHourUtil * 100 >= a.sessionLimitPercent ||
|
|
@@ -52,6 +118,10 @@ export class TokenPool {
|
|
|
52
118
|
if (this.accounts.length === 0) {
|
|
53
119
|
throw new EmptyPoolError("token pool is empty — add an account first");
|
|
54
120
|
}
|
|
121
|
+
// Sweep expired cooldowns before filtering — otherwise rate_limited accounts
|
|
122
|
+
// whose reset window has passed would never re-enter rotation.
|
|
123
|
+
for (const a of this.accounts)
|
|
124
|
+
clearExpiredCooldown(a, this.onCooldownExpired);
|
|
55
125
|
const available = this.accounts.filter(a => a.healthy &&
|
|
56
126
|
!a.busy &&
|
|
57
127
|
a.rateLimits.status !== "rate_limited" &&
|
|
@@ -84,6 +154,18 @@ export class TokenPool {
|
|
|
84
154
|
/** Optional listener fired when a request is routed to a capped account
|
|
85
155
|
* because every account in the pool was over its user-configured cap. */
|
|
86
156
|
onCapBypass;
|
|
157
|
+
/** Optional listener fired when a rate-limited account's cooldown expires
|
|
158
|
+
* and it is automatically returned to the rotation. */
|
|
159
|
+
onCooldownExpired;
|
|
160
|
+
/**
|
|
161
|
+
* Sweep the pool for accounts whose rate_limited cooldown has passed and
|
|
162
|
+
* clear the flag in place. Intended for periodic calls from the dashboard
|
|
163
|
+
* poll loop so the UI reflects recovery without waiting for a new request.
|
|
164
|
+
*/
|
|
165
|
+
sweepExpiredCooldowns() {
|
|
166
|
+
for (const a of this.accounts)
|
|
167
|
+
clearExpiredCooldown(a, this.onCooldownExpired);
|
|
168
|
+
}
|
|
87
169
|
getAll() {
|
|
88
170
|
return this.accounts;
|
|
89
171
|
}
|