@goodandready/dsh-key-rotation 0.8.77 → 0.8.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/lib/client-src/00-header.js +40 -0
  3. package/lib/client-src/10-locales.js +585 -0
  4. package/lib/client-src/20-hooks.js +55 -0
  5. package/lib/client-src/30-utils.js +185 -0
  6. package/lib/client-src/40-components.js +346 -0
  7. package/lib/client-src/50-card-main.js +647 -0
  8. package/lib/client-src/51-card-pool-mutators.js +501 -0
  9. package/lib/client-src/52-card-pool-rows.js +590 -0
  10. package/lib/client-src/53-card-dialogs-and-render.js +568 -0
  11. package/lib/client-src/60-header-chip.js +137 -0
  12. package/lib/client-src/90-apply.js +69 -0
  13. package/lib/config-compat.js +18 -1
  14. package/lib/http-bridge.js +2 -0
  15. package/lib/index.js +7 -1
  16. package/lib/key-state-machine.js +267 -0
  17. package/lib/model-catalog.js +296 -0
  18. package/lib/ops-balance.js +2 -147
  19. package/lib/ops-config-io.js +2 -456
  20. package/lib/ops-keys.js +2 -567
  21. package/lib/ops-metrics.js +2 -92
  22. package/lib/ops-paths.js +2 -24
  23. package/lib/ops-quick-action.js +2 -168
  24. package/lib/ops-status.js +2 -269
  25. package/lib/ops-telemetry.js +2 -276
  26. package/lib/ops-test.js +2 -255
  27. package/lib/ops-webhook.js +2 -218
  28. package/lib/pool-selection.js +36 -13
  29. package/lib/pool-state.js +3 -0
  30. package/lib/pool.js +10 -15
  31. package/lib/quota-runway.js +222 -0
  32. package/lib/routes/balance.js +147 -0
  33. package/lib/routes/config.js +456 -0
  34. package/lib/routes/guard.js +13 -0
  35. package/lib/routes/index.js +43 -0
  36. package/lib/routes/keys.js +567 -0
  37. package/lib/routes/metrics.js +92 -0
  38. package/lib/routes/paths.js +24 -0
  39. package/lib/routes/quick-action.js +168 -0
  40. package/lib/routes/status.js +269 -0
  41. package/lib/routes/telemetry.js +276 -0
  42. package/lib/routes/testing.js +255 -0
  43. package/lib/routes/webhook.js +218 -0
  44. package/lib/routes-ops.js +2 -29
  45. package/lib/stream-metrics.js +13 -5
  46. package/lib/transports/anthropic.js +125 -0
  47. package/lib/transports/index.js +23 -0
  48. package/lib/transports/openai.js +93 -0
  49. package/lib/transports/sse.js +160 -0
  50. package/lib/transports/tokens.js +124 -0
  51. package/package.json +15 -10
@@ -1,218 +1,2 @@
1
- import crypto from 'node:crypto';
2
- // lib/ops-webhook.js — webhook interactive action route (#312 split from routes-ops.js, #388 telegram secret-token & answerCallbackQuery).
3
- import {
4
- json,
5
- readJson,
6
- } from './http-bridge.js';
7
- import { bestEffort } from './best-effort.js';
8
- import { WEBHOOK_ACTION_PATH } from './ops-paths.js';
9
- import { resolveAction } from './webhook.js';
10
-
11
- function constantTimeEqual(a, b) {
12
- if (typeof a !== 'string' || typeof b !== 'string') return false;
13
- const aBuf = Buffer.from(a);
14
- const bBuf = Buffer.from(b);
15
- return aBuf.length === bBuf.length && crypto.timingSafeEqual(aBuf, bBuf);
16
- }
17
-
18
- function getHeader(headers, name) {
19
- if (!headers || typeof headers !== 'object') return '';
20
- const target = name.toLowerCase();
21
- for (const [k, v] of Object.entries(headers)) {
22
- if (k.toLowerCase() === target) return String(v ?? '');
23
- }
24
- return '';
25
- }
26
-
27
- /**
28
- * @param {object} ctx cordis context
29
- * @param {object} deps live dependencies from apply()
30
- */
31
- export function registerWebhookActionRoute(ctx, deps) {
32
- const {
33
- buildRuntime,
34
- poolState,
35
- setRotationDisabled,
36
- circuitBreaker,
37
- } = deps;
38
-
39
- ctx.effect(() => ctx.webServer.register({
40
- kind: 'exact',
41
- path: WEBHOOK_ACTION_PATH,
42
- handler: async (req, res) => {
43
- if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
44
- const runtime = buildRuntime();
45
- const expected = runtime.webhookActionToken;
46
- if (!expected) { json(res, 503, { error: { code: 'no-token', message: 'dsh-key-rotation: webhookActionToken is not configured' } }); return; }
47
-
48
- const authHeader = getHeader(req.headers, 'authorization');
49
- const secretHeader = getHeader(req.headers, 'x-telegram-bot-api-secret-token');
50
- const bearerMatch = constantTimeEqual(authHeader, `Bearer ${expected}`);
51
- const secretMatch = constantTimeEqual(secretHeader, expected);
52
- if (!bearerMatch && !secretMatch) {
53
- json(res, 401, { error: { code: 'unauthorized', message: 'dsh-key-rotation: bad webhook action token' } });
54
- return;
55
- }
56
-
57
- let body;
58
- try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
59
-
60
- // #222 / #388: Telegram setWebhook registration helper
61
- if (typeof body?.setWebhook === 'object' && body.setWebhook) {
62
- const botToken = typeof body.setWebhook.botToken === 'string' ? body.setWebhook.botToken : '';
63
- if (!botToken) { json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: setWebhook.botToken required' } }); return; }
64
- const TELEGRAM_BOT_TOKEN_RE = /^[0-9]{5,16}:[a-zA-Z0-9_-]{20,50}$/;
65
- if (!TELEGRAM_BOT_TOKEN_RE.test(botToken)) {
66
- json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: invalid telegram bot token format' } });
67
- return;
68
- }
69
- const rawUrl = typeof body.setWebhook.url === 'string' ? body.setWebhook.url.trim() : '';
70
- if (!rawUrl || !rawUrl.startsWith('https://')) {
71
- json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: explicit public https url required in setWebhook.url' } });
72
- return;
73
- }
74
- try {
75
- const hookRes = await fetch(`https://api.telegram.org/bot${botToken}/setWebhook`, {
76
- method: 'POST',
77
- headers: { 'content-type': 'application/json' },
78
- body: JSON.stringify({
79
- url: rawUrl,
80
- secret_token: expected,
81
- allowed_updates: ['callback_query'],
82
- }),
83
- signal: AbortSignal.timeout(10000),
84
- });
85
- const hookData = await hookRes.json().catch(() => ({}));
86
- json(res, 200, { ok: hookRes.ok, url: rawUrl, secret_token_set: true, telegram: hookData });
87
- } catch (e) {
88
- json(res, 502, { error: { code: 'telegram-failed', message: String(e?.message ?? e) } });
89
- }
90
- return;
91
- }
92
-
93
- const callbackQueryId = typeof body?.callback_query?.id === 'string'
94
- ? body.callback_query.id
95
- : (typeof body?.callback_query?.id === 'number' ? String(body.callback_query.id) : '');
96
- const botToken = typeof body?.botToken === 'string' ? body.botToken : '';
97
-
98
- const respond = async (statusCode, payload, answerText) => {
99
- if (callbackQueryId) {
100
- payload.method = 'answerCallbackQuery';
101
- payload.callback_query_id = callbackQueryId;
102
- payload.text = answerText || payload.error?.message || 'OK';
103
- if (botToken) {
104
- await bestEffort('telegram-answerCallbackQuery', () => fetch(`https://api.telegram.org/bot${botToken}/answerCallbackQuery`, {
105
- method: 'POST',
106
- headers: { 'content-type': 'application/json' },
107
- body: JSON.stringify({ callback_query_id: callbackQueryId, text: payload.text }),
108
- signal: AbortSignal.timeout(5000),
109
- }), ctx?.logger);
110
- }
111
- }
112
- json(res, statusCode, payload);
113
- };
114
-
115
- // Accept callback payloads from formatInteractive (Telegram/Discord/Slack) or plain {action}
116
- let action = typeof body?.action === 'string' ? body.action : '';
117
- if (!action && typeof body?.data === 'string') {
118
- try { action = String(JSON.parse(body.data)?.id ?? ''); } catch { action = ''; }
119
- }
120
- if (!action && typeof body?.callback_data === 'string') {
121
- try { action = String(JSON.parse(body.callback_data)?.id ?? ''); } catch { action = ''; }
122
- }
123
- // #222 / #388: Telegram update envelope {update_id, callback_query:{id, data}}
124
- if (!action && typeof body?.callback_query?.data === 'string') {
125
- try {
126
- const parsed = JSON.parse(body.callback_query.data);
127
- action = typeof parsed?.id === 'string' ? parsed.id : String(parsed ?? '');
128
- } catch {
129
- action = body.callback_query.data;
130
- }
131
- }
132
- action = resolveAction(action);
133
- if (!action) {
134
- await respond(400, { error: { code: 'bad-action', message: 'dsh-key-rotation: no action in payload' } }, 'No action in payload');
135
- return;
136
- }
137
- const provider = action.startsWith('pause-') || action.startsWith('reset-') ? action.replace(/^(pause|reset)-/, '') : '';
138
- try {
139
- if (action === 'disable-rotation') {
140
- setRotationDisabled(true);
141
- (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.('[dsh-key-rotation] rotation DISABLED via webhook action');
142
- await respond(200, { ok: true, action }, 'Rotation disabled');
143
- return;
144
- }
145
- if (action === 'enable-rotation') {
146
- setRotationDisabled(false);
147
- await respond(200, { ok: true, action }, 'Rotation enabled');
148
- return;
149
- }
150
- if (action.startsWith('pause-') || action.startsWith('reset-')) {
151
- const st = poolState.get(provider);
152
- if (!st) {
153
- await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
154
- return;
155
- }
156
- if (action.startsWith('pause-')) {
157
- const until = Date.now() + 3600000; // 1h pause
158
- let found = false;
159
- for (const [key, s] of poolState.entries()) {
160
- if (key === provider || key.startsWith(provider + '::')) {
161
- found = true;
162
- for (const ref of (s.failedUntil ? [...s.failedUntil.keys()] : [])) {
163
- s.failedUntil.set(ref, Math.max(s.failedUntil.get(ref) ?? 0, until));
164
- }
165
- }
166
- }
167
- for (const p of buildRuntime().poolByRef.values()) {
168
- if (p.provider === provider || p.base === provider || p.base?.startsWith(provider + '::')) {
169
- found = true;
170
- for (const ref of p.refs) p.state?.failedUntil?.set(ref, Math.max(p.state?.failedUntil?.get(ref) ?? 0, until));
171
- }
172
- }
173
- if (!found) {
174
- await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
175
- return;
176
- }
177
- (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] pool ${provider} PAUSED 1h via webhook action`);
178
- await respond(200, { ok: true, action, provider, until }, `Pool ${provider} paused 1h`);
179
- return;
180
- }
181
- let found = false;
182
- let cleared = 0;
183
- for (const [key, s] of poolState.entries()) {
184
- if (key === provider || key.startsWith(provider + '::')) {
185
- found = true;
186
- cleared += s.failedUntil ? s.failedUntil.size : 0;
187
- s.failedUntil?.clear();
188
- s.failCounts?.clear();
189
- s.authFailCounts?.clear();
190
- s.brokenUntil?.clear();
191
- s.revokedRefs?.clear();
192
- s.tokenUsage?.clear();
193
- s.switches = 0;
194
- s.lastReason = undefined;
195
- s.lastSwitchAt = undefined;
196
- }
197
- }
198
- if (!found) {
199
- await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
200
- return;
201
- }
202
- let circuitReset = false;
203
- const br = circuitBreaker ?? buildRuntime().breaker;
204
- if (br) {
205
- if (typeof br.reset === 'function') { br.reset(provider); circuitReset = true; }
206
- else if (typeof br.onSuccess === 'function') { br.onSuccess(provider); circuitReset = true; }
207
- }
208
- (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] pool ${provider} RESET via webhook action (circuitReset=${circuitReset})`);
209
- await respond(200, { ok: true, action, provider, cleared, circuitReset }, `Pool ${provider} reset`);
210
- return;
211
- }
212
- await respond(400, { error: { code: 'unknown-action', message: `dsh-key-rotation: unknown action '${action}'` } }, `Unknown action: ${action}`);
213
- } catch (e) {
214
- await respond(500, { error: { code: 'action-failed', message: String(e?.message ?? e) } }, 'Action failed');
215
- }
216
- },
217
- }), 'dsh-key-rotation: webhook-action');
218
- }
1
+ // lib/ops-webhook.js — Re-export facade for lib/routes/ (#522).
2
+ export * from "./routes/webhook.js";
@@ -1,4 +1,4 @@
1
- // lib/pool-selection.js — pool selection, sticky sessions, attempt sorting and budget metrics (#512, #548).
1
+ // lib/pool-selection.js — pool selection, sticky sessions, attempt sorting and budget metrics (#512, #548, #549).
2
2
  // Extracted from lib/pool.js to enforce < 600 line modularity standard.
3
3
 
4
4
  import {
@@ -8,11 +8,18 @@ import {
8
8
  SESSION_TTL_MS,
9
9
  } from './session-router.js';
10
10
 
11
+ import {
12
+ QuotaRunway,
13
+ defaultQuotaRunway,
14
+ } from './quota-runway.js';
15
+
11
16
  export {
12
17
  SessionRouter,
13
18
  defaultSessionRouter,
14
19
  MAX_SESSIONS,
15
20
  SESSION_TTL_MS,
21
+ QuotaRunway,
22
+ defaultQuotaRunway,
16
23
  };
17
24
 
18
25
  export const MAX_SESSION_STICKY_ENTRIES = MAX_SESSIONS;
@@ -103,11 +110,13 @@ export function selectPool(modelPoolByProvider, providerToPool, provider, model)
103
110
 
104
111
  /**
105
112
  * Sort candidate refs according to pool routing strategy.
113
+ * Preemptively deprioritizes keys projected to exhaust their quota before reset (#549).
106
114
  * Returns a new array; does not mutate input.
107
115
  */
108
116
  export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
109
117
  if (!Array.isArray(refs) || refs.length <= 1) return (refs ?? []).slice();
110
- const list = refs.slice();
118
+ let list = refs.slice();
119
+
111
120
  if (strategy === 'sticky') {
112
121
  const router = getOrCreateSessionRouter(deps.pool);
113
122
  let chosenKey = null;
@@ -133,19 +142,16 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
133
142
  list.unshift(last);
134
143
  }
135
144
  }
136
- return list;
137
- }
138
- if (strategy === 'tier-priority') {
145
+ } else if (strategy === 'tier-priority') {
139
146
  const tiers = deps.tiers || (deps.pool && deps.pool.tiers) || {};
140
- return list.sort((a, b) => {
147
+ list = list.sort((a, b) => {
141
148
  const ta = (tiers[a] && Number(tiers[a]) >= 1) ? Number(tiers[a]) : 1;
142
149
  const tb = (tiers[b] && Number(tiers[b]) >= 1) ? Number(tiers[b]) : 1;
143
150
  return ta - tb;
144
151
  });
145
- }
146
- if (strategy === 'least-loaded' && deps.concurrencyTracker && typeof deps.concurrencyTracker.getActive === 'function') {
152
+ } else if (strategy === 'least-loaded' && deps.concurrencyTracker && typeof deps.concurrencyTracker.getActive === 'function') {
147
153
  const weights = deps.weights || (deps.pool && deps.pool.weightsMap) || {};
148
- return list.sort((a, b) => {
154
+ list = list.sort((a, b) => {
149
155
  const ca = deps.concurrencyTracker.getActive(a) ?? 0;
150
156
  const cb = deps.concurrencyTracker.getActive(b) ?? 0;
151
157
  const wa = (weights[a] && Number(weights[a]) > 0) ? Number(weights[a]) : 1;
@@ -155,9 +161,8 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
155
161
  if (scoreA !== scoreB) return scoreA - scoreB;
156
162
  return ca - cb;
157
163
  });
158
- }
159
- if (strategy === 'lowest-latency' && deps.latencyHistogram && typeof deps.latencyHistogram.snapshot === 'function') {
160
- return list.sort((a, b) => {
164
+ } else if (strategy === 'lowest-latency' && deps.latencyHistogram && typeof deps.latencyHistogram.snapshot === 'function') {
165
+ list = list.sort((a, b) => {
161
166
  const sa = deps.latencyHistogram.snapshot(a);
162
167
  const sb = deps.latencyHistogram.snapshot(b);
163
168
  const la = (sa && Number.isFinite(sa.p95)) ? sa.p95 : ((sa && Number.isFinite(sa.avg)) ? sa.avg : 500);
@@ -165,6 +170,25 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
165
170
  return la - lb;
166
171
  });
167
172
  }
173
+
174
+ // Preemptively deprioritize keys projected to exhaust before quota window reset (#549)
175
+ const runway = deps.quotaRunway ?? deps.pool?.state?.quotaRunway;
176
+ if (runway && typeof runway.isExhaustingBeforeReset === 'function' && list.length > 1) {
177
+ const now = typeof deps.now === 'function' ? deps.now() : (Number(deps.now) || Date.now());
178
+ const healthy = [];
179
+ const exhausting = [];
180
+ for (const ref of list) {
181
+ if (runway.isExhaustingBeforeReset(ref, now)) {
182
+ exhausting.push(ref);
183
+ } else {
184
+ healthy.push(ref);
185
+ }
186
+ }
187
+ if (healthy.length > 0 && exhausting.length > 0) {
188
+ return healthy.concat(exhausting);
189
+ }
190
+ }
191
+
168
192
  return list;
169
193
  }
170
194
 
@@ -221,4 +245,3 @@ export function budgetVerdict(spend, budget) {
221
245
  const ratio = spend / budget;
222
246
  return { spend, budget, ratio, warn: ratio >= 0.8, exceeded: ratio >= 1 };
223
247
  }
224
-
package/lib/pool-state.js CHANGED
@@ -1,5 +1,7 @@
1
1
  // Shared runtime shape for fresh and persisted pools. Hydrate in place so cached
2
2
  // runtime pools, persistence and timers continue to observe the same object.
3
+ import { KeyStateMachine } from './key-state-machine.js';
4
+
3
5
  const MAP_FIELDS = [
4
6
  'failedUntil', 'failCounts', 'authFailCounts', 'brokenUntil', 'costPerKey',
5
7
  'lastUsedAt', 'usageCounts', 'byModel', 'usageDays', 'quotaWindows', 'costDays',
@@ -18,6 +20,7 @@ export function initializePoolState(state = {}) {
18
20
  if (!Array.isArray(state.events)) state.events = [];
19
21
  if (!Number.isSafeInteger(state.pointer) || state.pointer < 0) state.pointer = 0;
20
22
  if (typeof state.lastUsed !== 'string') state.lastUsed = null;
23
+ if (!state.keyStateMachine) state.keyStateMachine = new KeyStateMachine();
21
24
  return state;
22
25
  }
23
26
 
package/lib/pool.js CHANGED
@@ -195,6 +195,7 @@ export function pickNext(pool, now, refsCount = pool.refs.length) {
195
195
  /** Apply a failed-key cooldown to pool state. Pure: returns next state shape. */
196
196
  export function applyCooldown(pool, ref, cooldownMs, now = Date.now()) {
197
197
  unstickFailedKey(pool, ref);
198
+ pool?.state?.keyStateMachine?.transitionToCooldown(ref, { cooldownMs }, now);
198
199
  return {
199
200
  ...pool,
200
201
  state: {
@@ -231,10 +232,9 @@ export function recordFailure(pool, ref, now, baseMs, maxMs, isSoft = false, jit
231
232
  pool.state.failCounts.set(ref, next);
232
233
  }
233
234
  let backoff = computeBackoff(baseMs, next || 1, maxMs, isSoft);
234
- if (jitter) {
235
- backoff = applyJitter(backoff);
236
- }
235
+ if (jitter) backoff = applyJitter(backoff);
237
236
  pool.state.failedUntil.set(ref, now + backoff);
237
+ pool.state?.keyStateMachine?.transitionToCooldown(ref, { cooldownMs: backoff, reason: isSoft ? 'SOFT_FAILURE' : 'FAILURE' }, now);
238
238
  return backoff;
239
239
  }
240
240
 
@@ -245,6 +245,7 @@ export function recordSuccess(pool, ref, now = Date.now()) {
245
245
  pool.state.quotaWindows?.delete(ref);
246
246
  if (!pool.state.lastSuccessAt) pool.state.lastSuccessAt = new Map();
247
247
  pool.state.lastSuccessAt.set(ref, now);
248
+ pool.state?.keyStateMachine?.activate(ref, { reason: 'SUCCESS' }, now);
248
249
  }
249
250
 
250
251
  /** Decay penalty failCounts for stable keys that haven't failed in decayIntervalMs. */
@@ -376,12 +377,9 @@ export function parseDurationToSeconds(v) {
376
377
  /** Check if a key reference is in maintenance/paused mode. */
377
378
  export function isKeyPaused(pool, ref) {
378
379
  if (!pool || !ref) return false;
379
- if (pool.pausedRefs instanceof Set && pool.pausedRefs.has(ref)) return true;
380
- if (Array.isArray(pool.pausedRefs) && pool.pausedRefs.includes(ref)) return true;
381
- if (pool.state && pool.state.pausedRefs instanceof Set && pool.state.pausedRefs.has(ref)) return true;
382
- if (pool.state && Array.isArray(pool.state.pausedRefs) && pool.state.pausedRefs.includes(ref)) return true;
383
- if (pool.basePool) return isKeyPaused(pool.basePool, ref);
384
- return false;
380
+ if (pool.pausedRefs?.has?.(ref) || pool.pausedRefs?.includes?.(ref)) return true;
381
+ if (pool.state?.pausedRefs?.has?.(ref) || pool.state?.pausedRefs?.includes?.(ref)) return true;
382
+ return pool.basePool ? isKeyPaused(pool.basePool, ref) : false;
385
383
  }
386
384
 
387
385
  /** Extract rate-limit info from an object that may carry response headers. */
@@ -578,12 +576,9 @@ export function resetCircuitForProvider(circuitBreaker, provider) {
578
576
  /** Check if a key reference is permanently revoked due to auth/401 failure (#369). */
579
577
  export function isKeyRevoked(pool, ref) {
580
578
  if (!pool || !ref) return false;
581
- if (pool.revokedRefs instanceof Set && pool.revokedRefs.has(ref)) return true;
582
- if (Array.isArray(pool.revokedRefs) && pool.revokedRefs.includes(ref)) return true;
583
- if (pool.state && pool.state.revokedRefs instanceof Set && pool.state.revokedRefs.has(ref)) return true;
584
- if (pool.state && Array.isArray(pool.state.revokedRefs) && pool.state.revokedRefs.includes(ref)) return true;
585
- if (pool.basePool) return isKeyRevoked(pool.basePool, ref);
586
- return false;
579
+ if (pool.revokedRefs?.has?.(ref) || pool.revokedRefs?.includes?.(ref)) return true;
580
+ if (pool.state?.revokedRefs?.has?.(ref) || pool.state?.revokedRefs?.includes?.(ref)) return true;
581
+ return pool.basePool ? isKeyRevoked(pool.basePool, ref) : false;
587
582
  }
588
583
 
589
584
  /** Check if a key reference is in active recoverable or permanent quarantine (#427). */
@@ -0,0 +1,222 @@
1
+ // lib/quota-runway.js — Predictive Quota Runway and burn rate estimation (#549).
2
+ // Estimates time to quota exhaustion and preemptively rotates keys before hitting HTTP 429.
3
+
4
+ export const MINUTE = 60 * 1000;
5
+ export const HOUR = 60 * MINUTE;
6
+ export const DAY = 24 * HOUR;
7
+
8
+ export const DEFAULT_WINDOW_DURATIONS = {
9
+ minute: MINUTE,
10
+ fiveMinute: 5 * MINUTE,
11
+ hourly: HOUR,
12
+ daily: DAY,
13
+ monthly: 30 * DAY,
14
+ };
15
+
16
+ /**
17
+ * Bounds percentage value to [0, 100].
18
+ * @param {number|unknown} value
19
+ * @returns {number}
20
+ */
21
+ export function clampPercent(value) {
22
+ const num = Number(value);
23
+ return Number.isFinite(num) ? Math.max(0, Math.min(100, num)) : 0;
24
+ }
25
+
26
+ /**
27
+ * Estimates when a quota window reaches 100% at the average burn rate observed.
28
+ *
29
+ * @param {object|undefined} window
30
+ * @param {number} window.percentUsed Current percentage used (0..100)
31
+ * @param {string|number} window.resetsAt ISO timestamp or epoch ms when window resets
32
+ * @param {number} durationMs Window duration in milliseconds
33
+ * @param {number} [now] Current epoch timestamp in milliseconds
34
+ * @returns {{
35
+ * etaMs: number,
36
+ * resetsAtMs: number,
37
+ * exhaustsBeforeReset: boolean,
38
+ * burnRatePerSec: number
39
+ * } | null}
40
+ */
41
+ export function estimateRunway(window, durationMs, now = Date.now()) {
42
+ if (!window || window.resetsAt == null || !durationMs || durationMs <= 0) return null;
43
+
44
+ const resetsAtMs = typeof window.resetsAt === 'number' ? window.resetsAt : Date.parse(window.resetsAt);
45
+ if (!Number.isFinite(resetsAtMs) || resetsAtMs <= now) return null;
46
+
47
+ const used = clampPercent(window.percentUsed);
48
+ const timeToReset = resetsAtMs - now;
49
+
50
+ // Already 100% or more: exhausts immediately
51
+ if (used >= 100) {
52
+ return {
53
+ etaMs: 0,
54
+ resetsAtMs,
55
+ exhaustsBeforeReset: true,
56
+ burnRatePerSec: 0,
57
+ };
58
+ }
59
+
60
+ const startedAt = resetsAtMs - durationMs;
61
+ const elapsed = Math.max(1, now - startedAt);
62
+ const ratePerMs = used / elapsed;
63
+
64
+ if (!Number.isFinite(ratePerMs) || ratePerMs <= 0) {
65
+ return {
66
+ etaMs: Number.POSITIVE_INFINITY,
67
+ resetsAtMs,
68
+ exhaustsBeforeReset: false,
69
+ burnRatePerSec: 0,
70
+ };
71
+ }
72
+
73
+ const remainingPercent = 100 - used;
74
+ const etaMs = remainingPercent / ratePerMs;
75
+ const exhaustsBeforeReset = etaMs < timeToReset;
76
+ const burnRatePerSec = ratePerMs * 1000;
77
+
78
+ return {
79
+ etaMs,
80
+ resetsAtMs,
81
+ exhaustsBeforeReset,
82
+ burnRatePerSec,
83
+ };
84
+ }
85
+
86
+ /**
87
+ * Stateful Quota Runway tracker for credentials and pools.
88
+ */
89
+ export class QuotaRunway {
90
+ /**
91
+ * @param {object} [options]
92
+ * @param {number} [options.slidingWindowMs]
93
+ */
94
+ constructor(options = {}) {
95
+ this.slidingWindowMs = options.slidingWindowMs ?? (5 * MINUTE);
96
+ /** @type {Map<string, Array<{ at: number, amount: number }>>} */
97
+ this.samples = new Map();
98
+ /** @type {Map<string, { limit: number, remaining: number, resetsAtMs: number, durationMs: number }>} */
99
+ this.knownLimits = new Map();
100
+ }
101
+
102
+ /**
103
+ * Records usage tokens or requests for a key.
104
+ * @param {string} keyRef
105
+ * @param {number} amount
106
+ * @param {number} [now]
107
+ */
108
+ recordUsage(keyRef, amount, now = Date.now()) {
109
+ if (!keyRef || !amount || amount <= 0) return;
110
+ let list = this.samples.get(keyRef);
111
+ if (!list) {
112
+ list = [];
113
+ this.samples.set(keyRef, list);
114
+ }
115
+ list.push({ at: now, amount });
116
+ this._prune(keyRef, now);
117
+ }
118
+
119
+ /**
120
+ * Updates key limits detected from response headers or provider metadata.
121
+ * @param {string} keyRef
122
+ * @param {object} limits
123
+ * @param {number} [limits.limit]
124
+ * @param {number} [limits.remaining]
125
+ * @param {number|string} [limits.resetsAt]
126
+ * @param {number} [limits.durationMs]
127
+ * @param {number} [now]
128
+ */
129
+ updateLimits(keyRef, limits, now = Date.now()) {
130
+ if (!keyRef || !limits) return;
131
+ const limit = Number(limits.tpmLimit ?? limits.rpmLimit ?? limits.limit);
132
+ const remaining = Number(limits.remainingTokens ?? limits.remainingRequests ?? limits.remaining);
133
+ const resetSec = Number(limits.resetTokens ?? limits.resetRequests ?? limits.retryAfter);
134
+
135
+ let resetsAtMs = NaN;
136
+ if (Number.isFinite(resetSec)) {
137
+ resetsAtMs = now + (resetSec * 1000);
138
+ } else if (limits.resetsAt || limits.resetAt) {
139
+ const at = limits.resetsAt || limits.resetAt;
140
+ resetsAtMs = typeof at === 'number' ? at : Date.parse(at);
141
+ }
142
+
143
+ const durationMs = Number(limits.durationMs) || DEFAULT_WINDOW_DURATIONS.minute;
144
+
145
+ if (Number.isFinite(limit) && Number.isFinite(remaining) && Number.isFinite(resetsAtMs)) {
146
+ this.knownLimits.set(keyRef, {
147
+ limit,
148
+ remaining,
149
+ resetsAtMs,
150
+ durationMs,
151
+ });
152
+ }
153
+ }
154
+
155
+ /**
156
+ * Checks if key is projected to exhaust its quota before the current window resets.
157
+ * @param {string} keyRef
158
+ * @param {number} [now]
159
+ * @returns {boolean}
160
+ */
161
+ isExhaustingBeforeReset(keyRef, now = Date.now()) {
162
+ const assessment = this.assessRunway(keyRef, now);
163
+ return Boolean(assessment?.exhaustsBeforeReset);
164
+ }
165
+
166
+ /**
167
+ * Returns complete runway assessment for a key.
168
+ * @param {string} keyRef
169
+ * @param {number} [now]
170
+ * @returns {{
171
+ * exhaustsBeforeReset: boolean,
172
+ * etaMs: number,
173
+ * burnRatePerSec: number,
174
+ * timeToResetMs: number
175
+ * } | null}
176
+ */
177
+ assessRunway(keyRef, now = Date.now()) {
178
+ if (!keyRef) return null;
179
+
180
+ const limits = this.knownLimits.get(keyRef);
181
+ if (!limits || limits.resetsAtMs <= now) {
182
+ return null;
183
+ }
184
+
185
+ const { limit, remaining, resetsAtMs, durationMs } = limits;
186
+ if (limit <= 0) return null;
187
+
188
+ const used = Math.max(0, limit - remaining);
189
+ const percentUsed = (used / limit) * 100;
190
+
191
+ const estimate = estimateRunway(
192
+ { percentUsed, resetsAt: resetsAtMs },
193
+ durationMs,
194
+ now,
195
+ );
196
+
197
+ if (!estimate) return null;
198
+
199
+ return {
200
+ exhaustsBeforeReset: estimate.exhaustsBeforeReset,
201
+ etaMs: estimate.etaMs,
202
+ burnRatePerSec: estimate.burnRatePerSec,
203
+ timeToResetMs: resetsAtMs - now,
204
+ };
205
+ }
206
+
207
+ _prune(keyRef, now) {
208
+ const list = this.samples.get(keyRef);
209
+ if (!list) return;
210
+ const cutoff = now - this.slidingWindowMs;
211
+ while (list.length > 0 && list[0].at < cutoff) {
212
+ list.shift();
213
+ }
214
+ }
215
+
216
+ clear() {
217
+ this.samples.clear();
218
+ this.knownLimits.clear();
219
+ }
220
+ }
221
+
222
+ export const defaultQuotaRunway = new QuotaRunway();