@goodandready/dsh-key-rotation 0.6.1 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -17,6 +17,13 @@
17
17
  - **rotation counter** — how many times a provider switched key, on which failure, and how long ago.
18
18
  - **key order** — ↑/↓ buttons; the order of keys is the order they are tried.
19
19
  - **switch codes as checkboxes** instead of a comma-separated string.
20
+ - **Exponential backoff** — repeated failures on the same key double its cooldown (base → ×2 → ×4 → cap ×8), so a dead key is not retried every window.
21
+ - **Reset cooldown** — a *Reset cooldown* button in the card clears a provider's cooldown immediately (also via `POST /dsh-key-rotation/reset`).
22
+ - **Env bootstrap** — if a pool ref (e.g. `MYPROVIDER_API_KEY`) is already set in `process.env`, it is treated as a transient credential without needing a DSH credential first.
23
+ - **Per-provider cooldown** — override `cooldownMs` (and `maxCooldownMs`) per provider, fallback to the global values.
24
+ - **Exhaustion warning** — when every key is cooling, a red warning appears in the card and `lastExhaustionAt`/`exhaustionCount` are exposed via `GET /dsh-key-rotation/status`.
25
+ - **Failure log** — last 20 failures per provider (`at`, `ref`, `reason`, `cooldownMs`) via `/status` and a collapsible *Recent failures* list.
26
+ - **Non-stream safety net** — an `agent/request-error` hook retries sync calls (embeddings, batch) with the next key when the error is switchable.
20
27
 
21
28
  ## Install
22
29
 
package/lib/client.js CHANGED
@@ -60,7 +60,11 @@ window.__ModuleLoader__.load({
60
60
  keyFromEnv: 'from the environment, read-only here',
61
61
  keyWriteFailed: 'could not store the key: {msg}',
62
62
  keyHint: 'The value is stored in DSH credentials and never sent back to the browser — only its last 5 characters are shown. Names are generated for you; hover a key to see the one it uses.',
63
+ brokenKey: 'broken (3× AUTH)',
63
64
  resetCooldown: 'Reset cooldown',
65
+ testKey: 'Test',
66
+ testOk: 'OK',
67
+ testFail: 'FAIL',
64
68
  poolExhausted: 'pool exhausted — all keys cooling',
65
69
  resetting: 'Resetting…',
66
70
  keyLabel: 'Key {n}',
@@ -100,7 +104,11 @@ window.__ModuleLoader__.load({
100
104
  keyFromEnv: 'задан в окружении, отсюда не меняется',
101
105
  keyWriteFailed: 'не удалось сохранить ключ: {msg}',
102
106
  keyHint: 'Значение хранится в учётных данных DSH и обратно в браузер не отдаётся — показываются только последние 5 символов. Имена переменных создаются автоматически; наведите на ключ, чтобы увидеть используемое имя.',
107
+ brokenKey: 'сломан (3× AUTH)',
103
108
  resetCooldown: 'Сбросить кулдаун',
109
+ testKey: 'Тест',
110
+ testOk: 'OK',
111
+ testFail: 'FAIL',
104
112
  poolExhausted: 'пул исчерпан — все ключи остывают',
105
113
  resetting: 'Сброс…',
106
114
  keyLabel: 'Ключ {n}',
@@ -266,6 +274,16 @@ window.__ModuleLoader__.load({
266
274
  .catch((e) => setSecretError(t('keyWriteFailed').replace('{msg}', String(e?.message ?? e))))
267
275
  .finally(() => setResetting(''));
268
276
  };
277
+ const [testing, setTesting] = React.useState('');
278
+ const [testResult, setTestResult] = React.useState({});
279
+ const doTest = (ref) => {
280
+ setTesting(ref); setTestResult((m) => ({ ...m, [ref]: null }));
281
+ fetch('/dsh-key-rotation/test', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ ref }) })
282
+ .then((r) => r.json())
283
+ .then((data) => setTestResult((m) => ({ ...m, [ref]: data })))
284
+ .catch((e) => setTestResult((m) => ({ ...m, [ref]: { ok: false, message: String(e?.message ?? e) } })))
285
+ .finally(() => setTesting(''));
286
+ };
269
287
 
270
288
  const keyInfo = (providerId, ref) => {
271
289
  const entryStatus = status[providerId];
@@ -400,6 +418,7 @@ window.__ModuleLoader__.load({
400
418
  const keyStatus = (providerId, ref) => {
401
419
  const hit = keyInfo(providerId, ref);
402
420
  if (!hit) return null;
421
+ if (hit.broken) return { color: 'var(--dsw-alias-state-error-primary)', text: t('brokenKey') };
403
422
  if (!hit.present) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyMissing') };
404
423
  if (hit.cooldownMsLeft > 0) {
405
424
  return {
@@ -453,6 +472,10 @@ window.__ModuleLoader__.load({
453
472
  }));
454
473
  if (typed) meta.push(btn('✓', () => saveSecret(key, rowKey), { title: t('keySave'), key: 'w' }));
455
474
  }
475
+ if (info && typeof info.usage === 'number' && info.usage > 0) meta.push(h('span', { key: 'u', className: 'krot-tail', title: 'requests through this key' }, String(info.usage)));
476
+ const tr = testResult[key];
477
+ if (tr) meta.push(h('span', { key: 'tr', className: 'krot-tail', title: tr.message || (tr.ok ? t('testOk') : t('testFail')) }, tr.ok ? '✓' : '✕'));
478
+ meta.push(h('button', { key: 't', className: 'krot-btn', onClick: () => doTest(key), disabled: testing === key, title: t('testKey') }, testing === key ? '…' : t('testKey')));
456
479
  meta.push(h('span', { key: 'a', className: 'krot-acts' },
457
480
  btn('↑', () => moveKey(pIndex, kIndex, -1), { disabled: kIndex === 0, title: t('moveUp') }),
458
481
  btn('↓', () => moveKey(pIndex, kIndex, 1), { disabled: kIndex === keys.length - 1, title: t('moveDown') }),
package/lib/index.js CHANGED
@@ -33,7 +33,7 @@
33
33
  // providers: array [{ provider, keys: [envName, ...] }]
34
34
  // ─────────────────────────────────────────────────────────────────────────────
35
35
  import Schema from '@deepseek-ai/schemastery';
36
- import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue } from './pool.js';
36
+ import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter } from './pool.js';
37
37
 
38
38
  export const name = 'dsh-key-rotation';
39
39
  export const inject = ['llm', 'webServer', 'settings', 'credentials'];
@@ -46,6 +46,7 @@ const CONFIG_PATH = '/dsh-key-rotation/config';
46
46
  const STATUS_PATH = '/dsh-key-rotation/status';
47
47
  const KEY_PATH = '/dsh-key-rotation/key';
48
48
  const RESET_PATH = '/dsh-key-rotation/reset';
49
+ const TEST_PATH = '/dsh-key-rotation/test';
49
50
 
50
51
  /** The llm-pi-ai namespace whose provider profiles map providers to pools. */
51
52
  const PIAI_NS = 'llm-pi-ai';
@@ -77,10 +78,15 @@ const DEFAULT_PROVIDERS = [];
77
78
  export const Config = Schema.object({
78
79
  switchCodes: Schema.array(Schema.string()).default([...DEFAULT_SWITCH_CODES]),
79
80
  cooldownMs: Schema.number().default(60000),
81
+ maxCooldownMs: Schema.number(),
82
+ notifyWebhook: Schema.string().default(''),
83
+ notifyThreshold: Schema.number().default(3),
80
84
  providers: Schema.array(Schema.object({
81
85
  provider: Schema.string().required(),
82
86
  keys: Schema.array(Schema.string()).default([]),
87
+ weights: Schema.array(Schema.number()).default([]),
83
88
  cooldownMs: Schema.number(),
89
+ maxCooldownMs: Schema.number(),
84
90
  })).default([...DEFAULT_PROVIDERS]),
85
91
  });
86
92
 
@@ -241,6 +247,14 @@ export function apply(ctx, config = {}) {
241
247
  // ── key-pool state, persisted across config reloads ──
242
248
  // base provider -> { failedUntil: Map<ref, epochMs>, pointer: number, lastUsed: ref }
243
249
  const poolState = new Map();
250
+ // Periodic sweep of expired cooldowns — keeps health probe cheap and avoids waiting for next user request
251
+ ctx.effect(() => {
252
+ const id = setInterval(() => {
253
+ const n = sweepExpired(poolState, Date.now());
254
+ if (n > 0) console.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
255
+ }, 30000);
256
+ return () => clearInterval(id);
257
+ }, 'dsh-key-rotation: sweep expired cooldowns');
244
258
 
245
259
  // ── runtime snapshot: config + llm-pi-ai profile mapping ──
246
260
  function buildRuntime() {
@@ -249,6 +263,9 @@ export function apply(ctx, config = {}) {
249
263
  const cfg = Config(structuredClone(getConfig() ?? {})) ?? {};
250
264
  const switchCodes = new Set(cfg.switchCodes ?? DEFAULT_SWITCH_CODES);
251
265
  const cooldownMs = cfg.cooldownMs ?? 60000;
266
+ const maxCooldownMs = cfg.maxCooldownMs ?? undefined;
267
+ const notifyWebhook = cfg.notifyWebhook ?? '';
268
+ const notifyThreshold = cfg.notifyThreshold ?? 3;
252
269
 
253
270
  // ref -> pool (every key env of every configured provider)
254
271
  const poolByRef = new Map();
@@ -275,11 +292,19 @@ export function apply(ctx, config = {}) {
275
292
  lastExhaustionAt: undefined,
276
293
  exhaustionCount: 0,
277
294
  events: [],
295
+ usageCounts: new Map(),
278
296
  };
279
297
  poolState.set(p.provider, state);
280
298
  }
281
299
  const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
282
- const pool = { base: p.provider, refs, state, cooldownMs: poolCooldown };
300
+ const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
301
+ const weights = Array.isArray(p.weights) ? p.weights : [];
302
+ const weightedRefs = [];
303
+ for (let i = 0; i < refs.length; i++) {
304
+ const w = typeof weights[i] === 'number' && weights[i] > 0 ? Math.floor(weights[i]) : 1;
305
+ for (let k = 0; k < w; k++) weightedRefs.push(refs[i]);
306
+ }
307
+ const pool = { base: p.provider, refs, weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs, state, cooldownMs: poolCooldown, maxCooldownMs: poolMax };
283
308
  for (const ref of refs) poolByRef.set(ref, pool);
284
309
  for (let i = 1; i < refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
285
310
  }
@@ -296,7 +321,7 @@ export function apply(ctx, config = {}) {
296
321
  }
297
322
  }
298
323
 
299
- return { switchCodes, cooldownMs, poolByRef, providerToPool, cloneIds };
324
+ return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, poolByRef, providerToPool, cloneIds };
300
325
  }
301
326
 
302
327
  // ── patch credentials.resolve: pool refs resolve to the next healthy key ──
@@ -313,27 +338,61 @@ export function apply(ctx, config = {}) {
313
338
  const pool = poolByRef.get(ref);
314
339
  if (!pool) return original(ref);
315
340
  const now = Date.now();
341
+ const list = pool.weightedRefs ?? pool.refs;
316
342
  const start = pool.state.pointer ?? 0;
317
- for (let i = 0; i < pool.refs.length; i++) {
318
- const index = (start + i) % pool.refs.length;
319
- const candidate = pool.refs[index];
343
+ for (let i = 0; i < list.length; i++) {
344
+ const index = (start + i) % list.length;
345
+ const candidate = list[index];
320
346
  const until = pool.state.failedUntil.get(candidate);
321
347
  if (until !== undefined && until > now) continue;
348
+ // perHour quota check
349
+ if (pool.perHour) {
350
+ if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
351
+ let win = pool.state.quotaWindows.get(candidate);
352
+ if (!win || now - win.start >= 3600000) win = { count: 0, start: now };
353
+ if (win.count >= pool.perHour) {
354
+ const until = win.start + 3600000;
355
+ if ((pool.state.failedUntil.get(candidate) ?? 0) < until) pool.state.failedUntil.set(candidate, until);
356
+ continue;
357
+ }
358
+ }
322
359
  let hit = await original(candidate);
323
360
  if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
324
- pool.state.pointer = (index + 1) % pool.refs.length;
361
+ pool.state.pointer = (index + 1) % list.length;
325
362
  pool.state.lastUsed = candidate;
326
363
  if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
327
364
  pool.state.failedUntil.delete(candidate);
365
+ if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
366
+ if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
367
+ if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
368
+ pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
369
+ if (pool.perHour) {
370
+ if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
371
+ let win2 = pool.state.quotaWindows.get(candidate);
372
+ if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
373
+ win2.count++;
374
+ pool.state.quotaWindows.set(candidate, win2);
375
+ }
328
376
  return hit;
329
377
  }
330
378
  // fallback: env var (transient, not persisted)
331
379
  const envVal = envValue(candidate);
332
380
  if (envVal !== undefined) {
333
- pool.state.pointer = (index + 1) % pool.refs.length;
381
+ pool.state.pointer = (index + 1) % list.length;
334
382
  pool.state.lastUsed = candidate;
335
383
  if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
336
384
  pool.state.failedUntil.delete(candidate);
385
+ if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
386
+ if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
387
+ if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
388
+ pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
389
+ if (pool.perHour) {
390
+ if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
391
+ let win2 = pool.state.quotaWindows.get(candidate);
392
+ if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
393
+ win2.count++;
394
+ pool.state.quotaWindows.set(candidate, win2);
395
+ }
337
396
  return { value: envVal, source: 'env' };
338
397
  }
339
398
  }
@@ -352,10 +411,10 @@ export function apply(ctx, config = {}) {
352
411
  // the resolve patch hands out the next key on each dispatch.
353
412
  function rotate(options, pool) {
354
413
  return (async function* () {
355
- const { switchCodes, cooldownMs } = buildRuntime();
414
+ const { switchCodes, cooldownMs, maxCooldownMs } = buildRuntime();
356
415
  let lastFailure = null;
357
416
 
358
- for (let attempt = 0; attempt < pool.refs.length; attempt++) {
417
+ for (let attempt = 0; attempt < (pool.weightedRefs ?? pool.refs).length; attempt++) {
359
418
  let yielded = false;
360
419
  let switching = false;
361
420
  let inner;
@@ -363,7 +422,7 @@ export function apply(ctx, config = {}) {
363
422
  // mark the internal dispatch so the interceptor does not re-rotate
364
423
  inner = ctx.llm.stream({ ...options, [MARKER]: true });
365
424
  } catch (e) {
366
- if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), pool.cooldownMs ?? cooldownMs); pushEvent(pool, pool.state.lastUsed, e?.code ?? 'TRANSPORT', _b); }
425
+ if (pool.state.lastUsed) { const _retry = parseRetryAfter(String(e?.message ?? '')); const _base = pool.cooldownMs ?? cooldownMs; const _max = pool.maxCooldownMs ?? maxCooldownMs; const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base; const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max); pushEvent(pool, pool.state.lastUsed, e?.code ?? 'TRANSPORT', _b); const _code = String(e?.code ?? ''); if (_code === 'AUTH' || /auth/i.test(String(e?.message ?? ''))) { const _c = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1; pool.state.authFailCounts.set(pool.state.lastUsed, _c); if (_c >= 3) { pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); } } else { pool.state.authFailCounts.delete(pool.state.lastUsed); } }
367
426
  lastFailure = finishError(e?.code ?? 'TRANSPORT',
368
427
  `dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
369
428
  console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(pool.state.lastUsed ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
@@ -387,7 +446,7 @@ export function apply(ctx, config = {}) {
387
446
  const switchable = !yielded && kind === 'error' &&
388
447
  (switchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message));
389
448
  if (switchable) {
390
- if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), pool.cooldownMs ?? cooldownMs); pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b); }
449
+ if (pool.state.lastUsed) { const _retry = parseRetryAfter(message); const _base = pool.cooldownMs ?? cooldownMs; const _max = pool.maxCooldownMs ?? maxCooldownMs; const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base; const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max); pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b); const _code2 = String(code ?? ''); if (_code2 === 'AUTH' || /auth/i.test(message)) { const _c2 = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1; pool.state.authFailCounts.set(pool.state.lastUsed, _c2); if (_c2 >= 3) { pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); } } else { pool.state.authFailCounts.delete(pool.state.lastUsed); } }
391
450
  pool.state.switches = (pool.state.switches ?? 0) + 1;
392
451
  pool.state.lastReason = String(code ?? 'UNKNOWN');
393
452
  pool.state.lastSwitchAt = Date.now();
@@ -414,6 +473,15 @@ export function apply(ctx, config = {}) {
414
473
  pool.state.lastExhaustionAt = Date.now();
415
474
  pool.state.exhaustionCount = (pool.state.exhaustionCount ?? 0) + 1;
416
475
  console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — all ${pool.refs.length} keys cooling`);
476
+ // notify webhook if configured and threshold reached
477
+ try {
478
+ const { notifyWebhook, notifyThreshold } = buildRuntime();
479
+ if (notifyWebhook && pool.state.exhaustionCount >= notifyThreshold) {
480
+ const payload = JSON.stringify({ provider: options.provider, exhaustionCount: pool.state.exhaustionCount, at: pool.state.lastExhaustionAt, keys: pool.refs });
481
+ fetch(notifyWebhook, { method: 'POST', headers: { 'content-type': 'application/json' }, body: payload }).catch(()=>{});
482
+ }
483
+ } catch {}
484
+
417
485
  yield lastFailure ?? finishError('TRANSPORT', 'dsh-key-rotation: all keys failed');
418
486
  })();
419
487
  }
@@ -482,6 +550,8 @@ export function apply(ctx, config = {}) {
482
550
  writable,
483
551
  active: pool.state.lastUsed === ref,
484
552
  cooldownMsLeft: until !== undefined && until > now ? until - now : 0,
553
+ usage: pool.state.usageCounts?.get(ref) ?? 0,
554
+ broken: pool.state.brokenUntil?.has(ref) ?? false,
485
555
  });
486
556
  }
487
557
  providers.push({
@@ -582,6 +652,8 @@ export function apply(ctx, config = {}) {
582
652
  const cleared = st.failedUntil.size;
583
653
  st.failedUntil.clear();
584
654
  st.failCounts?.clear();
655
+ st.authFailCounts?.clear();
656
+ st.brokenUntil?.clear();
585
657
  st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
586
658
  json(res, 200, { ok: true, provider, cleared });
587
659
  return;
@@ -592,6 +664,8 @@ export function apply(ctx, config = {}) {
592
664
  if (st.failedUntil.has(ref) || st.failCounts?.has(ref)) {
593
665
  st.failedUntil.delete(ref);
594
666
  st.failCounts?.delete(ref);
667
+ st.authFailCounts?.delete(ref);
668
+ st.brokenUntil?.delete(ref);
595
669
  if (st.lastUsed === ref) st.lastUsed = undefined;
596
670
  found = true; break;
597
671
  }
@@ -605,6 +679,35 @@ export function apply(ctx, config = {}) {
605
679
  },
606
680
  }), 'dsh-key-rotation: reset route');
607
681
 
682
+ // ── test route: dry-run a single key without rotation ──
683
+ ctx.effect(() => ctx.webServer.register({
684
+ kind: 'exact',
685
+ path: TEST_PATH,
686
+ handler: async (req, res) => {
687
+ if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
688
+ if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: test is local-only' } }); return; }
689
+ let body; try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
690
+ const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
691
+ if (!isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
692
+ const base = ctx.get('credentials');
693
+ try {
694
+ const hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
695
+ const present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
696
+ if (!present) { json(res, 200, { ok: false, ref, code: 'no-credential', message: 'no such credential' }); return; }
697
+ const tail = keyTail(hit.value);
698
+ // Check env source
699
+ let source = null; try { const d = await base?.describe?.(ref); source = d?.source ?? null; } catch {}
700
+ if (!present) {
701
+ const ev = envValue(ref);
702
+ if (ev !== undefined) json(res, 200, { ok: true, ref, tail: keyTail(ev), source: 'env' });
703
+ }
704
+ json(res, 200, { ok: true, ref, tail, source });
705
+ } catch (e) {
706
+ json(res, 200, { ok: false, ref, code: 'error', message: String(e?.message ?? e) });
707
+ }
708
+ },
709
+ }), 'dsh-key-rotation: test route');
710
+
608
711
  // Intercept the llm/stream waterfall: rotate any request whose provider maps
609
712
  // to a configured key pool; pass everything else (and internal dispatches)
610
713
  // straight through.
@@ -613,7 +716,7 @@ export function apply(ctx, config = {}) {
613
716
  const { providerToPool } = buildRuntime();
614
717
  const pool = providerToPool.get(options.provider);
615
718
  if (!pool) return next();
616
- console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${pool.refs.length} keys`);
719
+ console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${(pool.weightedRefs ?? pool.refs).length} slots (${pool.refs.length} keys)`);
617
720
  return rotate(options, pool);
618
721
  });
619
722
 
package/lib/pool.js CHANGED
@@ -139,3 +139,40 @@ export function envValue(ref) {
139
139
  const v = typeof process !== 'undefined' ? process.env?.[ref] : undefined;
140
140
  return typeof v === 'string' && v.length > 0 ? v : undefined;
141
141
  }
142
+
143
+ /** Sweep expired cooldown entries from poolState. Returns count of cleared refs. */
144
+ export function sweepExpired(poolState, now = Date.now()) {
145
+ let cleared = 0;
146
+ for (const st of poolState.values()) {
147
+ for (const [ref, until] of [...st.failedUntil.entries()]) {
148
+ if (until <= now) {
149
+ st.failedUntil.delete(ref);
150
+ st.failCounts?.delete(ref);
151
+ cleared++;
152
+ }
153
+ }
154
+ }
155
+ return cleared;
156
+ }
157
+
158
+ /** Parse Retry-After value from a header string or message.
159
+ * Supports seconds ("60", "Retry-After: 60") and HTTP-date ("Wed, 21 Oct 2026 07:28:00 GMT").
160
+ * Returns milliseconds or undefined if not parseable. */
161
+ export function parseRetryAfter(value) {
162
+ if (typeof value !== 'string' || !value) return undefined;
163
+ // Try to extract "Retry-After: <val>" from a larger message
164
+ const m = value.match(/retry-after\s*[:=]\s*(.+)/i);
165
+ const raw = m ? m[1].trim().split(/[\n\r;]/)[0].trim() : value.trim();
166
+ // Seconds
167
+ if (/^\d+$/.test(raw)) {
168
+ const sec = Number(raw);
169
+ if (sec >= 0 && sec <= 86400 * 7) return sec * 1000;
170
+ }
171
+ // HTTP-date
172
+ const ts = Date.parse(raw);
173
+ if (!Number.isNaN(ts)) {
174
+ const diff = ts - Date.now();
175
+ if (diff > 0 && diff < 86400 * 7 * 1000) return diff;
176
+ }
177
+ return undefined;
178
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@goodandready/dsh-key-rotation",
3
- "version": "0.6.1",
3
+ "version": "0.7.1",
4
4
  "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
5
5
  "keywords": [
6
6
  "deepseek-harness",