@goodandready/dsh-key-rotation 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/lib/client.js +23 -0
- package/lib/index.js +116 -13
- package/lib/pool.js +37 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -17,6 +17,13 @@
|
|
|
17
17
|
- **rotation counter** — how many times a provider switched key, on which failure, and how long ago.
|
|
18
18
|
- **key order** — ↑/↓ buttons; the order of keys is the order they are tried.
|
|
19
19
|
- **switch codes as checkboxes** instead of a comma-separated string.
|
|
20
|
+
- **Exponential backoff** — repeated failures on the same key double its cooldown (base → ×2 → ×4 → cap ×8), so a dead key is not retried every window.
|
|
21
|
+
- **Reset cooldown** — a *Reset cooldown* button in the card clears a provider's cooldown immediately (also via `POST /dsh-key-rotation/reset`).
|
|
22
|
+
- **Env bootstrap** — if a pool ref (e.g. `MYPROVIDER_API_KEY`) is already set in `process.env`, it is treated as a transient credential without needing a DSH credential first.
|
|
23
|
+
- **Per-provider cooldown** — override `cooldownMs` (and `maxCooldownMs`) per provider, fallback to the global values.
|
|
24
|
+
- **Exhaustion warning** — when every key is cooling, a red warning appears in the card and `lastExhaustionAt`/`exhaustionCount` are exposed via `GET /dsh-key-rotation/status`.
|
|
25
|
+
- **Failure log** — last 20 failures per provider (`at`, `ref`, `reason`, `cooldownMs`) via `/status` and a collapsible *Recent failures* list.
|
|
26
|
+
- **Non-stream safety net** — an `agent/request-error` hook retries sync calls (embeddings, batch) with the next key when the error is switchable.
|
|
20
27
|
|
|
21
28
|
## Install
|
|
22
29
|
|
package/lib/client.js
CHANGED
|
@@ -60,7 +60,11 @@ window.__ModuleLoader__.load({
|
|
|
60
60
|
keyFromEnv: 'from the environment, read-only here',
|
|
61
61
|
keyWriteFailed: 'could not store the key: {msg}',
|
|
62
62
|
keyHint: 'The value is stored in DSH credentials and never sent back to the browser — only its last 5 characters are shown. Names are generated for you; hover a key to see the one it uses.',
|
|
63
|
+
brokenKey: 'broken (3× AUTH)',
|
|
63
64
|
resetCooldown: 'Reset cooldown',
|
|
65
|
+
testKey: 'Test',
|
|
66
|
+
testOk: 'OK',
|
|
67
|
+
testFail: 'FAIL',
|
|
64
68
|
poolExhausted: 'pool exhausted — all keys cooling',
|
|
65
69
|
resetting: 'Resetting…',
|
|
66
70
|
keyLabel: 'Key {n}',
|
|
@@ -100,7 +104,11 @@ window.__ModuleLoader__.load({
|
|
|
100
104
|
keyFromEnv: 'задан в окружении, отсюда не меняется',
|
|
101
105
|
keyWriteFailed: 'не удалось сохранить ключ: {msg}',
|
|
102
106
|
keyHint: 'Значение хранится в учётных данных DSH и обратно в браузер не отдаётся — показываются только последние 5 символов. Имена переменных создаются автоматически; наведите на ключ, чтобы увидеть используемое имя.',
|
|
107
|
+
brokenKey: 'сломан (3× AUTH)',
|
|
103
108
|
resetCooldown: 'Сбросить кулдаун',
|
|
109
|
+
testKey: 'Тест',
|
|
110
|
+
testOk: 'OK',
|
|
111
|
+
testFail: 'FAIL',
|
|
104
112
|
poolExhausted: 'пул исчерпан — все ключи остывают',
|
|
105
113
|
resetting: 'Сброс…',
|
|
106
114
|
keyLabel: 'Ключ {n}',
|
|
@@ -266,6 +274,16 @@ window.__ModuleLoader__.load({
|
|
|
266
274
|
.catch((e) => setSecretError(t('keyWriteFailed').replace('{msg}', String(e?.message ?? e))))
|
|
267
275
|
.finally(() => setResetting(''));
|
|
268
276
|
};
|
|
277
|
+
const [testing, setTesting] = React.useState('');
|
|
278
|
+
const [testResult, setTestResult] = React.useState({});
|
|
279
|
+
const doTest = (ref) => {
|
|
280
|
+
setTesting(ref); setTestResult((m) => ({ ...m, [ref]: null }));
|
|
281
|
+
fetch('/dsh-key-rotation/test', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ ref }) })
|
|
282
|
+
.then((r) => r.json())
|
|
283
|
+
.then((data) => setTestResult((m) => ({ ...m, [ref]: data })))
|
|
284
|
+
.catch((e) => setTestResult((m) => ({ ...m, [ref]: { ok: false, message: String(e?.message ?? e) } })))
|
|
285
|
+
.finally(() => setTesting(''));
|
|
286
|
+
};
|
|
269
287
|
|
|
270
288
|
const keyInfo = (providerId, ref) => {
|
|
271
289
|
const entryStatus = status[providerId];
|
|
@@ -400,6 +418,7 @@ window.__ModuleLoader__.load({
|
|
|
400
418
|
const keyStatus = (providerId, ref) => {
|
|
401
419
|
const hit = keyInfo(providerId, ref);
|
|
402
420
|
if (!hit) return null;
|
|
421
|
+
if (hit.broken) return { color: 'var(--dsw-alias-state-error-primary)', text: t('brokenKey') };
|
|
403
422
|
if (!hit.present) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyMissing') };
|
|
404
423
|
if (hit.cooldownMsLeft > 0) {
|
|
405
424
|
return {
|
|
@@ -453,6 +472,10 @@ window.__ModuleLoader__.load({
|
|
|
453
472
|
}));
|
|
454
473
|
if (typed) meta.push(btn('✓', () => saveSecret(key, rowKey), { title: t('keySave'), key: 'w' }));
|
|
455
474
|
}
|
|
475
|
+
if (info && typeof info.usage === 'number' && info.usage > 0) meta.push(h('span', { key: 'u', className: 'krot-tail', title: 'requests through this key' }, String(info.usage)));
|
|
476
|
+
const tr = testResult[key];
|
|
477
|
+
if (tr) meta.push(h('span', { key: 'tr', className: 'krot-tail', title: tr.message || (tr.ok ? t('testOk') : t('testFail')) }, tr.ok ? '✓' : '✕'));
|
|
478
|
+
meta.push(h('button', { key: 't', className: 'krot-btn', onClick: () => doTest(key), disabled: testing === key, title: t('testKey') }, testing === key ? '…' : t('testKey')));
|
|
456
479
|
meta.push(h('span', { key: 'a', className: 'krot-acts' },
|
|
457
480
|
btn('↑', () => moveKey(pIndex, kIndex, -1), { disabled: kIndex === 0, title: t('moveUp') }),
|
|
458
481
|
btn('↓', () => moveKey(pIndex, kIndex, 1), { disabled: kIndex === keys.length - 1, title: t('moveDown') }),
|
package/lib/index.js
CHANGED
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
// providers: array [{ provider, keys: [envName, ...] }]
|
|
34
34
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
35
35
|
import Schema from '@deepseek-ai/schemastery';
|
|
36
|
-
import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue } from './pool.js';
|
|
36
|
+
import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter } from './pool.js';
|
|
37
37
|
|
|
38
38
|
export const name = 'dsh-key-rotation';
|
|
39
39
|
export const inject = ['llm', 'webServer', 'settings', 'credentials'];
|
|
@@ -46,6 +46,7 @@ const CONFIG_PATH = '/dsh-key-rotation/config';
|
|
|
46
46
|
const STATUS_PATH = '/dsh-key-rotation/status';
|
|
47
47
|
const KEY_PATH = '/dsh-key-rotation/key';
|
|
48
48
|
const RESET_PATH = '/dsh-key-rotation/reset';
|
|
49
|
+
const TEST_PATH = '/dsh-key-rotation/test';
|
|
49
50
|
|
|
50
51
|
/** The llm-pi-ai namespace whose provider profiles map providers to pools. */
|
|
51
52
|
const PIAI_NS = 'llm-pi-ai';
|
|
@@ -77,10 +78,15 @@ const DEFAULT_PROVIDERS = [];
|
|
|
77
78
|
export const Config = Schema.object({
|
|
78
79
|
switchCodes: Schema.array(Schema.string()).default([...DEFAULT_SWITCH_CODES]),
|
|
79
80
|
cooldownMs: Schema.number().default(60000),
|
|
81
|
+
maxCooldownMs: Schema.number(),
|
|
82
|
+
notifyWebhook: Schema.string().default(''),
|
|
83
|
+
notifyThreshold: Schema.number().default(3),
|
|
80
84
|
providers: Schema.array(Schema.object({
|
|
81
85
|
provider: Schema.string().required(),
|
|
82
86
|
keys: Schema.array(Schema.string()).default([]),
|
|
87
|
+
weights: Schema.array(Schema.number()).default([]),
|
|
83
88
|
cooldownMs: Schema.number(),
|
|
89
|
+
maxCooldownMs: Schema.number(),
|
|
84
90
|
})).default([...DEFAULT_PROVIDERS]),
|
|
85
91
|
});
|
|
86
92
|
|
|
@@ -241,6 +247,14 @@ export function apply(ctx, config = {}) {
|
|
|
241
247
|
// ── key-pool state, persisted across config reloads ──
|
|
242
248
|
// base provider -> { failedUntil: Map<ref, epochMs>, pointer: number, lastUsed: ref }
|
|
243
249
|
const poolState = new Map();
|
|
250
|
+
// Periodic sweep of expired cooldowns — keeps health probe cheap and avoids waiting for next user request
|
|
251
|
+
ctx.effect(() => {
|
|
252
|
+
const id = setInterval(() => {
|
|
253
|
+
const n = sweepExpired(poolState, Date.now());
|
|
254
|
+
if (n > 0) console.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
|
|
255
|
+
}, 30000);
|
|
256
|
+
return () => clearInterval(id);
|
|
257
|
+
}, 'dsh-key-rotation: sweep expired cooldowns');
|
|
244
258
|
|
|
245
259
|
// ── runtime snapshot: config + llm-pi-ai profile mapping ──
|
|
246
260
|
function buildRuntime() {
|
|
@@ -249,6 +263,9 @@ export function apply(ctx, config = {}) {
|
|
|
249
263
|
const cfg = Config(structuredClone(getConfig() ?? {})) ?? {};
|
|
250
264
|
const switchCodes = new Set(cfg.switchCodes ?? DEFAULT_SWITCH_CODES);
|
|
251
265
|
const cooldownMs = cfg.cooldownMs ?? 60000;
|
|
266
|
+
const maxCooldownMs = cfg.maxCooldownMs ?? undefined;
|
|
267
|
+
const notifyWebhook = cfg.notifyWebhook ?? '';
|
|
268
|
+
const notifyThreshold = cfg.notifyThreshold ?? 3;
|
|
252
269
|
|
|
253
270
|
// ref -> pool (every key env of every configured provider)
|
|
254
271
|
const poolByRef = new Map();
|
|
@@ -275,11 +292,19 @@ export function apply(ctx, config = {}) {
|
|
|
275
292
|
lastExhaustionAt: undefined,
|
|
276
293
|
exhaustionCount: 0,
|
|
277
294
|
events: [],
|
|
295
|
+
usageCounts: new Map(),
|
|
278
296
|
};
|
|
279
297
|
poolState.set(p.provider, state);
|
|
280
298
|
}
|
|
281
299
|
const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
|
|
282
|
-
const
|
|
300
|
+
const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
|
|
301
|
+
const weights = Array.isArray(p.weights) ? p.weights : [];
|
|
302
|
+
const weightedRefs = [];
|
|
303
|
+
for (let i = 0; i < refs.length; i++) {
|
|
304
|
+
const w = typeof weights[i] === 'number' && weights[i] > 0 ? Math.floor(weights[i]) : 1;
|
|
305
|
+
for (let k = 0; k < w; k++) weightedRefs.push(refs[i]);
|
|
306
|
+
}
|
|
307
|
+
const pool = { base: p.provider, refs, weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs, state, cooldownMs: poolCooldown, maxCooldownMs: poolMax };
|
|
283
308
|
for (const ref of refs) poolByRef.set(ref, pool);
|
|
284
309
|
for (let i = 1; i < refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
|
|
285
310
|
}
|
|
@@ -296,7 +321,7 @@ export function apply(ctx, config = {}) {
|
|
|
296
321
|
}
|
|
297
322
|
}
|
|
298
323
|
|
|
299
|
-
return { switchCodes, cooldownMs, poolByRef, providerToPool, cloneIds };
|
|
324
|
+
return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, poolByRef, providerToPool, cloneIds };
|
|
300
325
|
}
|
|
301
326
|
|
|
302
327
|
// ── patch credentials.resolve: pool refs resolve to the next healthy key ──
|
|
@@ -313,27 +338,61 @@ export function apply(ctx, config = {}) {
|
|
|
313
338
|
const pool = poolByRef.get(ref);
|
|
314
339
|
if (!pool) return original(ref);
|
|
315
340
|
const now = Date.now();
|
|
341
|
+
const list = pool.weightedRefs ?? pool.refs;
|
|
316
342
|
const start = pool.state.pointer ?? 0;
|
|
317
|
-
for (let i = 0; i <
|
|
318
|
-
const index = (start + i) %
|
|
319
|
-
const candidate =
|
|
343
|
+
for (let i = 0; i < list.length; i++) {
|
|
344
|
+
const index = (start + i) % list.length;
|
|
345
|
+
const candidate = list[index];
|
|
320
346
|
const until = pool.state.failedUntil.get(candidate);
|
|
321
347
|
if (until !== undefined && until > now) continue;
|
|
348
|
+
// perHour quota check
|
|
349
|
+
if (pool.perHour) {
|
|
350
|
+
if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
|
|
351
|
+
let win = pool.state.quotaWindows.get(candidate);
|
|
352
|
+
if (!win || now - win.start >= 3600000) win = { count: 0, start: now };
|
|
353
|
+
if (win.count >= pool.perHour) {
|
|
354
|
+
const until = win.start + 3600000;
|
|
355
|
+
if ((pool.state.failedUntil.get(candidate) ?? 0) < until) pool.state.failedUntil.set(candidate, until);
|
|
356
|
+
continue;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
322
359
|
let hit = await original(candidate);
|
|
323
360
|
if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
|
|
324
|
-
pool.state.pointer = (index + 1) %
|
|
361
|
+
pool.state.pointer = (index + 1) % list.length;
|
|
325
362
|
pool.state.lastUsed = candidate;
|
|
326
363
|
if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
|
|
327
364
|
pool.state.failedUntil.delete(candidate);
|
|
365
|
+
if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
|
|
366
|
+
if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
|
|
367
|
+
if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
|
|
368
|
+
pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
|
|
369
|
+
if (pool.perHour) {
|
|
370
|
+
if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
|
|
371
|
+
let win2 = pool.state.quotaWindows.get(candidate);
|
|
372
|
+
if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
|
|
373
|
+
win2.count++;
|
|
374
|
+
pool.state.quotaWindows.set(candidate, win2);
|
|
375
|
+
}
|
|
328
376
|
return hit;
|
|
329
377
|
}
|
|
330
378
|
// fallback: env var (transient, not persisted)
|
|
331
379
|
const envVal = envValue(candidate);
|
|
332
380
|
if (envVal !== undefined) {
|
|
333
|
-
pool.state.pointer = (index + 1) %
|
|
381
|
+
pool.state.pointer = (index + 1) % list.length;
|
|
334
382
|
pool.state.lastUsed = candidate;
|
|
335
383
|
if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
|
|
336
384
|
pool.state.failedUntil.delete(candidate);
|
|
385
|
+
if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
|
|
386
|
+
if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
|
|
387
|
+
if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
|
|
388
|
+
pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
|
|
389
|
+
if (pool.perHour) {
|
|
390
|
+
if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
|
|
391
|
+
let win2 = pool.state.quotaWindows.get(candidate);
|
|
392
|
+
if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
|
|
393
|
+
win2.count++;
|
|
394
|
+
pool.state.quotaWindows.set(candidate, win2);
|
|
395
|
+
}
|
|
337
396
|
return { value: envVal, source: 'env' };
|
|
338
397
|
}
|
|
339
398
|
}
|
|
@@ -352,10 +411,10 @@ export function apply(ctx, config = {}) {
|
|
|
352
411
|
// the resolve patch hands out the next key on each dispatch.
|
|
353
412
|
function rotate(options, pool) {
|
|
354
413
|
return (async function* () {
|
|
355
|
-
const { switchCodes, cooldownMs } = buildRuntime();
|
|
414
|
+
const { switchCodes, cooldownMs, maxCooldownMs } = buildRuntime();
|
|
356
415
|
let lastFailure = null;
|
|
357
416
|
|
|
358
|
-
for (let attempt = 0; attempt < pool.refs.length; attempt++) {
|
|
417
|
+
for (let attempt = 0; attempt < (pool.weightedRefs ?? pool.refs).length; attempt++) {
|
|
359
418
|
let yielded = false;
|
|
360
419
|
let switching = false;
|
|
361
420
|
let inner;
|
|
@@ -363,7 +422,7 @@ export function apply(ctx, config = {}) {
|
|
|
363
422
|
// mark the internal dispatch so the interceptor does not re-rotate
|
|
364
423
|
inner = ctx.llm.stream({ ...options, [MARKER]: true });
|
|
365
424
|
} catch (e) {
|
|
366
|
-
if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(),
|
|
425
|
+
if (pool.state.lastUsed) { const _retry = parseRetryAfter(String(e?.message ?? '')); const _base = pool.cooldownMs ?? cooldownMs; const _max = pool.maxCooldownMs ?? maxCooldownMs; const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base; const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max); pushEvent(pool, pool.state.lastUsed, e?.code ?? 'TRANSPORT', _b); const _code = String(e?.code ?? ''); if (_code === 'AUTH' || /auth/i.test(String(e?.message ?? ''))) { const _c = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1; pool.state.authFailCounts.set(pool.state.lastUsed, _c); if (_c >= 3) { pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); } } else { pool.state.authFailCounts.delete(pool.state.lastUsed); } }
|
|
367
426
|
lastFailure = finishError(e?.code ?? 'TRANSPORT',
|
|
368
427
|
`dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
|
|
369
428
|
console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(pool.state.lastUsed ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
|
|
@@ -387,7 +446,7 @@ export function apply(ctx, config = {}) {
|
|
|
387
446
|
const switchable = !yielded && kind === 'error' &&
|
|
388
447
|
(switchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message));
|
|
389
448
|
if (switchable) {
|
|
390
|
-
if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(),
|
|
449
|
+
if (pool.state.lastUsed) { const _retry = parseRetryAfter(message); const _base = pool.cooldownMs ?? cooldownMs; const _max = pool.maxCooldownMs ?? maxCooldownMs; const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base; const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max); pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b); const _code2 = String(code ?? ''); if (_code2 === 'AUTH' || /auth/i.test(message)) { const _c2 = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1; pool.state.authFailCounts.set(pool.state.lastUsed, _c2); if (_c2 >= 3) { pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); } } else { pool.state.authFailCounts.delete(pool.state.lastUsed); } }
|
|
391
450
|
pool.state.switches = (pool.state.switches ?? 0) + 1;
|
|
392
451
|
pool.state.lastReason = String(code ?? 'UNKNOWN');
|
|
393
452
|
pool.state.lastSwitchAt = Date.now();
|
|
@@ -414,6 +473,15 @@ export function apply(ctx, config = {}) {
|
|
|
414
473
|
pool.state.lastExhaustionAt = Date.now();
|
|
415
474
|
pool.state.exhaustionCount = (pool.state.exhaustionCount ?? 0) + 1;
|
|
416
475
|
console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — all ${pool.refs.length} keys cooling`);
|
|
476
|
+
// notify webhook if configured and threshold reached
|
|
477
|
+
try {
|
|
478
|
+
const { notifyWebhook, notifyThreshold } = buildRuntime();
|
|
479
|
+
if (notifyWebhook && pool.state.exhaustionCount >= notifyThreshold) {
|
|
480
|
+
const payload = JSON.stringify({ provider: options.provider, exhaustionCount: pool.state.exhaustionCount, at: pool.state.lastExhaustionAt, keys: pool.refs });
|
|
481
|
+
fetch(notifyWebhook, { method: 'POST', headers: { 'content-type': 'application/json' }, body: payload }).catch(()=>{});
|
|
482
|
+
}
|
|
483
|
+
} catch {}
|
|
484
|
+
|
|
417
485
|
yield lastFailure ?? finishError('TRANSPORT', 'dsh-key-rotation: all keys failed');
|
|
418
486
|
})();
|
|
419
487
|
}
|
|
@@ -482,6 +550,8 @@ export function apply(ctx, config = {}) {
|
|
|
482
550
|
writable,
|
|
483
551
|
active: pool.state.lastUsed === ref,
|
|
484
552
|
cooldownMsLeft: until !== undefined && until > now ? until - now : 0,
|
|
553
|
+
usage: pool.state.usageCounts?.get(ref) ?? 0,
|
|
554
|
+
broken: pool.state.brokenUntil?.has(ref) ?? false,
|
|
485
555
|
});
|
|
486
556
|
}
|
|
487
557
|
providers.push({
|
|
@@ -582,6 +652,8 @@ export function apply(ctx, config = {}) {
|
|
|
582
652
|
const cleared = st.failedUntil.size;
|
|
583
653
|
st.failedUntil.clear();
|
|
584
654
|
st.failCounts?.clear();
|
|
655
|
+
st.authFailCounts?.clear();
|
|
656
|
+
st.brokenUntil?.clear();
|
|
585
657
|
st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
|
|
586
658
|
json(res, 200, { ok: true, provider, cleared });
|
|
587
659
|
return;
|
|
@@ -592,6 +664,8 @@ export function apply(ctx, config = {}) {
|
|
|
592
664
|
if (st.failedUntil.has(ref) || st.failCounts?.has(ref)) {
|
|
593
665
|
st.failedUntil.delete(ref);
|
|
594
666
|
st.failCounts?.delete(ref);
|
|
667
|
+
st.authFailCounts?.delete(ref);
|
|
668
|
+
st.brokenUntil?.delete(ref);
|
|
595
669
|
if (st.lastUsed === ref) st.lastUsed = undefined;
|
|
596
670
|
found = true; break;
|
|
597
671
|
}
|
|
@@ -605,6 +679,35 @@ export function apply(ctx, config = {}) {
|
|
|
605
679
|
},
|
|
606
680
|
}), 'dsh-key-rotation: reset route');
|
|
607
681
|
|
|
682
|
+
// ── test route: dry-run a single key without rotation ──
|
|
683
|
+
ctx.effect(() => ctx.webServer.register({
|
|
684
|
+
kind: 'exact',
|
|
685
|
+
path: TEST_PATH,
|
|
686
|
+
handler: async (req, res) => {
|
|
687
|
+
if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
|
|
688
|
+
if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: test is local-only' } }); return; }
|
|
689
|
+
let body; try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
|
|
690
|
+
const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
|
|
691
|
+
if (!isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
|
|
692
|
+
const base = ctx.get('credentials');
|
|
693
|
+
try {
|
|
694
|
+
const hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
|
|
695
|
+
const present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
|
|
696
|
+
if (!present) { json(res, 200, { ok: false, ref, code: 'no-credential', message: 'no such credential' }); return; }
|
|
697
|
+
const tail = keyTail(hit.value);
|
|
698
|
+
// Check env source
|
|
699
|
+
let source = null; try { const d = await base?.describe?.(ref); source = d?.source ?? null; } catch {}
|
|
700
|
+
if (!present) {
|
|
701
|
+
const ev = envValue(ref);
|
|
702
|
+
if (ev !== undefined) json(res, 200, { ok: true, ref, tail: keyTail(ev), source: 'env' });
|
|
703
|
+
}
|
|
704
|
+
json(res, 200, { ok: true, ref, tail, source });
|
|
705
|
+
} catch (e) {
|
|
706
|
+
json(res, 200, { ok: false, ref, code: 'error', message: String(e?.message ?? e) });
|
|
707
|
+
}
|
|
708
|
+
},
|
|
709
|
+
}), 'dsh-key-rotation: test route');
|
|
710
|
+
|
|
608
711
|
// Intercept the llm/stream waterfall: rotate any request whose provider maps
|
|
609
712
|
// to a configured key pool; pass everything else (and internal dispatches)
|
|
610
713
|
// straight through.
|
|
@@ -613,7 +716,7 @@ export function apply(ctx, config = {}) {
|
|
|
613
716
|
const { providerToPool } = buildRuntime();
|
|
614
717
|
const pool = providerToPool.get(options.provider);
|
|
615
718
|
if (!pool) return next();
|
|
616
|
-
console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${pool.refs.length} keys`);
|
|
719
|
+
console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${(pool.weightedRefs ?? pool.refs).length} slots (${pool.refs.length} keys)`);
|
|
617
720
|
return rotate(options, pool);
|
|
618
721
|
});
|
|
619
722
|
|
package/lib/pool.js
CHANGED
|
@@ -139,3 +139,40 @@ export function envValue(ref) {
|
|
|
139
139
|
const v = typeof process !== 'undefined' ? process.env?.[ref] : undefined;
|
|
140
140
|
return typeof v === 'string' && v.length > 0 ? v : undefined;
|
|
141
141
|
}
|
|
142
|
+
|
|
143
|
+
/** Sweep expired cooldown entries from poolState. Returns count of cleared refs. */
|
|
144
|
+
export function sweepExpired(poolState, now = Date.now()) {
|
|
145
|
+
let cleared = 0;
|
|
146
|
+
for (const st of poolState.values()) {
|
|
147
|
+
for (const [ref, until] of [...st.failedUntil.entries()]) {
|
|
148
|
+
if (until <= now) {
|
|
149
|
+
st.failedUntil.delete(ref);
|
|
150
|
+
st.failCounts?.delete(ref);
|
|
151
|
+
cleared++;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return cleared;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Parse Retry-After value from a header string or message.
|
|
159
|
+
* Supports seconds ("60", "Retry-After: 60") and HTTP-date ("Wed, 21 Oct 2026 07:28:00 GMT").
|
|
160
|
+
* Returns milliseconds or undefined if not parseable. */
|
|
161
|
+
export function parseRetryAfter(value) {
|
|
162
|
+
if (typeof value !== 'string' || !value) return undefined;
|
|
163
|
+
// Try to extract "Retry-After: <val>" from a larger message
|
|
164
|
+
const m = value.match(/retry-after\s*[:=]\s*(.+)/i);
|
|
165
|
+
const raw = m ? m[1].trim().split(/[\n\r;]/)[0].trim() : value.trim();
|
|
166
|
+
// Seconds
|
|
167
|
+
if (/^\d+$/.test(raw)) {
|
|
168
|
+
const sec = Number(raw);
|
|
169
|
+
if (sec >= 0 && sec <= 86400 * 7) return sec * 1000;
|
|
170
|
+
}
|
|
171
|
+
// HTTP-date
|
|
172
|
+
const ts = Date.parse(raw);
|
|
173
|
+
if (!Number.isNaN(ts)) {
|
|
174
|
+
const diff = ts - Date.now();
|
|
175
|
+
if (diff > 0 && diff < 86400 * 7 * 1000) return diff;
|
|
176
|
+
}
|
|
177
|
+
return undefined;
|
|
178
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-key-rotation",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.1",
|
|
4
4
|
"description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"deepseek-harness",
|