@goodandready/dsh-key-rotation 0.7.10 → 0.7.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +44 -1
- package/lib/index.js +241 -42
- package/lib/pool.js +24 -0
- package/package.json +1 -4
package/lib/client.js
CHANGED
|
@@ -39,6 +39,7 @@ window.__ModuleLoader__.load({
|
|
|
39
39
|
addProvider: '+ Add provider',
|
|
40
40
|
desc: 'Per-provider API key rotation. For each provider, list its API keys (env names, stored in DSH credentials). The plugin routes a model through that provider\u2019s keys in order and switches to the next on a quota/rate-limit failure.',
|
|
41
41
|
cooldown: 'Cooldown after failure (ms)',
|
|
42
|
+
scheduleDays: 'Rotation schedule (days, 0=off)',
|
|
42
43
|
switchCodes: 'Switch codes (comma-separated)',
|
|
43
44
|
providersTitle: 'Providers and their keys',
|
|
44
45
|
save: 'Save',
|
|
@@ -63,6 +64,8 @@ window.__ModuleLoader__.load({
|
|
|
63
64
|
keyWriteFailed: 'could not store the key: {msg}',
|
|
64
65
|
keyHint: 'The value is stored in DSH credentials and never sent back to the browser — only its last 5 characters are shown. Names are generated for you; hover a key to see the one it uses.',
|
|
65
66
|
brokenKey: 'broken (3× AUTH)',
|
|
67
|
+
keyExpired: 'expired',
|
|
68
|
+
keyExpiringSoon: 'expires in {n} d',
|
|
66
69
|
exportPools: 'Export',
|
|
67
70
|
exportOne: '⬇',
|
|
68
71
|
usedAgo: '{ago} ago',
|
|
@@ -92,6 +95,7 @@ window.__ModuleLoader__.load({
|
|
|
92
95
|
addProvider: '+ Добавить провайдера',
|
|
93
96
|
desc: 'Ротация API-ключей по провайдерам. Для каждого провайдера укажите его API-ключи (имена env, хранятся в учётных данных DSH). Плагин ведёт модель по ключам провайдера по порядку и переключается на следующий при исчерпании квоты/превышении лимита.',
|
|
94
97
|
cooldown: 'Задержка после сбоя (мс)',
|
|
98
|
+
scheduleDays: 'Расписание ротации (дней, 0=выкл)',
|
|
95
99
|
switchCodes: 'Коды переключения (через запятую)',
|
|
96
100
|
providersTitle: 'Провайдеры и их ключи',
|
|
97
101
|
save: 'Сохранить',
|
|
@@ -116,6 +120,8 @@ window.__ModuleLoader__.load({
|
|
|
116
120
|
keyWriteFailed: 'не удалось сохранить ключ: {msg}',
|
|
117
121
|
keyHint: 'Значение хранится в учётных данных DSH и обратно в браузер не отдаётся — показываются только последние 5 символов. Имена переменных создаются автоматически; наведите на ключ, чтобы увидеть используемое имя.',
|
|
118
122
|
brokenKey: 'сломан (3× AUTH)',
|
|
123
|
+
keyExpired: 'истёк',
|
|
124
|
+
keyExpiringSoon: 'истекает через {n} д',
|
|
119
125
|
exportPools: 'Экспорт',
|
|
120
126
|
exportOne: '⬇',
|
|
121
127
|
usedAgo: '{ago} назад',
|
|
@@ -347,10 +353,34 @@ window.__ModuleLoader__.load({
|
|
|
347
353
|
return (entryStatus.keys ?? []).find((k) => k.ref === ref) ?? null;
|
|
348
354
|
};
|
|
349
355
|
|
|
350
|
-
const
|
|
356
|
+
const [validating, setValidating] = React.useState('');
|
|
357
|
+
const [validationResult, setValidationResult] = React.useState({});
|
|
358
|
+
const validateBeforeSave = (ref, value) => {
|
|
359
|
+
setValidating(ref);
|
|
360
|
+
return fetch('/dsh-key-rotation/test', {
|
|
361
|
+
method: 'POST',
|
|
362
|
+
headers: { 'content-type': 'application/json' },
|
|
363
|
+
body: JSON.stringify({ ref, value }),
|
|
364
|
+
})
|
|
365
|
+
.then((r) => r.json())
|
|
366
|
+
.then((data) => { setValidationResult((m) => ({ ...m, [ref]: data })); return data; })
|
|
367
|
+
.catch(() => null)
|
|
368
|
+
.finally(() => setValidating(''));
|
|
369
|
+
};
|
|
370
|
+
const saveSecret = async (ref, rowKey) => {
|
|
351
371
|
const value = secretDraft[rowKey];
|
|
352
372
|
if (!value) return;
|
|
353
373
|
setSecretError('');
|
|
374
|
+
// Pre-save validation (issue #118)
|
|
375
|
+
setValidating(ref);
|
|
376
|
+
const vres = await validateBeforeSave(ref, value);
|
|
377
|
+
setValidating('');
|
|
378
|
+
if (vres && vres.ok === false && vres.code === 'no-credential') {
|
|
379
|
+
// No credential yet is fine for a new key being saved
|
|
380
|
+
} else if (vres && !vres.ok) {
|
|
381
|
+
setSecretError(t('keyWriteFailed').replace('{msg}', vres.message || 'validation failed'));
|
|
382
|
+
return;
|
|
383
|
+
}
|
|
354
384
|
fetch('/dsh-key-rotation/key', {
|
|
355
385
|
method: 'PUT',
|
|
356
386
|
headers: { 'content-type': 'application/json' },
|
|
@@ -476,6 +506,11 @@ window.__ModuleLoader__.load({
|
|
|
476
506
|
const keyStatus = (providerId, ref) => {
|
|
477
507
|
const hit = keyInfo(providerId, ref);
|
|
478
508
|
if (!hit) return null;
|
|
509
|
+
if (hit.expired) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyExpired') };
|
|
510
|
+
if (hit.expiresAt && !hit.expired) {
|
|
511
|
+
const days = Math.ceil((hit.expiresAt - Date.now()) / 86400000);
|
|
512
|
+
if (days <= 7) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyExpiringSoon').replace('{n}', String(days)) };
|
|
513
|
+
}
|
|
479
514
|
if (hit.broken) return { color: 'var(--dsw-alias-state-error-primary)', text: t('brokenKey') };
|
|
480
515
|
if (!hit.present) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyMissing') };
|
|
481
516
|
if (hit.cooldownMsLeft > 0) {
|
|
@@ -590,6 +625,13 @@ window.__ModuleLoader__.load({
|
|
|
590
625
|
});
|
|
591
626
|
}, { title: 'Sort by usage' }),
|
|
592
627
|
h('select', { className: 'krot-in', value: entry.provider, onChange: (e) => setProvider(pIndex, e.target.value) }, options),
|
|
628
|
+
(() => {
|
|
629
|
+
const ps = status[entry.provider];
|
|
630
|
+
const score = ps && typeof ps.healthScore === 'number' ? ps.healthScore : null;
|
|
631
|
+
if (score === null) return null;
|
|
632
|
+
const color = score > 80 ? 'var(--dsw-alias-state-success-primary)' : score >= 50 ? 'var(--dsw-alias-state-warning-primary)' : 'var(--dsw-alias-state-error-primary)';
|
|
633
|
+
return h('span', { className: 'krot-tail', title: 'health score', style: { flex: 'none', color, fontWeight: 700 } }, String(score));
|
|
634
|
+
})(),
|
|
593
635
|
(() => { const ps = status[entry.provider]; const tot = ps && typeof ps.totalUsage === 'number' ? ps.totalUsage : null; return tot !== null ? h('span', { className: 'krot-tail', title: 'total requests', style: { flex: 'none' } }, String(tot)) : null; })(),
|
|
594
636
|
btn('✕', () => removeProvider(pIndex), { title: t('removeProvider') }),
|
|
595
637
|
),
|
|
@@ -613,6 +655,7 @@ window.__ModuleLoader__.load({
|
|
|
613
655
|
return h('div', { className: 'krot' },
|
|
614
656
|
h('p', { className: 'krot-hint' }, t('desc')),
|
|
615
657
|
field(t('cooldown'), textInput(String(val.cooldownMs ?? 60000), (v) => setField((cur) => ({ ...cur, cooldownMs: Number(v) || 0 })))),
|
|
658
|
+
field(t('scheduleDays'), textInput(String(val.rotationScheduleDays ?? 0), (v) => setField((cur) => ({ ...cur, rotationScheduleDays: Number(v) || 0 })))),
|
|
616
659
|
field(t('codesTitle'), h('div', { className: 'krot-codes' }, codeList.map((code) => h('label', { key: code, className: 'krot-code' },
|
|
617
660
|
h('input', {
|
|
618
661
|
type: 'checkbox',
|
package/lib/index.js
CHANGED
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
// providers: array [{ provider, keys: [envName, ...] }]
|
|
34
34
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
35
35
|
import Schema from '@deepseek-ai/schemastery';
|
|
36
|
-
import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter } from './pool.js';
|
|
36
|
+
import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter, computeHealthScore } from './pool.js';
|
|
37
37
|
|
|
38
38
|
export const name = 'dsh-key-rotation';
|
|
39
39
|
export const inject = ['llm', 'webServer', 'settings', 'credentials'];
|
|
@@ -46,6 +46,8 @@ const CONFIG_PATH = '/dsh-key-rotation/config';
|
|
|
46
46
|
const STATUS_PATH = '/dsh-key-rotation/status';
|
|
47
47
|
const KEY_PATH = '/dsh-key-rotation/key';
|
|
48
48
|
const RESET_PATH = '/dsh-key-rotation/reset';
|
|
49
|
+
const IMPORT_PATH = '/dsh-key-rotation/import';
|
|
50
|
+
const HEALTH_PATH = '/dsh-key-rotation/health';
|
|
49
51
|
const TEST_PATH = '/dsh-key-rotation/test';
|
|
50
52
|
|
|
51
53
|
/** The llm-pi-ai namespace whose provider profiles map providers to pools. */
|
|
@@ -81,10 +83,19 @@ export const Config = Schema.object({
|
|
|
81
83
|
maxCooldownMs: Schema.number(),
|
|
82
84
|
notifyWebhook: Schema.string().default(''),
|
|
83
85
|
notifyThreshold: Schema.number().default(3),
|
|
86
|
+
backupDir: Schema.string().default(''),
|
|
87
|
+
backupIntervalMs: Schema.number().default(86400000),
|
|
88
|
+
backupKeep: Schema.number().default(7),
|
|
89
|
+
rotationScheduleDays: Schema.number().default(0),
|
|
84
90
|
providers: Schema.array(Schema.object({
|
|
85
91
|
provider: Schema.string().required(),
|
|
86
92
|
keys: Schema.array(Schema.string()).default([]),
|
|
87
93
|
weights: Schema.array(Schema.number()).default([]),
|
|
94
|
+
expiresAt: Schema.array(Schema.union([Schema.number(), Schema.string()])).default([]),
|
|
95
|
+
models: Schema.dict(Schema.object({
|
|
96
|
+
keys: Schema.array(Schema.string()).default([]),
|
|
97
|
+
weights: Schema.array(Schema.number()).default([]),
|
|
98
|
+
})).default({}),
|
|
88
99
|
cooldownMs: Schema.number(),
|
|
89
100
|
maxCooldownMs: Schema.number(),
|
|
90
101
|
})).default([...DEFAULT_PROVIDERS]),
|
|
@@ -247,6 +258,91 @@ export function apply(ctx, config = {}) {
|
|
|
247
258
|
// ── key-pool state, persisted across config reloads ──
|
|
248
259
|
// base provider -> { failedUntil: Map<ref, epochMs>, pointer: number, lastUsed: ref }
|
|
249
260
|
const poolState = new Map();
|
|
261
|
+
// Periodic backup of pools config
|
|
262
|
+
ctx.effect(() => {
|
|
263
|
+
const { backupDir, backupIntervalMs, backupKeep } = buildRuntime();
|
|
264
|
+
if (!backupDir) return;
|
|
265
|
+
const id = setInterval(() => {
|
|
266
|
+
try {
|
|
267
|
+
const fs = require('node:fs');
|
|
268
|
+
const path = require('node:path');
|
|
269
|
+
const dir = backupDir;
|
|
270
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
271
|
+
const now = new Date();
|
|
272
|
+
const dateStr = now.toISOString().slice(0,10).replace(/-/g,'');
|
|
273
|
+
const file = path.join(dir, 'pools-' + dateStr + '.json');
|
|
274
|
+
const data = JSON.stringify({ backup: now.toISOString(), providers: getConfig()?.providers ?? [] }, null, 2);
|
|
275
|
+
fs.writeFileSync(file, data, 'utf8');
|
|
276
|
+
// prune old backups
|
|
277
|
+
const keep = backupKeep || 7;
|
|
278
|
+
const files = fs.readdirSync(dir).filter((f) => f.startsWith('pools-') && f.endsWith('.json')).sort();
|
|
279
|
+
while (files.length > keep) {
|
|
280
|
+
const old = files.shift();
|
|
281
|
+
fs.unlinkSync(path.join(dir, old));
|
|
282
|
+
}
|
|
283
|
+
} catch (e) {
|
|
284
|
+
console.warn('[dsh-key-rotation] backup failed:', String(e?.message ?? e));
|
|
285
|
+
}
|
|
286
|
+
}, backupIntervalMs || 86400000);
|
|
287
|
+
return () => clearInterval(id);
|
|
288
|
+
}, 'dsh-key-rotation: backup pools');
|
|
289
|
+
// Periodic save of usage/cost stats to file
|
|
290
|
+
ctx.effect(() => {
|
|
291
|
+
const { backupDir } = buildRuntime();
|
|
292
|
+
if (!backupDir) return;
|
|
293
|
+
try {
|
|
294
|
+
const fs = require('node:fs');
|
|
295
|
+
const path = require('node:path');
|
|
296
|
+
const statsFile = path.join(backupDir, 'stats.json');
|
|
297
|
+
// Load existing stats at startup
|
|
298
|
+
try {
|
|
299
|
+
if (fs.existsSync(statsFile)) {
|
|
300
|
+
const saved = JSON.parse(fs.readFileSync(statsFile, 'utf8'));
|
|
301
|
+
for (const st of poolState.values()) {
|
|
302
|
+
if (saved.usageCounts && st.usageCounts) { for (const [k, v] of Object.entries(saved.usageCounts)) st.usageCounts.set(k, (st.usageCounts.get(k) ?? 0) + v); }
|
|
303
|
+
if (saved.costPerKey && st.costPerKey) { for (const [k, v] of Object.entries(saved.costPerKey)) st.costPerKey.set(k, (st.costPerKey.get(k) ?? 0) + v); }
|
|
304
|
+
if (saved.lastUsedAt && st.lastUsedAt) { for (const [k, v] of Object.entries(saved.lastUsedAt)) { if (!st.lastUsedAt.has(k) || v > st.lastUsedAt.get(k)) st.lastUsedAt.set(k, v); } }
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
} catch {}
|
|
308
|
+
// Periodic save
|
|
309
|
+
const id = setInterval(() => {
|
|
310
|
+
try {
|
|
311
|
+
const usageCounts = {}; const costPerKey = {}; const lastUsedAt = {};
|
|
312
|
+
for (const [base, st] of poolState) {
|
|
313
|
+
if (st.usageCounts) for (const [k, v] of st.usageCounts) usageCounts[k] = v;
|
|
314
|
+
if (st.costPerKey) for (const [k, v] of st.costPerKey) costPerKey[k] = v;
|
|
315
|
+
if (st.lastUsedAt) for (const [k, v] of st.lastUsedAt) lastUsedAt[k] = v;
|
|
316
|
+
}
|
|
317
|
+
fs.writeFileSync(statsFile, JSON.stringify({ t: Date.now(), usageCounts, costPerKey, lastUsedAt }), 'utf8');
|
|
318
|
+
} catch {}
|
|
319
|
+
}, 60000);
|
|
320
|
+
return () => clearInterval(id);
|
|
321
|
+
} catch { return () => {}; }
|
|
322
|
+
}, 'dsh-key-rotation: persist stats');
|
|
323
|
+
// Rotation schedule: shift pointer every N days
|
|
324
|
+
ctx.effect(() => {
|
|
325
|
+
const { rotationScheduleDays } = buildRuntime();
|
|
326
|
+
if (!rotationScheduleDays || rotationScheduleDays <= 0) return;
|
|
327
|
+
const intervalMs = Math.min(rotationScheduleDays * 86400000, 2147483647);
|
|
328
|
+
const id = setInterval(() => {
|
|
329
|
+
try {
|
|
330
|
+
const rt = buildRuntime();
|
|
331
|
+
let shifted = 0;
|
|
332
|
+
for (const pool of rt.poolByRef.values()) {
|
|
333
|
+
if (pool.refs.length < 2) continue;
|
|
334
|
+
const oldPtr = pool.state.pointer ?? 0;
|
|
335
|
+
pool.state.pointer = (oldPtr + 1) % pool.refs.length;
|
|
336
|
+
shifted++;
|
|
337
|
+
console.warn(`[dsh-key-rotation] ${pool.base}: scheduled rotation -> ${pool.refs[pool.state.pointer]} (day ${rotationScheduleDays})`);
|
|
338
|
+
}
|
|
339
|
+
if (shifted) console.warn(`[dsh-key-rotation] schedule: rotated ${shifted} pools`);
|
|
340
|
+
} catch (e) {
|
|
341
|
+
console.warn('[dsh-key-rotation] schedule error:', String(e?.message ?? e));
|
|
342
|
+
}
|
|
343
|
+
}, intervalMs);
|
|
344
|
+
return () => clearInterval(id);
|
|
345
|
+
}, 'dsh-key-rotation: rotation schedule');
|
|
250
346
|
// Periodic sweep of expired cooldowns — keeps health probe cheap and avoids waiting for next user request
|
|
251
347
|
ctx.effect(() => {
|
|
252
348
|
const id = setInterval(() => {
|
|
@@ -283,42 +379,67 @@ export function apply(ctx, config = {}) {
|
|
|
283
379
|
const poolByRef = new Map();
|
|
284
380
|
// provider route (from llm-pi-ai profiles) -> its key pool
|
|
285
381
|
const providerToPool = new Map();
|
|
382
|
+
// per-model key pools: provider -> Map<model, pool>
|
|
383
|
+
const modelPoolByProvider = new Map();
|
|
286
384
|
// clone route ids (for the settings dropdown filter)
|
|
287
385
|
const cloneIds = new Set();
|
|
288
386
|
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
if (
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
failCounts: new Map(),
|
|
297
|
-
pointer: 0,
|
|
298
|
-
lastUsed: undefined,
|
|
299
|
-
// Счётчики для карточки настроек: без них о работе ротации можно было
|
|
300
|
-
// судить только по console.warn на сервере.
|
|
301
|
-
switches: 0,
|
|
302
|
-
lastReason: undefined,
|
|
303
|
-
lastSwitchAt: undefined,
|
|
304
|
-
lastExhaustionAt: undefined,
|
|
305
|
-
exhaustionCount: 0,
|
|
306
|
-
events: [],
|
|
307
|
-
usageCounts: new Map(),
|
|
387
|
+
const makeState = (base) => {
|
|
388
|
+
let st = poolState.get(base);
|
|
389
|
+
if (!st) {
|
|
390
|
+
st = {
|
|
391
|
+
failedUntil: new Map(), failCounts: new Map(), pointer: 0, lastUsed: undefined,
|
|
392
|
+
switches: 0, lastReason: undefined, lastSwitchAt: undefined,
|
|
393
|
+
lastExhaustionAt: undefined, exhaustionCount: 0, events: [], usageCounts: new Map(),
|
|
308
394
|
};
|
|
309
|
-
poolState.set(
|
|
395
|
+
poolState.set(base, st);
|
|
310
396
|
}
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
397
|
+
return st;
|
|
398
|
+
};
|
|
399
|
+
const parseExpiry = (v) => {
|
|
400
|
+
if (typeof v === 'number' && v > 0) return v;
|
|
401
|
+
if (typeof v === 'string' && v.length > 0) { const ts = Date.parse(v); return Number.isNaN(ts) ? undefined : ts; }
|
|
402
|
+
return undefined;
|
|
403
|
+
};
|
|
404
|
+
const buildPool = (base, keys, weights, poolCooldown, poolMax, expiresAt) => {
|
|
405
|
+
const refs = (keys ?? []).filter((ref) => typeof ref === 'string' && ref.length > 0);
|
|
406
|
+
if (refs.length === 0) return null;
|
|
407
|
+
const w = Array.isArray(weights) ? weights : [];
|
|
314
408
|
const weightedRefs = [];
|
|
315
409
|
for (let i = 0; i < refs.length; i++) {
|
|
316
|
-
const
|
|
317
|
-
for (let k = 0; k <
|
|
410
|
+
const ww = typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1;
|
|
411
|
+
for (let k = 0; k < ww; k++) weightedRefs.push(refs[i]);
|
|
412
|
+
}
|
|
413
|
+
const parsedExpiry = {};
|
|
414
|
+
if (Array.isArray(expiresAt)) {
|
|
415
|
+
for (let i = 0; i < refs.length; i++) {
|
|
416
|
+
const exp = parseExpiry(expiresAt[i]);
|
|
417
|
+
if (exp !== undefined) parsedExpiry[refs[i]] = exp;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
return { base, refs, weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs,
|
|
421
|
+
state: makeState(base), cooldownMs: poolCooldown, maxCooldownMs: poolMax, expiresAt: parsedExpiry };
|
|
422
|
+
};
|
|
423
|
+
for (const p of cfg.providers ?? []) {
|
|
424
|
+
const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
|
|
425
|
+
const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
|
|
426
|
+
// base provider pool (fallback)
|
|
427
|
+
const pool = buildPool(p.provider, p.keys, p.weights, poolCooldown, poolMax);
|
|
428
|
+
if (pool) {
|
|
429
|
+
for (const ref of pool.refs) poolByRef.set(ref, pool);
|
|
430
|
+
for (let i = 1; i < pool.refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
|
|
431
|
+
}
|
|
432
|
+
// per-model pools
|
|
433
|
+
const models = p.models ?? {};
|
|
434
|
+
const byModel = new Map();
|
|
435
|
+
for (const [model, mp] of Object.entries(models)) {
|
|
436
|
+
const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, poolCooldown, poolMax);
|
|
437
|
+
if (mpool) {
|
|
438
|
+
byModel.set(model, mpool);
|
|
439
|
+
for (const ref of mpool.refs) poolByRef.set(ref, mpool);
|
|
440
|
+
}
|
|
318
441
|
}
|
|
319
|
-
|
|
320
|
-
for (const ref of refs) poolByRef.set(ref, pool);
|
|
321
|
-
for (let i = 1; i < refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
|
|
442
|
+
if (byModel.size > 0) modelPoolByProvider.set(p.provider, byModel);
|
|
322
443
|
}
|
|
323
444
|
|
|
324
445
|
let profiles = {};
|
|
@@ -337,7 +458,7 @@ export function apply(ctx, config = {}) {
|
|
|
337
458
|
for (const key of [...poolState.keys()]) {
|
|
338
459
|
if (![...poolByRef.values()].some((p) => p.base === key)) poolState.delete(key);
|
|
339
460
|
}
|
|
340
|
-
return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, poolByRef, providerToPool, cloneIds };
|
|
461
|
+
return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
|
|
341
462
|
}
|
|
342
463
|
|
|
343
464
|
// ── patch credentials.resolve: pool refs resolve to the next healthy key ──
|
|
@@ -361,6 +482,7 @@ export function apply(ctx, config = {}) {
|
|
|
361
482
|
const candidate = list[index];
|
|
362
483
|
const until = pool.state.failedUntil.get(candidate);
|
|
363
484
|
if (until !== undefined && until > now) continue;
|
|
485
|
+
if (pool.expiresAt?.[candidate] !== undefined && now >= pool.expiresAt[candidate]) continue;
|
|
364
486
|
// perHour quota check
|
|
365
487
|
if (pool.perHour) {
|
|
366
488
|
if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
|
|
@@ -575,6 +697,8 @@ export function apply(ctx, config = {}) {
|
|
|
575
697
|
usage: pool.state.usageCounts?.get(ref) ?? 0,
|
|
576
698
|
cost: pool.state.costPerKey?.get(ref) ?? 0,
|
|
577
699
|
lastUsedAt: pool.state.lastUsedAt?.get(ref) ?? null,
|
|
700
|
+
expiresAt: pool.expiresAt?.[ref] ?? null,
|
|
701
|
+
expired: pool.expiresAt?.[ref] !== undefined && now >= pool.expiresAt[ref],
|
|
578
702
|
broken: pool.state.brokenUntil?.has(ref) ?? false,
|
|
579
703
|
});
|
|
580
704
|
}
|
|
@@ -588,6 +712,7 @@ export function apply(ctx, config = {}) {
|
|
|
588
712
|
exhaustionCount: pool.state.exhaustionCount ?? 0,
|
|
589
713
|
totalUsage: [...(pool.state.usageCounts?.values() ?? [])].reduce((a, b) => a + b, 0),
|
|
590
714
|
events: (pool.state.events ?? []).slice(-50),
|
|
715
|
+
healthScore: computeHealthScore(pool.state),
|
|
591
716
|
});
|
|
592
717
|
}
|
|
593
718
|
json(res, 200, { providers });
|
|
@@ -704,6 +829,71 @@ export function apply(ctx, config = {}) {
|
|
|
704
829
|
},
|
|
705
830
|
}), 'dsh-key-rotation: reset route');
|
|
706
831
|
|
|
832
|
+
ctx.effect(() => ctx.webServer.register({
|
|
833
|
+
kind: 'exact',
|
|
834
|
+
path: IMPORT_PATH,
|
|
835
|
+
handler: async (req, res) => {
|
|
836
|
+
if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
|
|
837
|
+
if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: import is local-only' } }); return; }
|
|
838
|
+
let body; try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
|
|
839
|
+
const url = typeof body?.url === 'string' ? body.url.trim() : '';
|
|
840
|
+
if (!url || !url.startsWith('https://')) { json(res, 400, { error: { code: 'bad-url', message: 'dsh-key-rotation: only HTTPS URLs are allowed' } }); return; }
|
|
841
|
+
try {
|
|
842
|
+
const resp = await fetch(url);
|
|
843
|
+
if (!resp.ok) { json(res, 400, { error: { code: 'fetch-failed', message: 'dsh-key-rotation: fetch returned ' + resp.status } }); return; }
|
|
844
|
+
const data = await resp.json();
|
|
845
|
+
if (!Array.isArray(data)) { json(res, 400, { error: { code: 'bad-format', message: 'dsh-key-rotation: expected JSON array of providers' } }); return; }
|
|
846
|
+
const settings = ctx.get('settings');
|
|
847
|
+
if (!settings) { json(res, 503, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: no settings provider' } }); return; }
|
|
848
|
+
const desc = settings.describe({ redactSecrets: true }).find((c) => c.ns === NS);
|
|
849
|
+
const cur = desc?.value?.providers ?? [];
|
|
850
|
+
const merged = new Map();
|
|
851
|
+
for (const p of cur) if (p && p.provider) merged.set(p.provider, p);
|
|
852
|
+
for (const p of data) if (p && p.provider && typeof p.provider === 'string') merged.set(p.provider, p);
|
|
853
|
+
const mergedArr = [...merged.values()];
|
|
854
|
+
await settings.replace(NS, { ...(desc?.value ?? {}), providers: mergedArr }, desc?.revision);
|
|
855
|
+
json(res, 200, { ok: true, providersImported: data.length, total: mergedArr.length });
|
|
856
|
+
} catch (e) { json(res, 400, { error: { code: 'import-failed', message: String(e?.message ?? e) } }); }
|
|
857
|
+
},
|
|
858
|
+
}), 'dsh-key-rotation: import route');
|
|
859
|
+
|
|
860
|
+
// Health for external panels (Beszel/Uptime)
|
|
861
|
+
ctx.effect(() => ctx.webServer.register({
|
|
862
|
+
kind: 'exact',
|
|
863
|
+
path: HEALTH_PATH,
|
|
864
|
+
handler: async (req, res) => {
|
|
865
|
+
if (!isTrustedBridgeRequest(req) && req.socket?.remoteAddress !== '127.0.0.1' && req.socket?.remoteAddress !== '::1') { } // allow same-origin already checked
|
|
866
|
+
if (!isTrustedBridgeRequest(req)) {
|
|
867
|
+
// also allow plain loopback without Origin
|
|
868
|
+
if (!isLoopbackAddress(req.socket?.remoteAddress)) { res.writeHead(403); res.end(); return; }
|
|
869
|
+
if (req.headers['sec-fetch-site'] === 'cross-site') { res.writeHead(403); res.end(); return; }
|
|
870
|
+
}
|
|
871
|
+
if (req.method !== 'GET') { json(res, 405, { error: { code: 'method', message: 'GET only' } }); return; }
|
|
872
|
+
const now = Date.now();
|
|
873
|
+
const pools = {};
|
|
874
|
+
let exhaustedAny = false;
|
|
875
|
+
const { poolByRef: pr } = buildRuntime();
|
|
876
|
+
const seenH = new Set();
|
|
877
|
+
for (const pool of pr.values()) {
|
|
878
|
+
if (seenH.has(pool.base)) continue;
|
|
879
|
+
seenH.add(pool.base);
|
|
880
|
+
let healthy = 0;
|
|
881
|
+
for (const ref of pool.refs) {
|
|
882
|
+
const until = pool.state.failedUntil.get(ref);
|
|
883
|
+
if (until !== undefined && until > now) continue;
|
|
884
|
+
const exp = pool.expiresAt?.[ref];
|
|
885
|
+
if (exp !== undefined && now >= exp) continue;
|
|
886
|
+
healthy++;
|
|
887
|
+
}
|
|
888
|
+
const total = pool.refs.length;
|
|
889
|
+
const exhausted = healthy === 0 && total > 0;
|
|
890
|
+
if (exhausted) exhaustedAny = true;
|
|
891
|
+
pools[pool.base] = { healthy, total, exhausted, healthScore: computeHealthScore(pool.state) };
|
|
892
|
+
}
|
|
893
|
+
json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny });
|
|
894
|
+
},
|
|
895
|
+
}), 'dsh-key-rotation: health');
|
|
896
|
+
|
|
707
897
|
// ── test route: dry-run a single key without rotation ──
|
|
708
898
|
ctx.effect(() => ctx.webServer.register({
|
|
709
899
|
kind: 'exact',
|
|
@@ -714,17 +904,23 @@ export function apply(ctx, config = {}) {
|
|
|
714
904
|
let body; try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
|
|
715
905
|
const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
|
|
716
906
|
if (!isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
|
|
907
|
+
// Optional value for pre-save validation (issue #118)
|
|
908
|
+
const testValue = typeof body?.value === 'string' && body.value.length > 0 ? body.value : undefined;
|
|
717
909
|
const base = ctx.get('credentials');
|
|
718
910
|
try {
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
const
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
911
|
+
let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
|
|
912
|
+
let present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
|
|
913
|
+
// Pre-save validation: check the provided value directly (issue #118)
|
|
914
|
+
const effectiveValue = testValue || hit?.value;
|
|
915
|
+
const valid = present ? Boolean(effectiveValue && typeof effectiveValue === 'string' && effectiveValue.length > 0) : Boolean(testValue);
|
|
916
|
+
const tail = valid ? keyTail(effectiveValue) : '';
|
|
917
|
+
let source = null;
|
|
918
|
+
try { const d = await base?.describe?.(ref); source = d?.source ?? null; } catch {}
|
|
919
|
+
if (!present && !testValue) { json(res, 200, { ok: false, ref, code: 'no-credential', message: 'no such credential' }); return; }
|
|
920
|
+
if (!present && testValue) { source = 'pre-save'; }
|
|
921
|
+
else if (!present) {
|
|
726
922
|
const ev = envValue(ref);
|
|
727
|
-
if (ev !== undefined) json(res, 200, { ok: true, ref, tail: keyTail(ev), source: 'env' });
|
|
923
|
+
if (ev !== undefined) { present = true; json(res, 200, { ok: true, ref, tail: keyTail(ev), source: 'env' }); return; }
|
|
728
924
|
}
|
|
729
925
|
json(res, 200, { ok: true, ref, tail, source });
|
|
730
926
|
} catch (e) {
|
|
@@ -738,8 +934,9 @@ export function apply(ctx, config = {}) {
|
|
|
738
934
|
// straight through.
|
|
739
935
|
ctx.on('llm/stream', (options, next) => {
|
|
740
936
|
if (options[MARKER]) return next();
|
|
741
|
-
const { providerToPool } = buildRuntime();
|
|
742
|
-
const
|
|
937
|
+
const { providerToPool, modelPoolByProvider } = buildRuntime();
|
|
938
|
+
const byModel = modelPoolByProvider.get(options.provider);
|
|
939
|
+
const pool = (byModel && byModel.get(options.model)) || providerToPool.get(options.provider);
|
|
743
940
|
if (!pool) return next();
|
|
744
941
|
console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${(pool.weightedRefs ?? pool.refs).length} slots (${pool.refs.length} keys)`);
|
|
745
942
|
return rotate(options, pool);
|
|
@@ -752,8 +949,10 @@ export function apply(ctx, config = {}) {
|
|
|
752
949
|
ctx.on('agent/request-error', async (payload, next) => {
|
|
753
950
|
const provider = payload?.provider ?? payload?.failure?.provider ?? '';
|
|
754
951
|
if (!provider) return next();
|
|
755
|
-
const { providerToPool, switchCodes } = buildRuntime();
|
|
756
|
-
const
|
|
952
|
+
const { providerToPool, modelPoolByProvider, switchCodes } = buildRuntime();
|
|
953
|
+
const model = payload?.model || payload?.failure?.model || '';
|
|
954
|
+
const byModel = modelPoolByProvider.get(provider);
|
|
955
|
+
const pool = (byModel && byModel.get(model)) || providerToPool.get(provider);
|
|
757
956
|
if (!pool) return next();
|
|
758
957
|
const code = String(payload?.failure?.code ?? payload?.code ?? '');
|
|
759
958
|
const message = String(payload?.failure?.message ?? payload?.message ?? '');
|
package/lib/pool.js
CHANGED
|
@@ -176,3 +176,27 @@ export function parseRetryAfter(value) {
|
|
|
176
176
|
}
|
|
177
177
|
return undefined;
|
|
178
178
|
}
|
|
179
|
+
|
|
180
|
+
/** Pick a key pool for a (provider, model) pair. Model sub-pools win over the
|
|
181
|
+
* provider base pool; falls back to the base pool when no sub-pool matches. */
|
|
182
|
+
export function selectPool(modelPoolByProvider, providerToPool, provider, model) {
|
|
183
|
+
const byModel = modelPoolByProvider && modelPoolByProvider.get(provider);
|
|
184
|
+
return (byModel && byModel.get(model)) || (providerToPool && providerToPool.get(provider)) || null;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Parse an expiry value (timestamp ms or ISO date string) to epoch ms. */
|
|
188
|
+
export function parseExpiry(v) {
|
|
189
|
+
if (typeof v === 'number' && v > 0) return v;
|
|
190
|
+
if (typeof v === 'string' && v.length > 0) { const ts = Date.parse(v); return Number.isNaN(ts) ? undefined : ts; }
|
|
191
|
+
return undefined;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Compute a 0..100 health score for a pool based on its runtime state.
|
|
195
|
+
* Deductions: switches * 5, exhaustions * 10, broken keys * 15. */
|
|
196
|
+
export function computeHealthScore(state) {
|
|
197
|
+
if (!state || typeof state !== 'object') return 100;
|
|
198
|
+
const switches = state.switches ?? 0;
|
|
199
|
+
const exhaustions = state.exhaustionCount ?? 0;
|
|
200
|
+
const broken = state.brokenUntil ? state.brokenUntil.size : 0;
|
|
201
|
+
return Math.max(0, Math.min(100, 100 - (switches * 5) - (exhaustions * 10) - (broken * 15)));
|
|
202
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-key-rotation",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.12",
|
|
4
4
|
"description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"deepseek-harness",
|
|
@@ -54,8 +54,5 @@
|
|
|
54
54
|
},
|
|
55
55
|
"scripts": {
|
|
56
56
|
"test": "node --test test/*.test.js"
|
|
57
|
-
},
|
|
58
|
-
"publishConfig": {
|
|
59
|
-
"access": "public"
|
|
60
57
|
}
|
|
61
58
|
}
|