@goodandready/dsh-key-rotation 0.7.0 → 0.7.1

package/lib/client.js

@@ -60,7 +60,11 @@ window.__ModuleLoader__.load({
       keyFromEnv: 'from the environment, read-only here',
       keyWriteFailed: 'could not store the key: {msg}',
       keyHint: 'The value is stored in DSH credentials and never sent back to the browser — only its last 5 characters are shown. Names are generated for you; hover a key to see the one it uses.',
+      brokenKey: 'broken (3× AUTH)',
       resetCooldown: 'Reset cooldown',
+      testKey: 'Test',
+      testOk: 'OK',
+      testFail: 'FAIL',
       poolExhausted: 'pool exhausted — all keys cooling',
       resetting: 'Resetting…',
       keyLabel: 'Key {n}',
@@ -100,7 +104,11 @@ window.__ModuleLoader__.load({
       keyFromEnv: 'задан в окружении, отсюда не меняется',
       keyWriteFailed: 'не удалось сохранить ключ: {msg}',
       keyHint: 'Значение хранится в учётных данных DSH и обратно в браузер не отдаётся — показываются только последние 5 символов. Имена переменных создаются автоматически; наведите на ключ, чтобы увидеть используемое имя.',
+      brokenKey: 'сломан (3× AUTH)',
       resetCooldown: 'Сбросить кулдаун',
+      testKey: 'Тест',
+      testOk: 'OK',
+      testFail: 'FAIL',
       poolExhausted: 'пул исчерпан — все ключи остывают',
       resetting: 'Сброс…',
       keyLabel: 'Ключ {n}',
@@ -266,6 +274,16 @@ window.__ModuleLoader__.load({
           .catch((e) => setSecretError(t('keyWriteFailed').replace('{msg}', String(e?.message ?? e))))
           .finally(() => setResetting(''));
       };
+      const [testing, setTesting] = React.useState('');
+      const [testResult, setTestResult] = React.useState({});
+      const doTest = (ref) => {
+        setTesting(ref); setTestResult((m) => ({ ...m, [ref]: null }));
+        fetch('/dsh-key-rotation/test', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ ref }) })
+          .then((r) => r.json())
+          .then((data) => setTestResult((m) => ({ ...m, [ref]: data })))
+          .catch((e) => setTestResult((m) => ({ ...m, [ref]: { ok: false, message: String(e?.message ?? e) } })))
+          .finally(() => setTesting(''));
+      };
 
       const keyInfo = (providerId, ref) => {
         const entryStatus = status[providerId];
@@ -400,6 +418,7 @@ window.__ModuleLoader__.load({
       const keyStatus = (providerId, ref) => {
         const hit = keyInfo(providerId, ref);
         if (!hit) return null;
+        if (hit.broken) return { color: 'var(--dsw-alias-state-error-primary)', text: t('brokenKey') };
         if (!hit.present) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyMissing') };
         if (hit.cooldownMsLeft > 0) {
           return {
@@ -453,6 +472,10 @@ window.__ModuleLoader__.load({
             }));
             if (typed) meta.push(btn('✓', () => saveSecret(key, rowKey), { title: t('keySave'), key: 'w' }));
           }
+          if (info && typeof info.usage === 'number' && info.usage > 0) meta.push(h('span', { key: 'u', className: 'krot-tail', title: 'requests through this key' }, String(info.usage)));
+          const tr = testResult[key];
+          if (tr) meta.push(h('span', { key: 'tr', className: 'krot-tail', title: tr.message || (tr.ok ? t('testOk') : t('testFail')) }, tr.ok ? '✓' : '✕'));
+          meta.push(h('button', { key: 't', className: 'krot-btn', onClick: () => doTest(key), disabled: testing === key, title: t('testKey') }, testing === key ? '…' : t('testKey')));
           meta.push(h('span', { key: 'a', className: 'krot-acts' },
             btn('↑', () => moveKey(pIndex, kIndex, -1), { disabled: kIndex === 0, title: t('moveUp') }),
             btn('↓', () => moveKey(pIndex, kIndex, 1), { disabled: kIndex === keys.length - 1, title: t('moveDown') }),

package/lib/index.js

@@ -33,7 +33,7 @@
 //    providers:  array       [{ provider, keys: [envName, ...] }]
 // ─────────────────────────────────────────────────────────────────────────────
 import Schema from '@deepseek-ai/schemastery';
-import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired } from './pool.js';
+import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter } from './pool.js';
 
 export const name = 'dsh-key-rotation';
 export const inject = ['llm', 'webServer', 'settings', 'credentials'];
@@ -46,6 +46,7 @@ const CONFIG_PATH = '/dsh-key-rotation/config';
 const STATUS_PATH = '/dsh-key-rotation/status';
 const KEY_PATH = '/dsh-key-rotation/key';
 const RESET_PATH = '/dsh-key-rotation/reset';
+const TEST_PATH = '/dsh-key-rotation/test';
 
 /** The llm-pi-ai namespace whose provider profiles map providers to pools. */
 const PIAI_NS = 'llm-pi-ai';
@@ -78,9 +79,12 @@ export const Config = Schema.object({
   switchCodes: Schema.array(Schema.string()).default([...DEFAULT_SWITCH_CODES]),
   cooldownMs: Schema.number().default(60000),
   maxCooldownMs: Schema.number(),
+  notifyWebhook: Schema.string().default(''),
+  notifyThreshold: Schema.number().default(3),
   providers: Schema.array(Schema.object({
     provider: Schema.string().required(),
     keys: Schema.array(Schema.string()).default([]),
+    weights: Schema.array(Schema.number()).default([]),
     cooldownMs: Schema.number(),
     maxCooldownMs: Schema.number(),
   })).default([...DEFAULT_PROVIDERS]),
@@ -260,6 +264,8 @@ export function apply(ctx, config = {}) {
     const switchCodes = new Set(cfg.switchCodes ?? DEFAULT_SWITCH_CODES);
     const cooldownMs = cfg.cooldownMs ?? 60000;
     const maxCooldownMs = cfg.maxCooldownMs ?? undefined;
+    const notifyWebhook = cfg.notifyWebhook ?? '';
+    const notifyThreshold = cfg.notifyThreshold ?? 3;
 
     // ref -> pool (every key env of every configured provider)
     const poolByRef = new Map();
@@ -286,12 +292,19 @@ export function apply(ctx, config = {}) {
           lastExhaustionAt: undefined,
           exhaustionCount: 0,
           events: [],
+          usageCounts: new Map(),
         };
         poolState.set(p.provider, state);
       }
       const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
       const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
-      const pool = { base: p.provider, refs, state, cooldownMs: poolCooldown, maxCooldownMs: poolMax };
+      const weights = Array.isArray(p.weights) ? p.weights : [];
+      const weightedRefs = [];
+      for (let i = 0; i < refs.length; i++) {
+        const w = typeof weights[i] === 'number' && weights[i] > 0 ? Math.floor(weights[i]) : 1;
+        for (let k = 0; k < w; k++) weightedRefs.push(refs[i]);
+      }
+      const pool = { base: p.provider, refs, weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs, state, cooldownMs: poolCooldown, maxCooldownMs: poolMax };
       for (const ref of refs) poolByRef.set(ref, pool);
       for (let i = 1; i < refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
     }
@@ -308,7 +321,7 @@ export function apply(ctx, config = {}) {
       }
     }
 
-    return { switchCodes, cooldownMs, maxCooldownMs, poolByRef, providerToPool, cloneIds };
+    return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, poolByRef, providerToPool, cloneIds };
   }
 
   // ── patch credentials.resolve: pool refs resolve to the next healthy key ──
@@ -325,27 +338,61 @@ export function apply(ctx, config = {}) {
       const pool = poolByRef.get(ref);
       if (!pool) return original(ref);
       const now = Date.now();
+      const list = pool.weightedRefs ?? pool.refs;
       const start = pool.state.pointer ?? 0;
-      for (let i = 0; i < pool.refs.length; i++) {
-        const index = (start + i) % pool.refs.length;
-        const candidate = pool.refs[index];
+      for (let i = 0; i < list.length; i++) {
+        const index = (start + i) % list.length;
+        const candidate = list[index];
         const until = pool.state.failedUntil.get(candidate);
         if (until !== undefined && until > now) continue;
+        // perHour quota check
+        if (pool.perHour) {
+          if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+          let win = pool.state.quotaWindows.get(candidate);
+          if (!win || now - win.start >= 3600000) win = { count: 0, start: now };
+          if (win.count >= pool.perHour) {
+            const until = win.start + 3600000;
+            if ((pool.state.failedUntil.get(candidate) ?? 0) < until) pool.state.failedUntil.set(candidate, until);
+            continue;
+          }
+        }
         let hit = await original(candidate);
         if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
-          pool.state.pointer = (index + 1) % pool.refs.length;
+          pool.state.pointer = (index + 1) % list.length;
           pool.state.lastUsed = candidate;
           if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
           pool.state.failedUntil.delete(candidate);
+          if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
+          if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
+          if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
+          pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
+          if (pool.perHour) {
+            if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+            let win2 = pool.state.quotaWindows.get(candidate);
+            if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
+            win2.count++;
+            pool.state.quotaWindows.set(candidate, win2);
+          }
           return hit;
         }
         // fallback: env var (transient, not persisted)
         const envVal = envValue(candidate);
         if (envVal !== undefined) {
-          pool.state.pointer = (index + 1) % pool.refs.length;
+          pool.state.pointer = (index + 1) % list.length;
           pool.state.lastUsed = candidate;
           if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
           pool.state.failedUntil.delete(candidate);
+          if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
+          if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
+          if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
+          pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
+          if (pool.perHour) {
+            if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+            let win2 = pool.state.quotaWindows.get(candidate);
+            if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
+            win2.count++;
+            pool.state.quotaWindows.set(candidate, win2);
+          }
           return { value: envVal, source: 'env' };
         }
       }
@@ -367,7 +414,7 @@ export function apply(ctx, config = {}) {
       const { switchCodes, cooldownMs, maxCooldownMs } = buildRuntime();
       let lastFailure = null;
 
-      for (let attempt = 0; attempt < pool.refs.length; attempt++) {
+      for (let attempt = 0; attempt < (pool.weightedRefs ?? pool.refs).length; attempt++) {
         let yielded = false;
         let switching = false;
         let inner;
@@ -375,7 +422,7 @@ export function apply(ctx, config = {}) {
           // mark the internal dispatch so the interceptor does not re-rotate
           inner = ctx.llm.stream({ ...options, [MARKER]: true });
         } catch (e) {
-          if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), pool.cooldownMs ?? cooldownMs, pool.maxCooldownMs ?? maxCooldownMs); pushEvent(pool, pool.state.lastUsed, e?.code ?? 'TRANSPORT', _b); }
+          if (pool.state.lastUsed) { const _retry = parseRetryAfter(String(e?.message ?? '')); const _base = pool.cooldownMs ?? cooldownMs; const _max = pool.maxCooldownMs ?? maxCooldownMs; const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base; const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max); pushEvent(pool, pool.state.lastUsed, e?.code ?? 'TRANSPORT', _b); const _code = String(e?.code ?? ''); if (_code === 'AUTH' || /auth/i.test(String(e?.message ?? ''))) { const _c = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1; pool.state.authFailCounts.set(pool.state.lastUsed, _c); if (_c >= 3) { pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); } } else { pool.state.authFailCounts.delete(pool.state.lastUsed); } }
           lastFailure = finishError(e?.code ?? 'TRANSPORT',
             `dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
           console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(pool.state.lastUsed ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
@@ -399,7 +446,7 @@ export function apply(ctx, config = {}) {
               const switchable = !yielded && kind === 'error' &&
                 (switchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message));
               if (switchable) {
-                if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), pool.cooldownMs ?? cooldownMs, pool.maxCooldownMs ?? maxCooldownMs); pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b); }
+                if (pool.state.lastUsed) { const _retry = parseRetryAfter(message); const _base = pool.cooldownMs ?? cooldownMs; const _max = pool.maxCooldownMs ?? maxCooldownMs; const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base; const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max); pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b); const _code2 = String(code ?? ''); if (_code2 === 'AUTH' || /auth/i.test(message)) { const _c2 = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1; pool.state.authFailCounts.set(pool.state.lastUsed, _c2); if (_c2 >= 3) { pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); } } else { pool.state.authFailCounts.delete(pool.state.lastUsed); } }
                 pool.state.switches = (pool.state.switches ?? 0) + 1;
                 pool.state.lastReason = String(code ?? 'UNKNOWN');
                 pool.state.lastSwitchAt = Date.now();
@@ -426,6 +473,15 @@ export function apply(ctx, config = {}) {
       pool.state.lastExhaustionAt = Date.now();
       pool.state.exhaustionCount = (pool.state.exhaustionCount ?? 0) + 1;
       console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — all ${pool.refs.length} keys cooling`);
+      // notify webhook if configured and threshold reached
+      try {
+        const { notifyWebhook, notifyThreshold } = buildRuntime();
+        if (notifyWebhook && pool.state.exhaustionCount >= notifyThreshold) {
+          const payload = JSON.stringify({ provider: options.provider, exhaustionCount: pool.state.exhaustionCount, at: pool.state.lastExhaustionAt, keys: pool.refs });
+          fetch(notifyWebhook, { method: 'POST', headers: { 'content-type': 'application/json' }, body: payload }).catch(()=>{});
+        }
+      } catch {}
+
       yield lastFailure ?? finishError('TRANSPORT', 'dsh-key-rotation: all keys failed');
     })();
   }
@@ -494,6 +550,8 @@ export function apply(ctx, config = {}) {
             writable,
             active: pool.state.lastUsed === ref,
             cooldownMsLeft: until !== undefined && until > now ? until - now : 0,
+            usage: pool.state.usageCounts?.get(ref) ?? 0,
+            broken: pool.state.brokenUntil?.has(ref) ?? false,
           });
         }
         providers.push({
@@ -594,6 +652,8 @@ export function apply(ctx, config = {}) {
         const cleared = st.failedUntil.size;
         st.failedUntil.clear();
         st.failCounts?.clear();
+        st.authFailCounts?.clear();
+        st.brokenUntil?.clear();
         st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
         json(res, 200, { ok: true, provider, cleared });
         return;
@@ -604,6 +664,8 @@ export function apply(ctx, config = {}) {
           if (st.failedUntil.has(ref) || st.failCounts?.has(ref)) {
             st.failedUntil.delete(ref);
             st.failCounts?.delete(ref);
+            st.authFailCounts?.delete(ref);
+            st.brokenUntil?.delete(ref);
             if (st.lastUsed === ref) st.lastUsed = undefined;
             found = true; break;
           }
@@ -617,6 +679,35 @@ export function apply(ctx, config = {}) {
     },
   }), 'dsh-key-rotation: reset route');
 
+  // ── test route: dry-run a single key without rotation ──
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: TEST_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: test is local-only' } }); return; }
+      let body; try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
+      const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
+      if (!isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
+      const base = ctx.get('credentials');
+      try {
+        const hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
+        const present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
+        if (!present) { json(res, 200, { ok: false, ref, code: 'no-credential', message: 'no such credential' }); return; }
+        const tail = keyTail(hit.value);
+        // Check env source
+        let source = null; try { const d = await base?.describe?.(ref); source = d?.source ?? null; } catch {}
+        if (!present) {
+          const ev = envValue(ref);
+          if (ev !== undefined) json(res, 200, { ok: true, ref, tail: keyTail(ev), source: 'env' });
+        }
+        json(res, 200, { ok: true, ref, tail, source });
+      } catch (e) {
+        json(res, 200, { ok: false, ref, code: 'error', message: String(e?.message ?? e) });
+      }
+    },
+  }), 'dsh-key-rotation: test route');
+
   // Intercept the llm/stream waterfall: rotate any request whose provider maps
   // to a configured key pool; pass everything else (and internal dispatches)
   // straight through.
@@ -625,7 +716,7 @@ export function apply(ctx, config = {}) {
     const { providerToPool } = buildRuntime();
     const pool = providerToPool.get(options.provider);
     if (!pool) return next();
-    console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${pool.refs.length} keys`);
+    console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${(pool.weightedRefs ?? pool.refs).length} slots (${pool.refs.length} keys)`);
     return rotate(options, pool);
   });
 

package/lib/pool.js

@@ -154,3 +154,25 @@ export function sweepExpired(poolState, now = Date.now()) {
   }
   return cleared;
 }
+
+/** Parse Retry-After value from a header string or message.
+ *  Supports seconds ("60", "Retry-After: 60") and HTTP-date ("Wed, 21 Oct 2026 07:28:00 GMT").
+ *  Returns milliseconds or undefined if not parseable. */
+export function parseRetryAfter(value) {
+  if (typeof value !== 'string' || !value) return undefined;
+  // Try to extract "Retry-After: <val>" from a larger message
+  const m = value.match(/retry-after\s*[:=]\s*(.+)/i);
+  const raw = m ? m[1].trim().split(/[\n\r;]/)[0].trim() : value.trim();
+  // Seconds
+  if (/^\d+$/.test(raw)) {
+    const sec = Number(raw);
+    if (sec >= 0 && sec <= 86400 * 7) return sec * 1000;
+  }
+  // HTTP-date
+  const ts = Date.parse(raw);
+  if (!Number.isNaN(ts)) {
+    const diff = ts - Date.now();
+    if (diff > 0 && diff < 86400 * 7 * 1000) return diff;
+  }
+  return undefined;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.7.0",
+  "version": "0.7.1",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
     "deepseek-harness",