@goodandready/dsh-key-rotation 0.5.3 → 0.5.4

package/lib/client.js

@@ -60,6 +60,9 @@ window.__ModuleLoader__.load({
       keyFromEnv: 'from the environment, read-only here',
       keyWriteFailed: 'could not store the key: {msg}',
       keyHint: 'The value is stored in DSH credentials and never sent back to the browser — only its last 5 characters are shown. Names are generated for you; hover a key to see the one it uses.',
+      resetCooldown: 'Reset cooldown',
+      poolExhausted: 'pool exhausted — all keys cooling',
+      resetting: 'Resetting…',
       keyLabel: 'Key {n}',
     };
     const ru = {
@@ -97,6 +100,9 @@ window.__ModuleLoader__.load({
       keyFromEnv: 'задан в окружении, отсюда не меняется',
       keyWriteFailed: 'не удалось сохранить ключ: {msg}',
       keyHint: 'Значение хранится в учётных данных DSH и обратно в браузер не отдаётся — показываются только последние 5 символов. Имена переменных создаются автоматически; наведите на ключ, чтобы увидеть используемое имя.',
+      resetCooldown: 'Сбросить кулдаун',
+      poolExhausted: 'пул исчерпан — все ключи остывают',
+      resetting: 'Сброс…',
       keyLabel: 'Ключ {n}',
     };
 
@@ -250,6 +256,16 @@ window.__ModuleLoader__.load({
       // хоста они не приходят, в карточке видны лишь последние символы.
       const [secretDraft, setSecretDraft] = React.useState({});
       const [secretError, setSecretError] = React.useState('');
+      const [resetting, setResetting] = React.useState('');
+      const doReset = (providerId) => {
+        setResetting(providerId);
+        setSecretError('');
+        fetch('/dsh-key-rotation/reset', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ provider: providerId }) })
+          .then((r) => r.json().then((data) => ({ ok: r.ok, data })))
+          .then(({ ok, data }) => { if (!ok) throw new Error(data?.error?.message ?? 'unknown error'); })
+          .catch((e) => setSecretError(t('keyWriteFailed').replace('{msg}', String(e?.message ?? e))))
+          .finally(() => setResetting(''));
+      };
 
       const keyInfo = (providerId, ref) => {
         const entryStatus = status[providerId];
@@ -457,6 +473,9 @@ window.__ModuleLoader__.load({
                 .replace('{reason}', String(providerStatus.lastReason || '—'))
                 .replace('{ago}', formatAgo(t, providerStatus.lastSwitchAt))
             : t('switchesNone'));
+        const exhaustionWarning = providerStatus && providerStatus.lastExhaustionAt && (Date.now() - providerStatus.lastExhaustionAt) < 3600000
+          ? h('p', { className: 'krot-err' }, t('poolExhausted') + ' (' + formatAgo(t, providerStatus.lastExhaustionAt) + ')')
+          : null;
 
         return h('div', { key: pIndex, className: 'krot-prov' },
           h('div', { className: 'krot-prov-head' },
@@ -467,6 +486,9 @@ window.__ModuleLoader__.load({
           h('div', { className: 'krot-foot' },
             btn(t('addKey'), () => addKey(pIndex), { title: t('addKeyTitle') }),
             switchesLine,
+            exhaustionWarning,
+            (providerStatus && Array.isArray(providerStatus.events) && providerStatus.events.length > 0 ? h('details', { style: { fontSize: '11px', marginTop: '6px' } }, h('summary', null, 'Recent failures ('+providerStatus.events.length+')'), h('ul', { style: { margin: '4px 0 0', paddingLeft: '16px' } }, providerStatus.events.slice().reverse().map((ev, i) => h('li', { key: i }, new Date(ev.at).toLocaleTimeString() + ' ' + ev.ref + ' ' + ev.reason + ' cd=' + ev.cooldownMs)) )) : null),
+            btn(t('resetCooldown'), () => doReset(entry.provider), { disabled: !(providerStatus && providerStatus.switches > 0) || resetting === entry.provider, title: t('resetCooldown') }),
           ),
         );
       });

package/lib/index.js

@@ -33,7 +33,7 @@
 //    providers:  array       [{ provider, keys: [envName, ...] }]
 // ─────────────────────────────────────────────────────────────────────────────
 import Schema from '@deepseek-ai/schemastery';
-import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown } from './pool.js';
+import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue } from './pool.js';
 
 export const name = 'dsh-key-rotation';
 export const inject = ['llm', 'webServer', 'settings', 'credentials'];
@@ -45,11 +45,18 @@ const NS = 'dsh-key-rotation';
 const CONFIG_PATH = '/dsh-key-rotation/config';
 const STATUS_PATH = '/dsh-key-rotation/status';
 const KEY_PATH = '/dsh-key-rotation/key';
+const RESET_PATH = '/dsh-key-rotation/reset';
 
 /** The llm-pi-ai namespace whose provider profiles map providers to pools. */
 const PIAI_NS = 'llm-pi-ai';
 /** Marker on internally re-dispatched requests so the interceptor does not loop. */
 const MARKER = '__dshKeyRotation';
+const MAX_EVENTS = 50;
+function pushEvent(pool, ref, reason, cooldownMs) {
+  const ev = { at: Date.now(), ref, reason: String(reason ?? 'UNKNOWN'), cooldownMs };
+  pool.state.events.push(ev);
+  if (pool.state.events.length > MAX_EVENTS) pool.state.events.shift();
+}
 
 
 // Fallback classification by failure message. pi-ai surfaces many real quota /
@@ -73,6 +80,7 @@ export const Config = Schema.object({
   providers: Schema.array(Schema.object({
     provider: Schema.string().required(),
     keys: Schema.array(Schema.string()).default([]),
+    cooldownMs: Schema.number().optional(),
   })).default([...DEFAULT_PROVIDERS]),
 });
 
@@ -256,6 +264,7 @@ export function apply(ctx, config = {}) {
       if (!state) {
         state = {
           failedUntil: new Map(),
+          failCounts: new Map(),
           pointer: 0,
           lastUsed: undefined,
           // Счётчики для карточки настроек: без них о работе ротации можно было
@@ -263,10 +272,14 @@ export function apply(ctx, config = {}) {
           switches: 0,
           lastReason: undefined,
           lastSwitchAt: undefined,
+          lastExhaustionAt: undefined,
+          exhaustionCount: 0,
+          events: [],
         };
         poolState.set(p.provider, state);
       }
-      const pool = { base: p.provider, refs, state };
+      const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
+      const pool = { base: p.provider, refs, state, cooldownMs: poolCooldown };
       for (const ref of refs) poolByRef.set(ref, pool);
       for (let i = 1; i < refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
     }
@@ -306,12 +319,23 @@ export function apply(ctx, config = {}) {
         const candidate = pool.refs[index];
         const until = pool.state.failedUntil.get(candidate);
         if (until !== undefined && until > now) continue;
-        const hit = await original(candidate);
+        let hit = await original(candidate);
         if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
           pool.state.pointer = (index + 1) % pool.refs.length;
           pool.state.lastUsed = candidate;
+          if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
+          pool.state.failedUntil.delete(candidate);
           return hit;
         }
+        // fallback: env var (transient, not persisted)
+        const envVal = envValue(candidate);
+        if (envVal !== undefined) {
+          pool.state.pointer = (index + 1) % pool.refs.length;
+          pool.state.lastUsed = candidate;
+          if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
+          pool.state.failedUntil.delete(candidate);
+          return { value: envVal, source: 'env' };
+        }
       }
       return original(ref); // everything cooled/missing — surface the base value
     };
@@ -339,7 +363,7 @@ export function apply(ctx, config = {}) {
           // mark the internal dispatch so the interceptor does not re-rotate
           inner = ctx.llm.stream({ ...options, [MARKER]: true });
         } catch (e) {
-          if (pool.state.lastUsed) pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + cooldownMs);
+          if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), pool.cooldownMs ?? cooldownMs); pushEvent(pool, pool.state.lastUsed, e?.code ?? 'TRANSPORT', _b); }
           lastFailure = finishError(e?.code ?? 'TRANSPORT',
             `dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
           console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(pool.state.lastUsed ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
@@ -363,7 +387,7 @@ export function apply(ctx, config = {}) {
               const switchable = !yielded && kind === 'error' &&
                 (switchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message));
               if (switchable) {
-                if (pool.state.lastUsed) pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + cooldownMs);
+                if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), pool.cooldownMs ?? cooldownMs); pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b); }
                 pool.state.switches = (pool.state.switches ?? 0) + 1;
                 pool.state.lastReason = String(code ?? 'UNKNOWN');
                 pool.state.lastSwitchAt = Date.now();
@@ -386,6 +410,10 @@ export function apply(ctx, config = {}) {
         return; // clean end — served
       }
 
+      // pool exhausted — all keys cooling or missing
+      pool.state.lastExhaustionAt = Date.now();
+      pool.state.exhaustionCount = (pool.state.exhaustionCount ?? 0) + 1;
+      console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — all ${pool.refs.length} keys cooling`);
       yield lastFailure ?? finishError('TRANSPORT', 'dsh-key-rotation: all keys failed');
     })();
   }
@@ -427,11 +455,14 @@ export function apply(ctx, config = {}) {
             // The resolve patch is installed on this same service, so ask for
             // the exact ref: a pool ref would otherwise round-robin to another
             // key and report a missing name as present.
-            const hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
+            let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
             present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
-            // Only the last few characters travel to the browser: enough to
-            // tell two keys apart in the card, useless for authenticating.
             if (present) tail = keyTail(hit.value);
+            // fallback: env var bootstrapping (issue #7)
+            if (!present) {
+              const ev = envValue(ref);
+              if (ev !== undefined) { present = true; tail = keyTail(ev); source = 'env'; writable = false; }
+            }
           } catch {
             present = false;
           }
@@ -459,6 +490,9 @@ export function apply(ctx, config = {}) {
           switches: pool.state.switches ?? 0,
           lastReason: pool.state.lastReason ?? null,
           lastSwitchAt: pool.state.lastSwitchAt ?? null,
+          lastExhaustionAt: pool.state.lastExhaustionAt ?? null,
+          exhaustionCount: pool.state.exhaustionCount ?? 0,
+          events: (pool.state.events ?? []).slice(-20),
         });
       }
       json(res, 200, { providers });
@@ -522,6 +556,55 @@ export function apply(ctx, config = {}) {
     },
   }), 'dsh-key-rotation: key route');
 
+  // ── reset route: clear cooldown for a provider (or a single ref) ──
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: RESET_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'POST') {
+        json(res, 405, { error: { code: 'method', message: 'POST only' } });
+        return;
+      }
+      if (!isTrustedBridgeRequest(req)) {
+        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: reset is local-only' } });
+        return;
+      }
+      let body;
+      try { body = await readJson(req); } catch (e) {
+        json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } });
+        return;
+      }
+      const provider = typeof body?.provider === 'string' ? body.provider.trim() : '';
+      const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
+      if (provider) {
+        const st = poolState.get(provider);
+        if (!st) { json(res, 404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }); return; }
+        const cleared = st.failedUntil.size;
+        st.failedUntil.clear();
+        st.failCounts?.clear();
+        st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
+        json(res, 200, { ok: true, provider, cleared });
+        return;
+      }
+      if (ref) {
+        let found = false;
+        for (const st of poolState.values()) {
+          if (st.failedUntil.has(ref) || st.failCounts?.has(ref)) {
+            st.failedUntil.delete(ref);
+            st.failCounts?.delete(ref);
+            if (st.lastUsed === ref) st.lastUsed = undefined;
+            found = true; break;
+          }
+        }
+        // idempotent: even if ref was not cooling, report ok if it looks like a valid ref name
+        if (!found && !isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
+        json(res, 200, { ok: true, ref });
+        return;
+      }
+      json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: POST requires {"provider": "..."} or {"ref": "..."}' } });
+    },
+  }), 'dsh-key-rotation: reset route');
+
   // Intercept the llm/stream waterfall: rotate any request whose provider maps
   // to a configured key pool; pass everything else (and internal dispatches)
   // straight through.

package/lib/pool.js

@@ -105,3 +105,37 @@ export function applyCooldown(pool, ref, cooldownMs, now = Date.now()) {
     },
   };
 }
+
+/** Exponential backoff for a repeatedly failing key.
+ *  failCount 1 => baseMs, 2 => baseMs*2, 3 => baseMs*4, capped at baseMs*8 (or maxMs).
+ *  Pure and easily unit-tested. */
+export function computeBackoff(baseMs, failCount, maxMs) {
+  const cap = maxMs ?? baseMs * 8;
+  if (failCount <= 1) return Math.min(baseMs, cap);
+  const backoff = baseMs * (1 << (failCount - 1)); // 2^(n-1)
+  return Math.min(backoff, cap);
+}
+
+/** Record a failure for `ref` in `pool.state`, applying exponential backoff.
+ *  Mutates pool.state.failedUntil and pool.state.failCounts. Returns the backoff used. */
+export function recordFailure(pool, ref, now, baseMs, maxMs) {
+  if (!pool.state.failCounts) pool.state.failCounts = new Map();
+  const prev = pool.state.failCounts.get(ref) ?? 0;
+  const next = prev + 1;
+  pool.state.failCounts.set(ref, next);
+  const backoff = computeBackoff(baseMs, next, maxMs);
+  pool.state.failedUntil.set(ref, now + backoff);
+  return backoff;
+}
+
+/** Record a success for `ref` — clears its cooldown and resets its fail count. */
+export function recordSuccess(pool, ref) {
+  if (pool.state.failCounts) pool.state.failCounts.delete(ref);
+  pool.state.failedUntil.delete(ref);
+}
+
+/** Return env value for ref if present in process.env, else undefined. */
+export function envValue(ref) {
+  const v = typeof process !== 'undefined' ? process.env?.[ref] : undefined;
+  return typeof v === 'string' && v.length > 0 ? v : undefined;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.5.3",
+  "version": "0.5.4",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
     "deepseek-harness",