@goodandready/dsh-key-rotation 0.6.1 → 0.7.0

package/README.md

@@ -17,6 +17,13 @@
   - **rotation counter** — how many times a provider switched key, on which failure, and how long ago.
   - **key order** — ↑/↓ buttons; the order of keys is the order they are tried.
   - **switch codes as checkboxes** instead of a comma-separated string.
+  - **Exponential backoff** — repeated failures on the same key double its cooldown (base → ×2 → ×4 → cap ×8), so a dead key is not retried every window.
+  - **Reset cooldown** — a *Reset cooldown* button in the card clears a provider's cooldown immediately (also via `POST /dsh-key-rotation/reset`).
+  - **Env bootstrap** — if a pool ref (e.g. `MYPROVIDER_API_KEY`) is already set in `process.env`, it is treated as a transient credential without needing a DSH credential first.
+  - **Per-provider cooldown** — override `cooldownMs` (and `maxCooldownMs`) per provider, fallback to the global values.
+  - **Exhaustion warning** — when every key is cooling, a red warning appears in the card and `lastExhaustionAt`/`exhaustionCount` are exposed via `GET /dsh-key-rotation/status`.
+  - **Failure log** — last 20 failures per provider (`at`, `ref`, `reason`, `cooldownMs`) via `/status` and a collapsible *Recent failures* list.
+  - **Non-stream safety net** — an `agent/request-error` hook retries sync calls (embeddings, batch) with the next key when the error is switchable.
 
 ## Install
 

package/lib/index.js

@@ -33,7 +33,7 @@
 //    providers:  array       [{ provider, keys: [envName, ...] }]
 // ─────────────────────────────────────────────────────────────────────────────
 import Schema from '@deepseek-ai/schemastery';
-import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue } from './pool.js';
+import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired } from './pool.js';
 
 export const name = 'dsh-key-rotation';
 export const inject = ['llm', 'webServer', 'settings', 'credentials'];
@@ -77,10 +77,12 @@ const DEFAULT_PROVIDERS = [];
 export const Config = Schema.object({
   switchCodes: Schema.array(Schema.string()).default([...DEFAULT_SWITCH_CODES]),
   cooldownMs: Schema.number().default(60000),
+  maxCooldownMs: Schema.number(),
   providers: Schema.array(Schema.object({
     provider: Schema.string().required(),
     keys: Schema.array(Schema.string()).default([]),
     cooldownMs: Schema.number(),
+    maxCooldownMs: Schema.number(),
   })).default([...DEFAULT_PROVIDERS]),
 });
 
@@ -241,6 +243,14 @@ export function apply(ctx, config = {}) {
   // ── key-pool state, persisted across config reloads ──
   // base provider -> { failedUntil: Map<ref, epochMs>, pointer: number, lastUsed: ref }
   const poolState = new Map();
+  // Periodic sweep of expired cooldowns — keeps health probe cheap and avoids waiting for next user request
+  ctx.effect(() => {
+    const id = setInterval(() => {
+      const n = sweepExpired(poolState, Date.now());
+      if (n > 0) console.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
+    }, 30000);
+    return () => clearInterval(id);
+  }, 'dsh-key-rotation: sweep expired cooldowns');
 
   // ── runtime snapshot: config + llm-pi-ai profile mapping ──
   function buildRuntime() {
@@ -249,6 +259,7 @@ export function apply(ctx, config = {}) {
     const cfg = Config(structuredClone(getConfig() ?? {})) ?? {};
     const switchCodes = new Set(cfg.switchCodes ?? DEFAULT_SWITCH_CODES);
     const cooldownMs = cfg.cooldownMs ?? 60000;
+    const maxCooldownMs = cfg.maxCooldownMs ?? undefined;
 
     // ref -> pool (every key env of every configured provider)
     const poolByRef = new Map();
@@ -279,7 +290,8 @@ export function apply(ctx, config = {}) {
         poolState.set(p.provider, state);
       }
       const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
-      const pool = { base: p.provider, refs, state, cooldownMs: poolCooldown };
+      const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
+      const pool = { base: p.provider, refs, state, cooldownMs: poolCooldown, maxCooldownMs: poolMax };
       for (const ref of refs) poolByRef.set(ref, pool);
       for (let i = 1; i < refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
     }
@@ -296,7 +308,7 @@ export function apply(ctx, config = {}) {
       }
     }
 
-    return { switchCodes, cooldownMs, poolByRef, providerToPool, cloneIds };
+    return { switchCodes, cooldownMs, maxCooldownMs, poolByRef, providerToPool, cloneIds };
   }
 
   // ── patch credentials.resolve: pool refs resolve to the next healthy key ──
@@ -352,7 +364,7 @@ export function apply(ctx, config = {}) {
   // the resolve patch hands out the next key on each dispatch.
   function rotate(options, pool) {
     return (async function* () {
-      const { switchCodes, cooldownMs } = buildRuntime();
+      const { switchCodes, cooldownMs, maxCooldownMs } = buildRuntime();
       let lastFailure = null;
 
       for (let attempt = 0; attempt < pool.refs.length; attempt++) {
@@ -363,7 +375,7 @@ export function apply(ctx, config = {}) {
           // mark the internal dispatch so the interceptor does not re-rotate
           inner = ctx.llm.stream({ ...options, [MARKER]: true });
         } catch (e) {
-          if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), pool.cooldownMs ?? cooldownMs); pushEvent(pool, pool.state.lastUsed, e?.code ?? 'TRANSPORT', _b); }
+          if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), pool.cooldownMs ?? cooldownMs, pool.maxCooldownMs ?? maxCooldownMs); pushEvent(pool, pool.state.lastUsed, e?.code ?? 'TRANSPORT', _b); }
           lastFailure = finishError(e?.code ?? 'TRANSPORT',
             `dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
           console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(pool.state.lastUsed ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
@@ -387,7 +399,7 @@ export function apply(ctx, config = {}) {
               const switchable = !yielded && kind === 'error' &&
                 (switchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message));
               if (switchable) {
-                if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), pool.cooldownMs ?? cooldownMs); pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b); }
+                if (pool.state.lastUsed) { const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), pool.cooldownMs ?? cooldownMs, pool.maxCooldownMs ?? maxCooldownMs); pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b); }
                 pool.state.switches = (pool.state.switches ?? 0) + 1;
                 pool.state.lastReason = String(code ?? 'UNKNOWN');
                 pool.state.lastSwitchAt = Date.now();

package/lib/pool.js

@@ -139,3 +139,18 @@ export function envValue(ref) {
   const v = typeof process !== 'undefined' ? process.env?.[ref] : undefined;
   return typeof v === 'string' && v.length > 0 ? v : undefined;
 }
+
+/** Sweep expired cooldown entries from poolState. Returns count of cleared refs. */
+export function sweepExpired(poolState, now = Date.now()) {
+  let cleared = 0;
+  for (const st of poolState.values()) {
+    for (const [ref, until] of [...st.failedUntil.entries()]) {
+      if (until <= now) {
+        st.failedUntil.delete(ref);
+        st.failCounts?.delete(ref);
+        cleared++;
+      }
+    }
+  }
+  return cleared;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
     "deepseek-harness",