@goodandready/dsh-key-rotation 0.8.40 → 0.8.41

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## 0.8.41 - 2026-10-01
+
+### Fixed
+- **Reproducible CI and core compatibility matrix (#424)**:
+  - Added explicit test `devDependencies` (`@deepseek-ai/schemastery`, `@deepseek-ai/cordis`, `@deepseek-ai/dsh-llm`, `semver`) to `package.json` to ensure clean checkouts (`npm ci`) reproduce all unit tests without depending on global or ambient node_modules.
+  - Updated modern core detection in `test/support/real-core.mjs` to properly identify DSH `0.2.0-rc.1`, `0.2.0-rc.2`, and future releases as modern loaders rather than misclassifying them as legacy.
+- **Preserved exponential backoff across credential selection (#410)**:
+  - Removed premature deletion of `failCounts`, `authFailCounts`, and `brokenUntil` during resolver selection in `lib/resolver.js` (`settle()`).
+  - Quarantine backoff state is now strictly retained until upstream requests finish successfully via `recordSuccess()`, preventing flapping keys from resetting to first-level retry backoff on every dispatch.
+- **Cascade failover on open circuit breaker (#413)**:
+  - When a primary provider's circuit breaker is open (`breaker.acquire()` returns null), `lib/rotate.js` now evaluates `pickCascadeFallback` before terminating, seamlessly routing traffic to healthy fallback providers.
+  - `pickCascadeFallback` in `lib/cascade.js` now validates circuit breaker state, skipping fallback providers whose circuits are open and failing closed with `CIRCUIT_OPEN` only when all cascade options are exhausted.
+- **Plaintext secret decryption for diagnostic probes and self-healing (#415)**:
+  - Added `decryptSecret` decryption to `lib/ops-test.js` (`/dsh-key-rotation/test` route) and `lib/lifecycle.js` (`autoUnbreakBrokenKeys`), preventing encrypted ciphertext credentials (`enc:v1:...`) from being sent to upstream LLM probe endpoints.
+  - Probe success now cleanses `revokedRefs` alongside `failedUntil` and `failCounts`.
+- **Fan-out reset across all model pools and shared credentials (#416)**:
+  - Key reset (`POST /dsh-key-rotation/reset` with `ref`) now fans out across ALL pools in `poolState` containing the ref rather than halting after the first match, clearing `failedUntil`, `failCounts`, `authFailCounts`, `brokenUntil`, `tokenUsage`, and `revokedRefs`.
+  - Provider reset (`provider`) and webhook actions (`pause-*`, `reset-*`) now fan out across base pools and all model sub-pools (`provider::*`), resetting circuit breakers and all associated quotas.
+
 ## 0.8.40 - 2026-09-30
 
 ### Fixed

package/lib/cascade.js

@@ -5,7 +5,7 @@ import { isModelQuotaAvailable, hasModelQuotaConfig } from './model-quota.js';
 
 export const CASCADE_MAX_DEPTH = 3;
 
-export function pickCascadeFallback(provider, cfg, pools, currentModel, modelPoolByProvider) {
+export function pickCascadeFallback(provider, cfg, pools, currentModel, modelPoolByProvider, circuitBreaker) {
   const list = Array.isArray(cfg && cfg.cascade) ? cfg.cascade : [];
   const modelPools = modelPoolByProvider || cfg?.modelPoolByProvider || cfg?.index?.modelPoolByProvider || null;
   const provPools = pools || cfg?.providerToPool || cfg?.index?.providerToPool || null;
@@ -14,6 +14,11 @@ export function pickCascadeFallback(provider, cfg, pools, currentModel, modelPoo
     const fb = typeof entry === 'string' ? { provider: entry } : entry;
     if (!fb || !fb.provider || fb.provider === provider) continue;
 
+    const breaker = circuitBreaker || cfg?.circuitBreaker || null;
+    if (breaker && typeof breaker.canRequest === 'function' && !breaker.canRequest(fb.provider)) {
+      continue;
+    }
+
     const mappedModel = (fb.modelMapping && currentModel && fb.modelMapping[currentModel]) || fb.model || currentModel || null;
     let pool = null;
     if (modelPools && mappedModel) {

package/lib/lifecycle.js

@@ -1,5 +1,6 @@
 // lib/lifecycle.js — background timers and lifecycle effects
 import { healIdleCooldowns, autoUnbreakBrokenKeys } from './heal.js';
+import { decryptSecret } from './crypto-storage.js';
 import { StatePersistence, resolveStatePath } from './persistence.js';
 import path from 'node:path';
 import { envValue } from './pool.js';
@@ -45,8 +46,9 @@ export function setupAutoUnbreakEffect(ctx, getConfig, buildRuntime, ensureSandb
           const credentials = ctx.get?.('credentials') ?? ctx.credentials;
           const resolve = credentials?.__dshKeyRotationOriginalResolve ?? credentials?.resolve;
           const hit = await resolve?.call(credentials, ref);
-          const value = typeof hit?.value === 'string' && hit.value.length > 0
+          const raw = typeof hit?.value === 'string' && hit.value.length > 0
             ? hit.value : envValue(ref);
+          const value = raw ? decryptSecret(raw) : raw;
           if (!value || !isActive()) return { ok: false };
           return runner.probeModels(ref, value);
         }, Date.now(), { isActive });

package/lib/ops-keys.js

@@ -124,16 +124,27 @@ ctx.effect(() => ctx.webServer.register({
     const provider = typeof body?.provider === 'string' ? body.provider.trim() : '';
     const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
     if (provider) {
-      const st = poolState.get(provider);
-      if (!st) { json(res, 404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }); return; }
-      const cleared = st.failedUntil.size;
-      st.failedUntil.clear();
-      st.failCounts?.clear();
-      st.authFailCounts?.clear();
-      st.brokenUntil?.clear();
-      st.revokedRefs?.clear();
-      st.tokenUsage?.clear();
-      st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
+      let found = false;
+      let cleared = 0;
+      for (const [key, st] of poolState.entries()) {
+        if (key === provider || key.startsWith(provider + '::')) {
+          found = true;
+          cleared += st.failedUntil ? st.failedUntil.size : 0;
+          st.failedUntil?.clear();
+          st.failCounts?.clear();
+          st.authFailCounts?.clear();
+          st.brokenUntil?.clear();
+          st.revokedRefs?.clear();
+          st.tokenUsage?.clear();
+          st.switches = 0;
+          st.lastReason = undefined;
+          st.lastSwitchAt = undefined;
+        }
+      }
+      if (!found) {
+        json(res, 404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } });
+        return;
+      }
       let circuitReset = false;
       const br = circuitBreaker ?? buildRuntime().breaker;
       if (br) {
@@ -146,7 +157,15 @@ ctx.effect(() => ctx.webServer.register({
     if (ref) {
       let found = false;
       for (const st of poolState.values()) {
-        if (st.failedUntil?.has(ref) || st.failCounts?.has(ref) || st.authFailCounts?.has(ref) || st.brokenUntil?.has(ref) || st.tokenUsage?.has(ref)) {
+        if (
+          st.failedUntil?.has(ref) ||
+          st.failCounts?.has(ref) ||
+          st.authFailCounts?.has(ref) ||
+          st.brokenUntil?.has(ref) ||
+          st.revokedRefs?.has(ref) ||
+          st.tokenUsage?.has(ref) ||
+          st.lastUsed === ref
+        ) {
           st.failedUntil?.delete(ref);
           st.failCounts?.delete(ref);
           st.authFailCounts?.delete(ref);
@@ -154,11 +173,14 @@ ctx.effect(() => ctx.webServer.register({
           st.revokedRefs?.delete(ref);
           st.tokenUsage?.delete(ref);
           if (st.lastUsed === ref) st.lastUsed = undefined;
-          found = true; break;
+          found = true;
         }
       }
       // idempotent: even if ref was not cooling, report ok if it looks like a valid ref name
-      if (!found && !isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
+      if (!found && !isValidRef(ref)) {
+        json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } });
+        return;
+      }
       json(res, 200, { ok: true, ref });
       return;
     }

package/lib/ops-test.js

@@ -14,6 +14,7 @@ import {
   TEST_PATH,
   SANDBOX_CACHE_PATH,
 } from './ops-paths.js';
+import { decryptSecret } from './crypto-storage.js';
 
 /**
  * @param {object} ctx cordis context
@@ -43,7 +44,8 @@ ctx.effect(() => ctx.webServer.register({
     try {
       let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
       let present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
-      const effectiveValue = testValue || hit?.value;
+      const rawValue = testValue || hit?.value;
+      const effectiveValue = rawValue ? decryptSecret(rawValue) : rawValue;
       const valid = present ? Boolean(effectiveValue && typeof effectiveValue === 'string' && effectiveValue.length > 0) : Boolean(testValue);
       const tail = valid ? keyTail(effectiveValue) : '';
       let source = null;
@@ -63,11 +65,12 @@ ctx.effect(() => ctx.webServer.register({
         lastTestCache.set(ref, cached);
         if (cached.ok) {
           for (const st of poolState.values()) {
-            if (st.failedUntil?.has(ref) || st.failCounts?.has(ref) || st.brokenUntil?.has(ref)) {
+            if (st.failedUntil?.has(ref) || st.failCounts?.has(ref) || st.brokenUntil?.has(ref) || st.revokedRefs?.has(ref)) {
               st.failedUntil?.delete(ref);
               st.failCounts?.delete(ref);
               st.authFailCounts?.delete(ref);
               st.brokenUntil?.delete(ref);
+              st.revokedRefs?.delete(ref);
             }
           }
         }

package/lib/ops-webhook.js

@@ -153,24 +153,50 @@ ctx.effect(() => ctx.webServer.register({
         }
         if (action.startsWith('pause-')) {
           const until = Date.now() + 3600000; // 1h pause
-          for (const ref of (st.failedUntil ? [...st.failedUntil.keys()] : [])) st.failedUntil.set(ref, Math.max(st.failedUntil.get(ref) ?? 0, until));
-          // also pause every key currently healthy
+          let found = false;
+          for (const [key, s] of poolState.entries()) {
+            if (key === provider || key.startsWith(provider + '::')) {
+              found = true;
+              for (const ref of (s.failedUntil ? [...s.failedUntil.keys()] : [])) {
+                s.failedUntil.set(ref, Math.max(s.failedUntil.get(ref) ?? 0, until));
+              }
+            }
+          }
           for (const p of buildRuntime().poolByRef.values()) {
-            if (p.base !== provider) continue;
-            for (const ref of p.refs) st.failedUntil.set(ref, Math.max(st.failedUntil.get(ref) ?? 0, until));
+            if (p.provider === provider || p.base === provider || p.base?.startsWith(provider + '::')) {
+              found = true;
+              for (const ref of p.refs) p.state?.failedUntil?.set(ref, Math.max(p.state?.failedUntil?.get(ref) ?? 0, until));
+            }
+          }
+          if (!found) {
+            await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
+            return;
           }
           (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] pool ${provider} PAUSED 1h via webhook action`);
           await respond(200, { ok: true, action, provider, until }, `Pool ${provider} paused 1h`);
           return;
         }
-        const cleared = st.failedUntil.size;
-        st.failedUntil.clear();
-        st.failCounts?.clear();
-        st.authFailCounts?.clear();
-        st.brokenUntil?.clear();
-        st.switches = 0;
-        st.lastReason = undefined;
-        st.lastSwitchAt = undefined;
+        let found = false;
+        let cleared = 0;
+        for (const [key, s] of poolState.entries()) {
+          if (key === provider || key.startsWith(provider + '::')) {
+            found = true;
+            cleared += s.failedUntil ? s.failedUntil.size : 0;
+            s.failedUntil?.clear();
+            s.failCounts?.clear();
+            s.authFailCounts?.clear();
+            s.brokenUntil?.clear();
+            s.revokedRefs?.clear();
+            s.tokenUsage?.clear();
+            s.switches = 0;
+            s.lastReason = undefined;
+            s.lastSwitchAt = undefined;
+          }
+        }
+        if (!found) {
+          await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
+          return;
+        }
         let circuitReset = false;
         const br = circuitBreaker ?? buildRuntime().breaker;
         if (br) {

package/lib/resolver.js

@@ -237,10 +237,6 @@ function settle(pool, candidate, now, onPicked, concurrencyRef) {
   if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
   pool.state.lastUsedAt.set(candidate, now);
   onPicked?.(pool, candidate, { concurrencyRef });
-  pool.state.failCounts?.delete(candidate);
-  pool.state.failedUntil.delete(candidate);
-  pool.state.authFailCounts?.delete(candidate);
-  pool.state.brokenUntil?.delete(candidate);
   if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
   pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
   if (pool.perHour) {

package/lib/rotate.js

@@ -192,7 +192,30 @@ export function createRotate(deps) {
       for (let attempt = 0; attempt < attemptList.length; attempt++) {
         activePermit = breaker ? breaker.acquire(options.provider) : null;
         if (breaker && !activePermit) {
-          logWarn(`[dsh-key-rotation] ${options.provider}: circuit open — skipping dispatch`);
+          logWarn(`[dsh-key-rotation] ${options.provider}: circuit open — trying cascade fallback`);
+          const runtimeQ = buildRuntime();
+          const maxDepth = runtimeQ.cascadeMaxDepth ?? CASCADE_MAX_DEPTH;
+          const currentDepth = options.__cascadeDepth ?? (options.__isCascade ? 1 : 0);
+          if (currentDepth < maxDepth && Array.isArray(runtimeQ.cascade) && runtimeQ.cascade.length > 0) {
+            const fb = pickCascadeFallback(options.provider, runtimeQ, runtimeQ.providerToPool, options.model, runtimeQ.modelPoolByProvider, breaker);
+            if (fb && fb.pool && fb.pool !== pool) {
+              pool.state.lastReason = 'CASCADE';
+              pool.state.lastSwitchAt = now();
+              pool.state.cascadeCount = (pool.state.cascadeCount ?? 0) + 1;
+              pushEvent(pool, reqStore.pickedRef ?? 'pool', 'CASCADE_TRIGGERED', 0);
+              const innerCascade = rotate({
+                ...options,
+                provider: fb.provider,
+                model: fb.model || options.model,
+                __isCascade: true,
+                __cascadeDepth: currentDepth + 1,
+              }, fb.pool);
+              for await (const chunk of innerCascade) {
+                yield chunk;
+              }
+              return;
+            }
+          }
           yield finishError('CIRCUIT_OPEN', `[dsh-key-rotation] provider '${options.provider}' circuit is open`);
           return;
         }
@@ -425,7 +448,7 @@ export function createRotate(deps) {
       const currentDepth = options.__cascadeDepth ?? (options.__isCascade ? 1 : 0);
       if (currentDepth < maxDepth && Array.isArray(runtime.cascade) && runtime.cascade.length > 0) {
         const pools = runtime.providerToPool;
-        const fb = pickCascadeFallback(options.provider, runtime, pools, options.model, runtime.modelPoolByProvider);
+        const fb = pickCascadeFallback(options.provider, runtime, pools, options.model, runtime.modelPoolByProvider, circuitBreaker);
         if (fb && fb.pool && fb.pool !== pool) {
           logWarn(`[dsh-key-rotation] ${options.provider}: pool exhausted — cascading to ${fb.provider} (model: ${fb.model || options.model || 'default'})`);
           pool.state.lastReason = 'CASCADE';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.40",
+  "version": "0.8.41",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
@@ -69,7 +69,11 @@
   },
   "devDependencies": {
     "react": "18.3.1",
-    "react-test-renderer": "18.3.1"
+    "react-test-renderer": "18.3.1",
+    "@deepseek-ai/cordis": "^4.0.1",
+    "@deepseek-ai/dsh-llm": "^0.2.0-rc.2",
+    "@deepseek-ai/schemastery": "^3.18.4",
+    "semver": "^7.6.3"
   },
   "dependencies": {
     "undici": "^6.29.0"