@goodandready/dsh-key-rotation 0.8.39 → 0.8.41

package/CHANGELOG.md

@@ -1,5 +1,42 @@
 # Changelog
 
+## 0.8.41 - 2026-10-01
+
+### Fixed
+- **Reproducible CI and core compatibility matrix (#424)**:
+  - Added explicit test `devDependencies` (`@deepseek-ai/schemastery`, `@deepseek-ai/cordis`, `@deepseek-ai/dsh-llm`, `semver`) to `package.json` to ensure clean checkouts (`npm ci`) reproduce all unit tests without depending on global or ambient node_modules.
+  - Updated modern core detection in `test/support/real-core.mjs` to properly identify DSH `0.2.0-rc.1`, `0.2.0-rc.2`, and future releases as modern loaders rather than misclassifying them as legacy.
+- **Preserved exponential backoff across credential selection (#410)**:
+  - Removed premature deletion of `failCounts`, `authFailCounts`, and `brokenUntil` during resolver selection in `lib/resolver.js` (`settle()`).
+  - Quarantine backoff state is now strictly retained until upstream requests finish successfully via `recordSuccess()`, preventing flapping keys from resetting to first-level retry backoff on every dispatch.
+- **Cascade failover on open circuit breaker (#413)**:
+  - When a primary provider's circuit breaker is open (`breaker.acquire()` returns null), `lib/rotate.js` now evaluates `pickCascadeFallback` before terminating, seamlessly routing traffic to healthy fallback providers.
+  - `pickCascadeFallback` in `lib/cascade.js` now validates circuit breaker state, skipping fallback providers whose circuits are open and failing closed with `CIRCUIT_OPEN` only when all cascade options are exhausted.
+- **Plaintext secret decryption for diagnostic probes and self-healing (#415)**:
+  - Added `decryptSecret` decryption to `lib/ops-test.js` (`/dsh-key-rotation/test` route) and `lib/lifecycle.js` (`autoUnbreakBrokenKeys`), preventing encrypted ciphertext credentials (`enc:v1:...`) from being sent to upstream LLM probe endpoints.
+  - Probe success now cleanses `revokedRefs` alongside `failedUntil` and `failCounts`.
+- **Fan-out reset across all model pools and shared credentials (#416)**:
+  - Key reset (`POST /dsh-key-rotation/reset` with `ref`) now fans out across ALL pools in `poolState` containing the ref rather than halting after the first match, clearing `failedUntil`, `failCounts`, `authFailCounts`, `brokenUntil`, `tokenUsage`, and `revokedRefs`.
+  - Provider reset (`provider`) and webhook actions (`pause-*`, `reset-*`) now fan out across base pools and all model sub-pools (`provider::*`), resetting circuit breakers and all associated quotas.
+
+## 0.8.40 - 2026-09-30
+
+### Fixed
+- **Dynamic concurrency limits & least-loaded connection tracking (#407)**:
+  - Added `configure({ limit, staleMs })` to `ConcurrencyTracker` so runtime limit updates from configuration or settings take immediate effect.
+  - Track in-flight request counts across all keys regardless of whether global limit is 0, ensuring accurate `least-loaded` routing metrics and status telemetry.
+  - Atomically acquire in-flight permit at credential selection time in `lib/resolver.js` and roll back immediately if the credential resolver rejects.
+  - Safely release concurrency permits in the outer `finally` block of `lib/rotate.js` on stream completion, client abort, error, or unhandled rejection.
+  - Skip saturated keys in `pickLeastLoaded` and fail closed when all candidate keys exceed their configured concurrency limits.
+- **Model-aware cross-provider cascade failover (#414)**:
+  - `pickCascadeFallback` now maps fallback models to target model sub-pools (`modelPoolByProvider`) rather than falling back only to base provider pools.
+  - Supported model-only providers (providers where credentials are defined exclusively under `models`) as valid cascade fallback targets.
+  - Validated local model token budgets (`isModelQuotaAvailable`) during cascade selection to bypass exhausted model pools.
+- **Aggregated provider monetary budget & model pool enforcement (#422)**:
+  - Added `getProviderCost(provider, runtime, now)` to aggregate daily and weekly spend across provider base pools and all model sub-pools without double-counting shared state.
+  - `checkBudgetAndHealthAlerts` now consolidates provider spend across all pools, sends unified webhook alerts with correct provider names, and pauses all pools (base and model sub-pools) for 24h when `pauseOnBudget: true`.
+  - Enforced monetary budget refusal (`LOCAL_POOL_EXHAUSTED`) at credential resolution time in `lib/resolver.js` when provider budget is exceeded and `pauseOnBudget` is active.
+
 ## 0.8.39 - 2026-09-30
 
 ### Fixed

package/lib/budget-monitor.js

@@ -1,6 +1,7 @@
 // lib/budget-monitor.js — periodic budget, expiry, low health and SLO alerts
 import { expiringSoon, shouldNotifyDaily, costForDay, costForWeek, budgetVerdict, isKeyPaused, isKeyRevoked } from './pool.js';
 import { isModelQuotaAvailable, hasModelQuotaConfig } from './model-quota.js';
+import { getProviderCost } from './resolver.js';
 
 const DAY_MS = 86400000;
 
@@ -17,58 +18,61 @@ export function checkBudgetAndHealthAlerts({
   logger,
 }) {
   try {
-    const seen = new Set();
-    for (const pool of runtime.poolByRef.values()) {
-      if (seen.has(pool.base)) continue;
-      seen.add(pool.base);
-      // #207: keys expiring within expiryWarnDays -> one webhook per key/day
-      for (const { ref, expiresInDays } of expiringSoon(pool, runtime.expiryWarnDays, now)) {
-        if (!shouldNotifyDaily(expiryNotifiedAt, pool.base + ':' + ref, now)) continue;
-        logger?.warn?.(`[dsh-key-rotation] ${pool.base}: key ${ref} expires in ~${expiresInDays}d`);
-        if (runtime.notifyWebhook) {
-          webhookSender.send(runtime.notifyWebhook, {
-            title: `Key expiring soon: ${pool.base}`,
-            text: `${ref} expires in ~${expiresInDays} day(s)`,
-            provider: pool.base,
-            kind: 'expiry',
-            keys: [ref],
-          });
-        }
-      }
-      // #208: daily/weekly budget -> warn webhook, optional 1-day pause at 100%
-      const budget = runtime.providerBudgets?.get?.(pool.base);
-      if (budget) {
-        const daily = costForDay(pool.state.costDays);
-        const weekly = costForWeek(pool.state.costDays, now);
+    // #208 / #422: provider monetary budget alerts & pause across base and model pools
+    if (runtime.providerBudgets && runtime.providerBudgets.size > 0) {
+      for (const [provider, budget] of runtime.providerBudgets) {
+        const { daily, weekly, pools } = getProviderCost(provider, runtime, now);
         const verdict = budgetVerdict(daily, budget.costBudgetDaily);
         const wVerdict = budgetVerdict(weekly, budget.costBudgetWeekly);
         const hit = verdict.warn || wVerdict.warn;
-        if (hit && shouldNotifyDaily(budgetNotifiedAt, pool.base + ':budget', now)) {
-          logger?.warn?.(`[dsh-key-rotation] ${pool.base}: cost budget - day $${daily.toFixed(2)}/$${budget.costBudgetDaily} week $${weekly.toFixed(2)}/$${budget.costBudgetWeekly}`);
+        if (hit && shouldNotifyDaily(budgetNotifiedAt, provider + ':budget', now)) {
+          logger?.warn?.(`[dsh-key-rotation] ${provider}: cost budget - day $${daily.toFixed(2)}/$${budget.costBudgetDaily} week $${weekly.toFixed(2)}/$${budget.costBudgetWeekly}`);
           if (runtime.notifyWebhook) {
-            // #217: budget webhook gains action buttons when a callback token is configured
             const token = runtime.webhookActionToken ?? '';
             webhookSender.send(runtime.notifyWebhook, {
-              title: `Cost budget: ${pool.base}`,
+              title: `Cost budget: ${provider}`,
               text: `day $${daily.toFixed(2)} of $${budget.costBudgetDaily} · week $${weekly.toFixed(2)} of $${budget.costBudgetWeekly}` + (verdict.exceeded || wVerdict.exceeded ? ' · EXCEEDED' : ''),
-              provider: pool.base,
+              provider,
               kind: 'budget',
               spend: { daily, weekly },
               actionToken: token || undefined,
               actions: token ? [
-                { id: `pause-${pool.base}`, label: 'Pause 1h' },
-                { id: `reset-${pool.base}`, label: 'Reset cooldown' },
+                { id: `pause-${provider}`, label: 'Pause 1h' },
+                { id: `reset-${provider}`, label: 'Reset cooldown' },
               ] : undefined,
             });
           }
         }
         if ((verdict.exceeded || wVerdict.exceeded) && budget.pauseOnBudget) {
           const until = now + DAY_MS;
-          for (const ref of pool.refs) {
-            if ((pool.state.failedUntil.get(ref) ?? 0) < until) pool.state.failedUntil.set(ref, until);
+          for (const p of pools) {
+            for (const ref of p.refs) {
+              if ((p.state.failedUntil.get(ref) ?? 0) < until) p.state.failedUntil.set(ref, until);
+            }
           }
         }
       }
+    }
+
+    const seen = new Set();
+    for (const pool of runtime.poolByRef.values()) {
+      if (seen.has(pool.base)) continue;
+      seen.add(pool.base);
+      // #207: keys expiring within expiryWarnDays -> one webhook per key/day
+      for (const { ref, expiresInDays } of expiringSoon(pool, runtime.expiryWarnDays, now)) {
+        if (!shouldNotifyDaily(expiryNotifiedAt, pool.base + ':' + ref, now)) continue;
+        logger?.warn?.(`[dsh-key-rotation] ${pool.base}: key ${ref} expires in ~${expiresInDays}d`);
+        if (runtime.notifyWebhook) {
+          webhookSender.send(runtime.notifyWebhook, {
+            title: `Key expiring soon: ${pool.base}`,
+            text: `${ref} expires in ~${expiresInDays} day(s)`,
+            provider: pool.base,
+            kind: 'expiry',
+            keys: [ref],
+          });
+        }
+      }
+      // #208 / #422: handled provider-wide before pool loop
       // #221: pool running low - webhook while healthy < warnBelowHealthy
       const warnBelow = runtime.warnBelowHealthy ?? 0;
       if (warnBelow > 0) {

package/lib/cascade.js

@@ -1,28 +1,46 @@
-// cascade.js — cross-provider failover cascade (issues #194, #364, #376).
+// cascade.js — cross-provider failover cascade (issues #194, #364, #376, #414).
 // ponytail: minimal — pick fallback provider with model mapping from config.
-import { isKeyPaused, isKeyRevoked } from './pool.js';
+import { isKeyPaused, isKeyRevoked, selectPool } from './pool.js';
+import { isModelQuotaAvailable, hasModelQuotaConfig } from './model-quota.js';
 
 export const CASCADE_MAX_DEPTH = 3;
 
-export function pickCascadeFallback(provider, cfg, pools, currentModel) {
+export function pickCascadeFallback(provider, cfg, pools, currentModel, modelPoolByProvider, circuitBreaker) {
   const list = Array.isArray(cfg && cfg.cascade) ? cfg.cascade : [];
+  const modelPools = modelPoolByProvider || cfg?.modelPoolByProvider || cfg?.index?.modelPoolByProvider || null;
+  const provPools = pools || cfg?.providerToPool || cfg?.index?.providerToPool || null;
+
   for (const entry of list) {
     const fb = typeof entry === 'string' ? { provider: entry } : entry;
     if (!fb || !fb.provider || fb.provider === provider) continue;
-    const pool = pools instanceof Map ? pools.get(fb.provider) : (pools ? pools[fb.provider] : null);
+
+    const breaker = circuitBreaker || cfg?.circuitBreaker || null;
+    if (breaker && typeof breaker.canRequest === 'function' && !breaker.canRequest(fb.provider)) {
+      continue;
+    }
+
+    const mappedModel = (fb.modelMapping && currentModel && fb.modelMapping[currentModel]) || fb.model || currentModel || null;
+    let pool = null;
+    if (modelPools && mappedModel) {
+      pool = selectPool(modelPools, provPools, fb.provider, mappedModel);
+    }
+    if (!pool && provPools) {
+      pool = provPools instanceof Map ? provPools.get(fb.provider) : (provPools ? provPools[fb.provider] : null);
+    }
     if (!pool) continue;
+
     const now = Date.now();
     let healthy = 0;
     for (const ref of pool.refs) {
       if (isKeyPaused(pool, ref) || isKeyRevoked(pool, ref)) continue;
       const failedUntil = (pool.state && pool.state.failedUntil && pool.state.failedUntil.get(ref)) || 0;
       if (failedUntil > now) continue;
-      const exp = pool.expiresAt ? pool.expiresAt[ref] : undefined;
+      const exp = pool.expiresAt ? pool.expiresAt[ref] : (pool.basePool?.expiresAt ? pool.basePool.expiresAt[ref] : undefined);
       if (exp !== undefined && now >= exp) continue;
+      if (hasModelQuotaConfig(pool) && !isModelQuotaAvailable(pool, ref, now)) continue;
       healthy += 1;
     }
     if (healthy === 0) continue;
-    const mappedModel = (fb.modelMapping && currentModel && fb.modelMapping[currentModel]) || fb.model || currentModel || null;
     return { provider: fb.provider, pool, model: mappedModel };
   }
   return null;

package/lib/concurrency.js

@@ -1,4 +1,4 @@
-// concurrency.js — per-key in-flight counter + least-connections picking (issue #193).
+// concurrency.js — per-key in-flight counter + least-connections picking (issues #193, #407).
 
 const CONCURRENCY_DEFAULT_LIMIT = 0;
 const CONCURRENCY_STALE_LOCK_MS = 5 * 60 * 1000;
@@ -12,12 +12,22 @@ export class ConcurrencyTracker {
     this._inFlight = new Map();
   }
 
+  configure(opts) {
+    if (!opts) return;
+    if (opts.limit !== undefined) {
+      this._limit = (Number.isFinite(opts.limit) && opts.limit >= 0) ? Math.floor(opts.limit) : 0;
+    }
+    if (opts.staleMs !== undefined && Number.isFinite(opts.staleMs) && opts.staleMs > 0) {
+      this._staleMs = opts.staleMs;
+    }
+  }
+
   isEnabled() { return this._limit > 0; }
   get limit() { return this._limit; }
 
-  acquire(ref, now) {
+  acquire(ref, now, limit) {
     now = now || Date.now();
-    if (!this.isEnabled()) return true;
+    const effectiveLimit = (typeof limit === 'number' && limit >= 0) ? Math.floor(limit) : this._limit;
     let e = this._inFlight.get(ref);
     if (!e) {
       e = { count: 0, lastAcquired: now };
@@ -26,7 +36,7 @@ export class ConcurrencyTracker {
     if (now - e.lastAcquired > this._staleMs) {
       e.count = 0;
     }
-    if (e.count >= this._limit) return false;
+    if (effectiveLimit > 0 && e.count >= effectiveLimit) return false;
     e.count += 1;
     e.lastAcquired = now;
     return true;
@@ -55,8 +65,9 @@ export class ConcurrencyTracker {
     return out;
   }
 
-  pickLeastLoaded(candidates, now, weights) {
+  pickLeastLoaded(candidates, now, weights, limit) {
     now = now || Date.now();
+    const effectiveLimit = (typeof limit === 'number' && limit >= 0) ? Math.floor(limit) : this._limit;
     if (!Array.isArray(candidates) || candidates.length === 0) return null;
     let best = null;
     let bestScore = Infinity;
@@ -64,7 +75,7 @@ export class ConcurrencyTracker {
       const e = this._inFlight.get(ref);
       let count = e ? e.count : 0;
       if (e && now - e.lastAcquired > this._staleMs) count = 0;
-      if (this.isEnabled() && count >= this._limit) continue;
+      if (effectiveLimit > 0 && count >= effectiveLimit) continue;
       const w = (weights && Number(weights[ref]) > 0) ? Number(weights[ref]) : 1;
       const score = count / w;
       if (score < bestScore) {

package/lib/index.js

@@ -324,6 +324,7 @@ export function apply(ctx, config = {}) {
       budgetNotifiedAt,
     });
 
+    concurrencyTracker.configure({ limit: concurrencyLimit });
     cachedRuntime = {
       switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold,
       concurrencyLimit, cascade, quotaResetWindow, rateLimitThreshold, rpmLimit, tpmLimit,
@@ -355,9 +356,12 @@ export function apply(ctx, config = {}) {
       const resolve = createResolver({
         buildRuntime,
         currentPool: () => dispatchStorage.getStore()?.pool,
-        onPicked: (pool, candidate) => {
+        onPicked: (pool, candidate, meta) => {
           const store = dispatchStorage.getStore();
-          if (store && store.pool === pool) store.pickedRef = candidate;
+          if (store && store.pool === pool) {
+            store.pickedRef = candidate;
+            if (meta?.concurrencyRef) store.concurrencyRef = meta.concurrencyRef;
+          }
         },
         latencyHistogram,
         concurrencyTracker,

package/lib/lifecycle.js

@@ -1,5 +1,6 @@
 // lib/lifecycle.js — background timers and lifecycle effects
 import { healIdleCooldowns, autoUnbreakBrokenKeys } from './heal.js';
+import { decryptSecret } from './crypto-storage.js';
 import { StatePersistence, resolveStatePath } from './persistence.js';
 import path from 'node:path';
 import { envValue } from './pool.js';
@@ -45,8 +46,9 @@ export function setupAutoUnbreakEffect(ctx, getConfig, buildRuntime, ensureSandb
           const credentials = ctx.get?.('credentials') ?? ctx.credentials;
           const resolve = credentials?.__dshKeyRotationOriginalResolve ?? credentials?.resolve;
           const hit = await resolve?.call(credentials, ref);
-          const value = typeof hit?.value === 'string' && hit.value.length > 0
+          const raw = typeof hit?.value === 'string' && hit.value.length > 0
             ? hit.value : envValue(ref);
+          const value = raw ? decryptSecret(raw) : raw;
           if (!value || !isActive()) return { ok: false };
           return runner.probeModels(ref, value);
         }, Date.now(), { isActive });

package/lib/ops-keys.js

@@ -124,16 +124,27 @@ ctx.effect(() => ctx.webServer.register({
     const provider = typeof body?.provider === 'string' ? body.provider.trim() : '';
     const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
     if (provider) {
-      const st = poolState.get(provider);
-      if (!st) { json(res, 404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }); return; }
-      const cleared = st.failedUntil.size;
-      st.failedUntil.clear();
-      st.failCounts?.clear();
-      st.authFailCounts?.clear();
-      st.brokenUntil?.clear();
-      st.revokedRefs?.clear();
-      st.tokenUsage?.clear();
-      st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
+      let found = false;
+      let cleared = 0;
+      for (const [key, st] of poolState.entries()) {
+        if (key === provider || key.startsWith(provider + '::')) {
+          found = true;
+          cleared += st.failedUntil ? st.failedUntil.size : 0;
+          st.failedUntil?.clear();
+          st.failCounts?.clear();
+          st.authFailCounts?.clear();
+          st.brokenUntil?.clear();
+          st.revokedRefs?.clear();
+          st.tokenUsage?.clear();
+          st.switches = 0;
+          st.lastReason = undefined;
+          st.lastSwitchAt = undefined;
+        }
+      }
+      if (!found) {
+        json(res, 404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } });
+        return;
+      }
       let circuitReset = false;
       const br = circuitBreaker ?? buildRuntime().breaker;
       if (br) {
@@ -146,7 +157,15 @@ ctx.effect(() => ctx.webServer.register({
     if (ref) {
       let found = false;
       for (const st of poolState.values()) {
-        if (st.failedUntil?.has(ref) || st.failCounts?.has(ref) || st.authFailCounts?.has(ref) || st.brokenUntil?.has(ref) || st.tokenUsage?.has(ref)) {
+        if (
+          st.failedUntil?.has(ref) ||
+          st.failCounts?.has(ref) ||
+          st.authFailCounts?.has(ref) ||
+          st.brokenUntil?.has(ref) ||
+          st.revokedRefs?.has(ref) ||
+          st.tokenUsage?.has(ref) ||
+          st.lastUsed === ref
+        ) {
           st.failedUntil?.delete(ref);
           st.failCounts?.delete(ref);
           st.authFailCounts?.delete(ref);
@@ -154,11 +173,14 @@ ctx.effect(() => ctx.webServer.register({
           st.revokedRefs?.delete(ref);
           st.tokenUsage?.delete(ref);
           if (st.lastUsed === ref) st.lastUsed = undefined;
-          found = true; break;
+          found = true;
         }
       }
       // idempotent: even if ref was not cooling, report ok if it looks like a valid ref name
-      if (!found && !isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
+      if (!found && !isValidRef(ref)) {
+        json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } });
+        return;
+      }
       json(res, 200, { ok: true, ref });
       return;
     }

package/lib/ops-test.js

@@ -14,6 +14,7 @@ import {
   TEST_PATH,
   SANDBOX_CACHE_PATH,
 } from './ops-paths.js';
+import { decryptSecret } from './crypto-storage.js';
 
 /**
  * @param {object} ctx cordis context
@@ -43,7 +44,8 @@ ctx.effect(() => ctx.webServer.register({
     try {
       let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
       let present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
-      const effectiveValue = testValue || hit?.value;
+      const rawValue = testValue || hit?.value;
+      const effectiveValue = rawValue ? decryptSecret(rawValue) : rawValue;
       const valid = present ? Boolean(effectiveValue && typeof effectiveValue === 'string' && effectiveValue.length > 0) : Boolean(testValue);
       const tail = valid ? keyTail(effectiveValue) : '';
       let source = null;
@@ -63,11 +65,12 @@ ctx.effect(() => ctx.webServer.register({
         lastTestCache.set(ref, cached);
         if (cached.ok) {
           for (const st of poolState.values()) {
-            if (st.failedUntil?.has(ref) || st.failCounts?.has(ref) || st.brokenUntil?.has(ref)) {
+            if (st.failedUntil?.has(ref) || st.failCounts?.has(ref) || st.brokenUntil?.has(ref) || st.revokedRefs?.has(ref)) {
               st.failedUntil?.delete(ref);
               st.failCounts?.delete(ref);
               st.authFailCounts?.delete(ref);
               st.brokenUntil?.delete(ref);
+              st.revokedRefs?.delete(ref);
             }
           }
         }

package/lib/ops-webhook.js

@@ -153,24 +153,50 @@ ctx.effect(() => ctx.webServer.register({
         }
         if (action.startsWith('pause-')) {
           const until = Date.now() + 3600000; // 1h pause
-          for (const ref of (st.failedUntil ? [...st.failedUntil.keys()] : [])) st.failedUntil.set(ref, Math.max(st.failedUntil.get(ref) ?? 0, until));
-          // also pause every key currently healthy
+          let found = false;
+          for (const [key, s] of poolState.entries()) {
+            if (key === provider || key.startsWith(provider + '::')) {
+              found = true;
+              for (const ref of (s.failedUntil ? [...s.failedUntil.keys()] : [])) {
+                s.failedUntil.set(ref, Math.max(s.failedUntil.get(ref) ?? 0, until));
+              }
+            }
+          }
           for (const p of buildRuntime().poolByRef.values()) {
-            if (p.base !== provider) continue;
-            for (const ref of p.refs) st.failedUntil.set(ref, Math.max(st.failedUntil.get(ref) ?? 0, until));
+            if (p.provider === provider || p.base === provider || p.base?.startsWith(provider + '::')) {
+              found = true;
+              for (const ref of p.refs) p.state?.failedUntil?.set(ref, Math.max(p.state?.failedUntil?.get(ref) ?? 0, until));
+            }
+          }
+          if (!found) {
+            await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
+            return;
           }
           (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] pool ${provider} PAUSED 1h via webhook action`);
           await respond(200, { ok: true, action, provider, until }, `Pool ${provider} paused 1h`);
           return;
         }
-        const cleared = st.failedUntil.size;
-        st.failedUntil.clear();
-        st.failCounts?.clear();
-        st.authFailCounts?.clear();
-        st.brokenUntil?.clear();
-        st.switches = 0;
-        st.lastReason = undefined;
-        st.lastSwitchAt = undefined;
+        let found = false;
+        let cleared = 0;
+        for (const [key, s] of poolState.entries()) {
+          if (key === provider || key.startsWith(provider + '::')) {
+            found = true;
+            cleared += s.failedUntil ? s.failedUntil.size : 0;
+            s.failedUntil?.clear();
+            s.failCounts?.clear();
+            s.authFailCounts?.clear();
+            s.brokenUntil?.clear();
+            s.revokedRefs?.clear();
+            s.tokenUsage?.clear();
+            s.switches = 0;
+            s.lastReason = undefined;
+            s.lastSwitchAt = undefined;
+          }
+        }
+        if (!found) {
+          await respond(404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }, `Pool not found: ${provider}`);
+          return;
+        }
         let circuitReset = false;
         const br = circuitBreaker ?? buildRuntime().breaker;
         if (br) {

package/lib/pool-builder.js

@@ -27,6 +27,7 @@ export function buildPoolItem({
   poolGuard,
   rpmLimit,
   tpmLimit = 0,
+  concurrencyLimit = 0,
   makeState,
   provider = null,
   model = null,
@@ -103,6 +104,7 @@ export function buildPoolItem({
     expiresAt: parsedExpiry,
     rpmLimit,
     tpmLimit: typeof tpmLimit === 'number' && Number.isFinite(tpmLimit) && tpmLimit > 0 ? Math.floor(tpmLimit) : 0,
+    concurrencyLimit: typeof concurrencyLimit === 'number' && Number.isFinite(concurrencyLimit) && concurrencyLimit > 0 ? Math.floor(concurrencyLimit) : 0,
     routingStrategy: poolStrategy,
     proactiveRateLimitGuard: poolGuard,
   };
@@ -111,6 +113,7 @@ export function buildPoolItem({
 export function buildPools({ cfg = {}, poolState = new Map(), defaultState = initializePoolState } = {}) {
   const rpmLimit = cfg.rpmLimit ?? 0;
   const tpmLimit = cfg.tpmLimit ?? 0;
+  const concurrencyLimit = cfg.concurrencyLimit ?? 0;
   const quotaResetWindow = cfg.quotaResetWindow ?? null;
   const poolByRef = new Map();
   const providerToPool = new Map();
@@ -127,10 +130,10 @@ export function buildPools({ cfg = {}, poolState = new Map(), defaultState = ini
     return initializePoolState(st);
   };
 
-  const buildPool = (base, keys, weights, paused, revoked, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, meta, pRpm = rpmLimit, pTpm = tpmLimit) =>
+  const buildPool = (base, keys, weights, paused, revoked, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, meta, pRpm = rpmLimit, pTpm = tpmLimit, pConcurrency = concurrencyLimit) =>
     buildPoolItem({
       base, keys, weights, paused, revoked, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard,
-      rpmLimit: pRpm, tpmLimit: pTpm, makeState, quotaResetWindow, ...meta,
+      rpmLimit: pRpm, tpmLimit: pTpm, concurrencyLimit: pConcurrency, makeState, quotaResetWindow, ...meta,
     });
 
   for (const p of cfg.providers ?? []) {
@@ -138,10 +141,11 @@ export function buildPools({ cfg = {}, poolState = new Map(), defaultState = ini
     const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
     const provRpm = typeof p.rpmLimit === 'number' ? p.rpmLimit : rpmLimit;
     const provTpm = typeof p.tpmLimit === 'number' ? p.tpmLimit : tpmLimit;
+    const provConcurrency = typeof p.concurrencyLimit === 'number' ? p.concurrencyLimit : concurrencyLimit;
     const pool = buildPool(p.provider, p.keys, p.weights, p.paused, p.revoked, poolCooldown, poolMax, p.expiresAt, p.routingStrategy, p.proactiveRateLimitGuard, {
       provider: p.provider,
       model: null,
-    }, provRpm, provTpm);
+    }, provRpm, provTpm, provConcurrency);
     if (pool) {
       for (const ref of pool.refs) {
         if (!poolByRef.has(ref)) poolByRef.set(ref, pool);
@@ -153,6 +157,7 @@ export function buildPools({ cfg = {}, poolState = new Map(), defaultState = ini
     for (const [model, mp] of Object.entries(p.models ?? {})) {
       const modelRpm = typeof mp.rpmLimit === 'number' ? mp.rpmLimit : provRpm;
       const modelTpm = typeof mp.tpmLimit === 'number' ? mp.tpmLimit : provTpm;
+      const modelConcurrency = typeof mp.concurrencyLimit === 'number' ? mp.concurrencyLimit : provConcurrency;
       const mergedPaused = new Set([
         ...(pool?.pausedRefs ?? []),
         ...(Array.isArray(mp.paused) ? mp.paused : (mp.paused instanceof Set ? [...mp.paused] : [])),
@@ -166,7 +171,7 @@ export function buildPools({ cfg = {}, poolState = new Map(), defaultState = ini
         provider: p.provider,
         model,
         quotas: mp.quotas,
-      }, modelRpm, modelTpm);
+      }, modelRpm, modelTpm, modelConcurrency);
       if (!mpool) continue;
       mpool.basePool = pool;
       byModel.set(model, mpool);

package/lib/resolver.js

@@ -1,4 +1,3 @@
-// lib/resolver.js — the credential selector behind credentials.resolve (#381).
 //
 // Extracted from lib/index.js so the routing decisions that this feature
 // depends on — request-scoped pool ownership, model token quota enforcement,
@@ -10,7 +9,7 @@
 // delegates to `original(ref)`.
 
 import { bucketAllow, bucketRetryMs, tpmAllow, tpmRetryMs } from './bucket.js';
-import { envValue, sortAttemptList, isKeyPaused, isKeyRevoked } from './pool.js';
+import { envValue, sortAttemptList, isKeyPaused, isKeyRevoked, costForDay, costForWeek, budgetVerdict } from './pool.js';
 import { decryptSecret } from './crypto-storage.js';
 import { resolveRefPool, isQuotaManagedRequest } from './pool-index.js';
 import {
@@ -24,11 +23,51 @@ import {
 export const LOCAL_QUOTA_CODE = 'LOCAL_MODEL_QUOTA_EXHAUSTED';
 export const LOCAL_POOL_EXHAUSTED_CODE = 'LOCAL_POOL_EXHAUSTED';
 
+/**
+ * Calculates aggregated daily and weekly spend across base pool and model pools for a provider without double-counting (#422).
+ */
+export function getProviderCost(provider, runtime, now = Date.now()) {
+  const seenStates = new Set();
+  let daily = 0;
+  let weekly = 0;
+  const pools = [];
+  const basePool = runtime?.providerToPool?.get?.(provider);
+  if (basePool) pools.push(basePool);
+  const modelPools = runtime?.modelPoolByProvider?.get?.(provider);
+  if (modelPools) {
+    for (const mp of modelPools.values()) {
+      if (!pools.includes(mp)) pools.push(mp);
+    }
+  }
+  if (runtime?.index?.allPools) {
+    for (const p of runtime.index.allPools) {
+      if ((p.provider === provider || p.base === provider || p.base?.startsWith(provider + '::')) && !pools.includes(p)) {
+        pools.push(p);
+      }
+    }
+  }
+  if (runtime?.poolByRef) {
+    for (const p of runtime.poolByRef.values()) {
+      if ((p.provider === provider || p.base === provider || p.base?.startsWith(provider + '::')) && !pools.includes(p)) {
+        pools.push(p);
+      }
+    }
+  }
+  for (const p of pools) {
+    if (p.state && !seenStates.has(p.state)) {
+      seenStates.add(p.state);
+      daily += costForDay(p.state.costDays);
+      weekly += costForWeek(p.state.costDays, now);
+    }
+  }
+  return { daily, weekly, pools };
+}
+
 /**
  * @param {object} deps
  * @param {() => object} deps.buildRuntime         current runtime snapshot
  * @param {() => object|undefined} deps.currentPool request-scoped pool (ALS)
- * @param {(pool: object, candidate: string) => void} [deps.onPicked] picked-ref hook
+ * @param {(pool: object, candidate: string, meta?: object) => void} [deps.onPicked] picked-ref hook
  * @param {object} [deps.latencyHistogram]
  * @param {object} [deps.concurrencyTracker]
  * @param {() => number} [deps.now] wall clock; injectable for tests
@@ -60,6 +99,22 @@ export function createResolver({
     }
 
     const at = now();
+
+    // Provider monetary budget refusal (#422):
+    const providerName = pool.provider ?? pool.base?.split?.('::')?.[0];
+    const budget = runtime.providerBudgets?.get?.(providerName);
+    if (budget && budget.pauseOnBudget) {
+      const { daily, weekly } = getProviderCost(providerName, runtime, at);
+      const vDaily = budgetVerdict(daily, budget.costBudgetDaily);
+      const vWeekly = budgetVerdict(weekly, budget.costBudgetWeekly);
+      if (vDaily.exceeded || vWeekly.exceeded) {
+        throw Object.assign(
+          new Error(`dsh-key-rotation: provider "${providerName}" cost budget exceeded (daily: $${daily.toFixed(2)}/${budget.costBudgetDaily}, weekly: $${weekly.toFixed(2)}/${budget.costBudgetWeekly})`),
+          { code: LOCAL_POOL_EXHAUSTED_CODE, localExhausted: true },
+        );
+      }
+    }
+
     const strategy = pool.routingStrategy ?? runtime.routingStrategy ?? 'round-robin';
     let list = pool.weightedRefs ?? pool.refs;
     if (strategy === 'lowest-latency' || strategy === 'least-loaded') {
@@ -82,9 +137,6 @@ export function createResolver({
       const candidateExp = pool.expiresAt?.[candidate] ?? pool.basePool?.expiresAt?.[candidate];
       if (candidateExp !== undefined && at >= candidateExp) continue;
       // Local model token budget: a spent credential is simply not a candidate.
-      // This is deliberately NOT recorded as a failure — the key is healthy and
-      // merely out of budget — so it must never touch failedUntil / failCounts /
-      // authFailCounts / brokenUntil.
       if (quotaManaged && !isModelQuotaAvailable(pool, candidate, at)) continue;
       const rpmLimit = pool.rpmLimit ?? 0;
       if (rpmLimit > 0) {
@@ -119,20 +171,46 @@ export function createResolver({
         }
       }
 
+      // Concurrency limit per-key check (#407):
+      const concurrencyLimit = pool.concurrencyLimit ?? runtime.concurrencyLimit ?? 0;
+      if (concurrencyLimit > 0 && concurrencyTracker && typeof concurrencyTracker.getActive === 'function') {
+        if (concurrencyTracker.getActive(candidate, at) >= concurrencyLimit) {
+          continue;
+        }
+      }
+
+      // Atomically acquire concurrency permit upon candidate selection (#407):
+      let acquired = false;
+      if (concurrencyTracker && typeof concurrencyTracker.acquire === 'function') {
+        const ok = concurrencyTracker.acquire(candidate, at, concurrencyLimit);
+        if (!ok) continue;
+        acquired = true;
+      }
+
       // Advance the shared cursor at selection time: concurrent requests must
       // not all pick the same slot.
       pool.state.pointer = (index + 1) % list.length;
-      const hit = await original(candidate);
-      if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
-        settle(pool, candidate, at, onPicked);
-        return { ...hit, value: decryptSecret(hit.value) };
+      try {
+        const hit = await original(candidate);
+        if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
+          settle(pool, candidate, at, onPicked, acquired ? candidate : undefined);
+          return { ...hit, value: decryptSecret(hit.value) };
+        }
+        // No stored credential: fall back to the launching environment, which is
+        // how env-bootstrapped pools work (issue #7).
+        const envVal = envValue(candidate);
+        if (envVal !== undefined) {
+          settle(pool, candidate, at, onPicked, acquired ? candidate : undefined);
+          return { value: decryptSecret(envVal), source: 'env' };
+        }
+      } catch (err) {
+        if (acquired && typeof concurrencyTracker.release === 'function') {
+          concurrencyTracker.release(candidate, at);
+        }
+        throw err;
       }
-      // No stored credential: fall back to the launching environment, which is
-      // how env-bootstrapped pools work (issue #7).
-      const envVal = envValue(candidate);
-      if (envVal !== undefined) {
-        settle(pool, candidate, at, onPicked);
-        return { value: decryptSecret(envVal), source: 'env' };
+      if (acquired && typeof concurrencyTracker.release === 'function') {
+        concurrencyTracker.release(candidate, at);
       }
     }
 
@@ -154,15 +232,11 @@ export function createResolver({
 }
 
 /** Record a successful credential selection on the pool and request scope. */
-function settle(pool, candidate, now, onPicked) {
+function settle(pool, candidate, now, onPicked, concurrencyRef) {
   pool.state.lastUsed = candidate;
   if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
   pool.state.lastUsedAt.set(candidate, now);
-  onPicked?.(pool, candidate);
-  pool.state.failCounts?.delete(candidate);
-  pool.state.failedUntil.delete(candidate);
-  pool.state.authFailCounts?.delete(candidate);
-  pool.state.brokenUntil?.delete(candidate);
+  onPicked?.(pool, candidate, { concurrencyRef });
   if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
   pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
   if (pool.perHour) {

package/lib/rotate.js

@@ -80,7 +80,7 @@ export function createRotate(deps) {
       let lastFailure = null;
       const breaker = runtime0.circuitBreakerEnabled === false ? null : circuitBreaker;
       let activePermit = null;
-      const reqStore = { pool, pickedRef: undefined, startMs: now() };
+      const reqStore = { pool, pickedRef: undefined, concurrencyRef: undefined, startMs: now() };
       setRotateStartMs(reqStore.startMs);
       const wallNow = Date.now();
       const quotaManaged = hasModelQuotaConfig(pool);
@@ -97,7 +97,7 @@ export function createRotate(deps) {
           const maxDepth = runtimeQ.cascadeMaxDepth ?? CASCADE_MAX_DEPTH;
           const currentDepth = options.__cascadeDepth ?? (options.__isCascade ? 1 : 0);
           if (currentDepth < maxDepth && Array.isArray(runtimeQ.cascade) && runtimeQ.cascade.length > 0) {
-            const fb = pickCascadeFallback(options.provider, runtimeQ, runtimeQ.providerToPool, options.model);
+            const fb = pickCascadeFallback(options.provider, runtimeQ, runtimeQ.providerToPool, options.model, runtimeQ.modelPoolByProvider);
             if (fb && fb.pool && fb.pool !== pool) {
               pool.state.lastReason = 'CASCADE';
               pool.state.lastSwitchAt = now();
@@ -192,7 +192,30 @@ export function createRotate(deps) {
       for (let attempt = 0; attempt < attemptList.length; attempt++) {
         activePermit = breaker ? breaker.acquire(options.provider) : null;
         if (breaker && !activePermit) {
-          logWarn(`[dsh-key-rotation] ${options.provider}: circuit open — skipping dispatch`);
+          logWarn(`[dsh-key-rotation] ${options.provider}: circuit open — trying cascade fallback`);
+          const runtimeQ = buildRuntime();
+          const maxDepth = runtimeQ.cascadeMaxDepth ?? CASCADE_MAX_DEPTH;
+          const currentDepth = options.__cascadeDepth ?? (options.__isCascade ? 1 : 0);
+          if (currentDepth < maxDepth && Array.isArray(runtimeQ.cascade) && runtimeQ.cascade.length > 0) {
+            const fb = pickCascadeFallback(options.provider, runtimeQ, runtimeQ.providerToPool, options.model, runtimeQ.modelPoolByProvider, breaker);
+            if (fb && fb.pool && fb.pool !== pool) {
+              pool.state.lastReason = 'CASCADE';
+              pool.state.lastSwitchAt = now();
+              pool.state.cascadeCount = (pool.state.cascadeCount ?? 0) + 1;
+              pushEvent(pool, reqStore.pickedRef ?? 'pool', 'CASCADE_TRIGGERED', 0);
+              const innerCascade = rotate({
+                ...options,
+                provider: fb.provider,
+                model: fb.model || options.model,
+                __isCascade: true,
+                __cascadeDepth: currentDepth + 1,
+              }, fb.pool);
+              for await (const chunk of innerCascade) {
+                yield chunk;
+              }
+              return;
+            }
+          }
           yield finishError('CIRCUIT_OPEN', `[dsh-key-rotation] provider '${options.provider}' circuit is open`);
           return;
         }
@@ -225,7 +248,10 @@ export function createRotate(deps) {
           }
 
           const _pickedRef = reqStore.pickedRef ?? pool.state.lastUsed;
-          const acquired = (_pickedRef && runtime0.concurrencyLimit > 0) ? concurrencyTracker.acquire(_pickedRef) : false;
+          if (_pickedRef && !reqStore.concurrencyRef && typeof concurrencyTracker?.acquire === 'function') {
+            const ok = concurrencyTracker.acquire(_pickedRef, now(), pool.concurrencyLimit ?? runtime0.concurrencyLimit ?? 0);
+            if (ok) reqStore.concurrencyRef = _pickedRef;
+          }
           try {
             try {
               for await (const chunk of preserveDispatchContext(inner, dispatchStorage, reqStore)) {
@@ -398,11 +424,12 @@ export function createRotate(deps) {
             }
             return; // clean end — served
           } finally {
-            if (acquired && _pickedRef) {
-              concurrencyTracker.release(_pickedRef);
-            }
           }
         } finally {
+          if (reqStore.concurrencyRef) {
+            concurrencyTracker?.release(reqStore.concurrencyRef);
+            reqStore.concurrencyRef = null;
+          }
           // Also runs on consumer return(), abort, empty EOF and local errors.
           activePermit?.release();
         }
@@ -421,7 +448,7 @@ export function createRotate(deps) {
       const currentDepth = options.__cascadeDepth ?? (options.__isCascade ? 1 : 0);
       if (currentDepth < maxDepth && Array.isArray(runtime.cascade) && runtime.cascade.length > 0) {
         const pools = runtime.providerToPool;
-        const fb = pickCascadeFallback(options.provider, runtime, pools, options.model);
+        const fb = pickCascadeFallback(options.provider, runtime, pools, options.model, runtime.modelPoolByProvider, circuitBreaker);
         if (fb && fb.pool && fb.pool !== pool) {
           logWarn(`[dsh-key-rotation] ${options.provider}: pool exhausted — cascading to ${fb.provider} (model: ${fb.model || options.model || 'default'})`);
           pool.state.lastReason = 'CASCADE';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.39",
+  "version": "0.8.41",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
@@ -69,7 +69,11 @@
   },
   "devDependencies": {
     "react": "18.3.1",
-    "react-test-renderer": "18.3.1"
+    "react-test-renderer": "18.3.1",
+    "@deepseek-ai/cordis": "^4.0.1",
+    "@deepseek-ai/dsh-llm": "^0.2.0-rc.2",
+    "@deepseek-ai/schemastery": "^3.18.4",
+    "semver": "^7.6.3"
   },
   "dependencies": {
     "undici": "^6.29.0"