@goodandready/dsh-key-rotation 0.8.80 → 0.8.81

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## 0.8.81 (2026-10-30)
+
+### Direct Extension Pipeline & Pure Key Resolution (Block 9 Release)
+- **Direct Extension Pipeline without Monkey-Patching (Issue #526)**: Introduced clean credential injection and direct resolution pipeline via `resolveKeyDirectly` (`lib/resolver.js`). Bypasses monkey-patching of global `credentials.resolve` and direct mutations on foreign services.
+- **Explicit Stream Options Delivery (`lib/rotate.js`)**: Passes resolved secrets directly through `directOpts` (`apiKey`, `headers.Authorization`, `_rotationKeyRef`) into `llm.stream`, allowing provider adapters to consume credentials without touching global process state. Preserved `dispatchStorage` as backward-compatible fallback for legacy runtimes.
+- **Dedicated Key Penalty Manager (`lib/rotate-penalty.js`)**: Extracted `penalizeKey` into dedicated module (<200 lines) with dynamic header rate-limit tracking, backoff computation, quota window synchronization, early warning notifications, and cross-pool permanent pause fan-out, maintaining `lib/rotate.js` well below the 600-line modularity limit.
+
+## 0.8.80 (2026-10-30)
+
+### Layered Core Decoupling, Append-Only Journal & Dynamic Cordis Injection (Block 8 Release)
+- **Layered Core Architecture (Issue #530)**: Reorganized algorithmic core into `lib/core/` (`state-machine.js`, `concurrency.js`, `quota.js`, `pool.js`, `rotate.js`, `index.js`) with backwards-compatible top-level facades in `lib/*.js`.
+- **Dynamic Cordis Injection (Issue #528)**: Decoupled optional dependencies via dynamic `ctx.inject(['webServer'], ...)`, minimizing static requirements to `['llm']` and ensuring smooth headless/CLI execution.
+- **Append-Only Journal Persistence (Issue #525)**: Implemented incremental append-only journaling (`*-journal.jsonl`) in `StatePersistence`, eliminating O(N) full-disk state serialization on routine key cooldowns and rotation events with automated rotation upon `flush()`.
+
+## 0.8.79 (2026-10-30)
+
+### Modular Route Layer, Schemastery Volatile & Key State Machine (Block 7 Release)
+- **Modular Route Structure (Issue #522)**: Extracted route handlers into dedicated `lib/routes/` directory (`status.js`, `keys.js`, `metrics.js`, `balance.js`, `telemetry.js`, `testing.js`, `webhook.js`, `config.js`, `quick-action.js`, `guard.js`, `paths.js`) with legacy compatibility facades in `lib/ops-*.js`.
+- **Schemastery Volatile Config Migration (Issue #551)**: Implemented `unwrapVolatileConfig` and deprecation headers (`X-DSH-Deprecated`, `Deprecation`) on deprecated config endpoints.
+- **Key State Machine (Issue #529)**: Introduced finite state machine `KeyStateMachine` (`lib/key-state-machine.js`) with strict transition matrix (`VALID_TRANSITIONS`), audit trail, and state statistics.
+
 ## 0.8.78 (2026-10-30)
 
 ### Network Transports, Model Catalog & Predictive Quota Runway (Block 6 Release)

package/lib/resolver.js

@@ -309,3 +309,39 @@ function settle(pool, candidate, now, onPicked, concurrencyRef, sessionId) {
     pool.state.quotaWindows.set(candidate, win);
   }
 }
+
+/**
+ * Resolves credential secret directly without requiring monkey-patching (#526).
+ * Reads from credentials service safely or falls back to environment variables.
+ * @param {string} candidateRef
+ * @param {object} [ctx]
+ * @returns {Promise<string|null>}
+ */
+export async function resolveKeyDirectly(candidateRef, ctx) {
+  if (!candidateRef || typeof candidateRef !== 'string') return null;
+  try {
+    const creds = ctx?.get?.('credentials') ?? ctx?.credentials;
+    if (creds) {
+      if (typeof creds.get === 'function') {
+        const val = await creds.get(candidateRef);
+        if (val) return decryptSecret(typeof val === 'object' ? val.value : val);
+      }
+      if (typeof creds.resolve === 'function') {
+        const orig = creds.__dshKeyRotationOriginalResolve ?? creds.resolve;
+        const hit = await orig.call(creds, candidateRef);
+        if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
+          return decryptSecret(hit.value);
+        }
+      }
+    }
+  } catch {
+    /* credentials miss or access error falls through to env */
+  }
+
+  const envVal = envValue(candidateRef);
+  if (envVal !== undefined && typeof envVal === 'string' && envVal.length > 0) {
+    return decryptSecret(envVal);
+  }
+
+  return null;
+}

package/lib/rotate-penalty.js

@@ -0,0 +1,193 @@
+// lib/rotate-penalty.js — Key penalty, backoff, dynamic headers, and failure tracking (#526).
+import { classifyFailure } from './error-taxonomy.js';
+import {
+  recordFailure,
+  parseRetryAfter,
+  isKeyPaused,
+  isKeyRevoked,
+  isKeyBroken,
+  clearSessionSticky,
+  unstickFailedKey,
+} from './pool.js';
+import { extractProviderLimits } from './provider-limits-detector.js';
+import { poolResetAt } from './quota-window.js';
+
+export const LOCAL_QUOTA_CODE = 'LOCAL_MODEL_QUOTA_EXHAUSTED';
+
+/**
+ * Penalize a key upon error or upstream failure during rotation.
+ */
+export function penalizeKey(targetRef, errCode, errMsg, rawHeaders, {
+  pool,
+  options,
+  sessionId,
+  runtime0,
+  cooldownMs,
+  maxCooldownMs,
+  now,
+  activePermit,
+  trace,
+  pushEvent,
+  schedulePersist,
+  effectiveNotifyEarlyWarning,
+  logWarn,
+  buildRuntime,
+}) {
+  if (!targetRef) return;
+  if (sessionId) clearSessionSticky(pool, sessionId);
+  unstickFailedKey(pool, targetRef);
+  if (errCode === LOCAL_QUOTA_CODE) return;
+
+  let headerResetMs = null;
+  if (rawHeaders) {
+    const limits = (typeof rawHeaders === 'object' && rawHeaders.rpmResetMs !== undefined)
+      ? rawHeaders
+      : extractProviderLimits(rawHeaders);
+    headerResetMs = limits?.retryAfterMs ?? limits?.rpmResetMs ?? null;
+  }
+  if (!headerResetMs) {
+    const pRetry = parseRetryAfter(errMsg);
+    if (pRetry) headerResetMs = pRetry;
+  }
+
+  const _base = pool.cooldownMs ?? cooldownMs;
+  const _max = pool.maxCooldownMs ?? maxCooldownMs ?? 3600000;
+  let _b = 0;
+  const cls = classifyFailure({ status: Number(errCode) || 0, code: errCode, message: errMsg });
+
+  if (headerResetMs && headerResetMs > 0) {
+    const dynamicCooldownMs = Math.min(_max, Math.max(1000, Math.floor(headerResetMs) + 250));
+    pool.state.failedUntil.set(targetRef, now() + dynamicCooldownMs);
+    if (!pool.state.resetReasons) pool.state.resetReasons = new Map();
+    pool.state.resetReasons.set(targetRef, 'header');
+    if (!pool.state.headerResetTimes) pool.state.headerResetTimes = new Map();
+    pool.state.headerResetTimes.set(targetRef, dynamicCooldownMs);
+    _b = dynamicCooldownMs;
+    pushEvent(pool, targetRef, 'HEADER_RESET', dynamicCooldownMs);
+    if (logWarn) logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} dynamic cooldown set to ${dynamicCooldownMs}ms via upstream headers`);
+
+    const recTimer = setTimeout(() => {
+      if ((pool.state.failedUntil?.get?.(targetRef) ?? 0) <= now()) {
+        pool.state.failedUntil.delete(targetRef);
+        pushEvent(pool, targetRef, 'DYNAMIC_UNBREAK', 0);
+      }
+    }, dynamicCooldownMs + 50);
+    if (typeof recTimer.unref === 'function') recTimer.unref();
+  } else {
+    const _effBase = _base;
+    _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft, true);
+    if (!pool.state.resetReasons) pool.state.resetReasons = new Map();
+    pool.state.resetReasons.set(targetRef, 'backoff');
+  }
+
+  // #337 & #486 (GitHub #24): connect quota-window calendar reset for quota exhaustion
+  // Transient rate limiting (RPM/TPM, requests/tokens per minute, concurrency)
+  // must NOT be locked until midnight. Keep those on ordinary cooldowns.
+  const isRateThrottling = /\b(?:rpm|tpm|requests?\s+per\s+minute|tokens?\s+per\s+minute|per[-_\s]*minute|per[-_\s]*second|concurrency|requests?\s+limit|tokens?\s+limit|traffic)\b/i.test(errMsg) ||
+    (/\b(?:rate[-_\s]*limit)\b/i.test(errMsg) && !/\b(?:insufficient[_\s-]+quota|exceeded\s+(?:your\s+)?(?:current\s+)?quota|credit\s+balance|billing)\b/i.test(errMsg));
+
+  const isQuota = !isRateThrottling && (
+    (errCode === 'QUOTA' && !/\b(?:rpm|tpm|rate|traffic|throttle)\b/i.test(errMsg)) ||
+    /\b(?:quota(?:\s+is)?\s+(?:exhausted|exceeded)|exceeded\s+(?:your\s+)?(?:current\s+)?quota|insufficient[_\s-]+quota|credit\s+balance|billing\s+quota|out\s+of\s+credits?)\b/i.test(errMsg)
+  );
+  const qWindow = pool.quotaResetWindow ?? runtime0.quotaResetWindow;
+  if (isQuota && qWindow) {
+    const quotaResetAt = poolResetAt(pool, qWindow, now());
+    if (quotaResetAt && quotaResetAt > now()) {
+      if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+      pool.state.quotaWindows.set(targetRef, quotaResetAt);
+      pool.state.failedUntil.set(targetRef, quotaResetAt);
+    }
+  }
+
+  activePermit?.failure();
+  trace.addStep(targetRef, 'FAILED', errCode ?? 'UNKNOWN', errMsg);
+  pushEvent(pool, targetRef, errCode ?? 'UNKNOWN', _b);
+  if (typeof schedulePersist === 'function') {
+    try { schedulePersist(); } catch (_) { /* best-effort persistence */ }
+  }
+  if (runtime0.earlyWarningNotify && effectiveNotifyEarlyWarning) {
+    const total = pool.refs?.length || 1;
+    let available = 0;
+    const at = now();
+    for (const r of (pool.refs || [])) {
+      if (!isKeyRevoked(pool, r) && !isKeyPaused(pool, r) && !isKeyBroken(pool, r, at) && ((pool.state?.failedUntil?.get(r) ?? 0) <= at)) {
+        available++;
+      }
+    }
+    const threshold = Number.isFinite(runtime0.earlyWarningKeyThreshold) ? runtime0.earlyWarningKeyThreshold : 0.25;
+    if (total > 0 && (available / total) <= threshold) {
+      effectiveNotifyEarlyWarning(runtime0, pool, {
+        provider: options.provider,
+        availableCount: available,
+        totalCount: total,
+      });
+    }
+  }
+  if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
+  if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
+  const _cStr = String(errCode ?? '').toUpperCase();
+  const _isAuth = cls.reason === 'auth' || cls.code === '401' || cls.code === '403' ||
+    _cStr === 'AUTH' || _cStr === '401' || _cStr === '403' ||
+    _cStr === 'PERMISSION_DENIED' || _cStr === 'UNAUTHENTICATED' ||
+    /auth|unauthorized|invalid[_\s-]?api[_\s-]?key|unauthenticated|permission[_\s-]?denied/i.test(errMsg);
+
+  if (_isAuth) {
+    const _c2 = (pool.state.authFailCounts.get(targetRef) ?? 0) + 1;
+    pool.state.authFailCounts.set(targetRef, _c2);
+    if (!pool.state.revokedRefs) pool.state.revokedRefs = new Set();
+    pool.state.revokedRefs.add(targetRef);
+    if (!pool.state.pausedRefs) pool.state.pausedRefs = new Set();
+    if (pool.state.pausedRefs instanceof Set) pool.state.pausedRefs.add(targetRef);
+    else if (Array.isArray(pool.state.pausedRefs) && !pool.state.pausedRefs.includes(targetRef)) pool.state.pausedRefs.push(targetRef);
+    if (!pool.state.pauseReasons) pool.state.pauseReasons = new Map();
+    pool.state.pauseReasons.set(targetRef, 'AUTH_FAILURE_PAUSED');
+    pool.state.failedUntil.set(targetRef, Infinity);
+    pushEvent(pool, targetRef, 'KEY_AUTH_PAUSED', 0);
+    if (logWarn) logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently PAUSED on auth failure (${errCode})`);
+    // Fan-out permanent pause across all pools sharing this physical ref (#408):
+    if (typeof buildRuntime === 'function') {
+      const rAll = buildRuntime();
+      const candidatePools = [
+        ...(Array.isArray(rAll.pools) ? rAll.pools : []),
+        ...(rAll.index?.allPools ? rAll.index.allPools.values() : []),
+        ...(rAll.providerToPool ? rAll.providerToPool.values() : []),
+      ];
+      if (rAll.modelPoolByProvider) {
+        for (const map of rAll.modelPoolByProvider.values()) { if (map) candidatePools.push(...map.values()); }
+      }
+      const seenPools = new Set();
+      for (const other of candidatePools) {
+        if (!other || seenPools.has(other)) continue;
+        seenPools.add(other);
+        if (other === pool) continue;
+        const hasRef = (Array.isArray(other.refs) && other.refs.includes(targetRef)) ||
+          (other.weightsMap instanceof Map && other.weightsMap.has(targetRef));
+        if (hasRef && other.state) {
+          if (!other.state.revokedRefs) other.state.revokedRefs = new Set();
+          other.state.revokedRefs.add(targetRef);
+          if (!other.state.pausedRefs) other.state.pausedRefs = new Set();
+          if (other.state.pausedRefs instanceof Set) other.state.pausedRefs.add(targetRef);
+          else if (Array.isArray(other.state.pausedRefs) && !other.state.pausedRefs.includes(targetRef)) other.state.pausedRefs.push(targetRef);
+          if (!other.state.pauseReasons) other.state.pauseReasons = new Map();
+          other.state.pauseReasons.set(targetRef, 'AUTH_FAILURE_PAUSED');
+          if (other.state.failedUntil) other.state.failedUntil.set(targetRef, Infinity);
+          pushEvent(other, targetRef, 'KEY_AUTH_PAUSED', 0);
+        }
+      }
+    }
+  } else {
+    pool.state.authFailCounts.delete(targetRef);
+    // Soft/transient errors (TIMEOUT, 5xx, socket) do not count toward quarantine (#473).
+    // recordFailure already maintains pool.state.failCounts accurately for non-soft failures.
+    if (!isQuota && !cls.soft) {
+      const _fc = pool.state.failCounts.get(targetRef) ?? 0;
+      if (_fc >= 3 && !pool.state.brokenUntil.has(targetRef)) {
+        const quarantineMs = 1800000;
+        pool.state.brokenUntil.set(targetRef, Date.now() + quarantineMs);
+        pushEvent(pool, targetRef, 'KEY_QUARANTINED', quarantineMs);
+        if (logWarn) logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} put in recoverable quarantine after ${_fc} failures`);
+      }
+    }
+  }
+}

package/lib/rotate.js

@@ -1,12 +1,9 @@
 // lib/rotate.js — stream rotation / failover generator (#253).
 // Pure wiring helper: dependencies are injected so unit tests can supply mocks.
 import { nowMono } from './clock.js';
-import { classifyFailure } from './error-taxonomy.js';
 import {
   isSwitchableError,
-  recordFailure,
   recordSuccess,
-  parseRetryAfter,
   isKeyPaused,
   isKeyRevoked,
   isKeyBroken,
@@ -14,15 +11,12 @@ import {
   sortAttemptList,
   getSessionSticky,
   setSessionSticky,
-  clearSessionSticky,
-  unstickFailedKey,
 } from './pool.js';
 import { pickCascadeFallback, CASCADE_MAX_DEPTH } from './cascade.js';
 import { createRequestBudget, BUDGET_EXHAUSTED_TIMEOUT, BUDGET_EXHAUSTED_ATTEMPTS } from './request-budget.js';
 import { createSelectionTrace } from './selection-trace.js';
 import { isKeyDraining } from './key-drain.js';
 import { notifyEarlyWarning as defaultNotifyEarlyWarning } from './notify-events.js';
-import { nextQuotaReset, poolResetAt } from './quota-window.js';
 import {
   isModelQuotaAvailable,
   hasModelQuotaConfig,
@@ -30,9 +24,9 @@ import {
   formatModelQuotaExhaustion,
 } from './model-quota.js';
 
-import { LOCAL_POOL_EXHAUSTED_CODE } from './resolver.js';
+import { LOCAL_POOL_EXHAUSTED_CODE, resolveKeyDirectly } from './resolver.js';
 import { recordStreamSuccessMetrics, preserveDispatchContext } from './stream-metrics.js';
-import { extractProviderLimits } from './provider-limits-detector.js';
+import { penalizeKey } from './rotate-penalty.js';
 import { recordPromptCache } from './prompt-cache.js';
 import { recordRequestOutcome, recordFailureCategory } from './ops-analytics.js';
 
@@ -209,160 +203,22 @@ export function createRotate(deps) {
       }
 
       const penalizeRef = (targetRef, errCode, errMsg, rawHeaders) => {
-        if (!targetRef) return;
-        if (sessionId) clearSessionSticky(pool, sessionId);
-        unstickFailedKey(pool, targetRef);
-        if (errCode === LOCAL_QUOTA_CODE) return;
-        let headerResetMs = null;
-        if (rawHeaders) {
-          const limits = (typeof rawHeaders === 'object' && rawHeaders.rpmResetMs !== undefined)
-            ? rawHeaders
-            : extractProviderLimits(rawHeaders);
-          headerResetMs = limits?.retryAfterMs ?? limits?.rpmResetMs ?? null;
-        }
-        if (!headerResetMs) {
-          const pRetry = parseRetryAfter(errMsg);
-          if (pRetry) headerResetMs = pRetry;
-        }
-
-        const _base = pool.cooldownMs ?? cooldownMs;
-        const _max = pool.maxCooldownMs ?? maxCooldownMs ?? 3600000;
-        let _b = 0;
-        const cls = classifyFailure({ status: Number(errCode) || 0, code: errCode, message: errMsg });
-
-        if (headerResetMs && headerResetMs > 0) {
-          const dynamicCooldownMs = Math.min(_max, Math.max(1000, Math.floor(headerResetMs) + 250));
-          pool.state.failedUntil.set(targetRef, now() + dynamicCooldownMs);
-          if (!pool.state.resetReasons) pool.state.resetReasons = new Map();
-          pool.state.resetReasons.set(targetRef, 'header');
-          if (!pool.state.headerResetTimes) pool.state.headerResetTimes = new Map();
-          pool.state.headerResetTimes.set(targetRef, dynamicCooldownMs);
-          _b = dynamicCooldownMs;
-          pushEvent(pool, targetRef, 'HEADER_RESET', dynamicCooldownMs);
-          logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} dynamic cooldown set to ${dynamicCooldownMs}ms via upstream headers`);
-
-          const recTimer = setTimeout(() => {
-            if ((pool.state.failedUntil?.get?.(targetRef) ?? 0) <= now()) {
-              pool.state.failedUntil.delete(targetRef);
-              pushEvent(pool, targetRef, 'DYNAMIC_UNBREAK', 0);
-            }
-          }, dynamicCooldownMs + 50);
-          if (typeof recTimer.unref === 'function') recTimer.unref();
-        } else {
-          const _effBase = _base;
-          _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft, true);
-          if (!pool.state.resetReasons) pool.state.resetReasons = new Map();
-          pool.state.resetReasons.set(targetRef, 'backoff');
-        }
-
-        // #337 & #486 (GitHub #24): connect quota-window calendar reset for quota exhaustion
-        // Transient rate limiting (RPM/TPM, requests/tokens per minute, concurrency)
-        // must NOT be locked until midnight. Keep those on ordinary cooldowns.
-        const isRateThrottling = /\b(?:rpm|tpm|requests?\s+per\s+minute|tokens?\s+per\s+minute|per[-_\s]*minute|per[-_\s]*second|concurrency|requests?\s+limit|tokens?\s+limit|traffic)\b/i.test(errMsg) ||
-          (/\b(?:rate[-_\s]*limit)\b/i.test(errMsg) && !/\b(?:insufficient[_\s-]+quota|exceeded\s+(?:your\s+)?(?:current\s+)?quota|credit\s+balance|billing)\b/i.test(errMsg));
-
-        const isQuota = !isRateThrottling && (
-          (errCode === 'QUOTA' && !/\b(?:rpm|tpm|rate|traffic|throttle)\b/i.test(errMsg)) ||
-          /\b(?:quota(?:\s+is)?\s+(?:exhausted|exceeded)|exceeded\s+(?:your\s+)?(?:current\s+)?quota|insufficient[_\s-]+quota|credit\s+balance|billing\s+quota|out\s+of\s+credits?)\b/i.test(errMsg)
-        );
-        const qWindow = pool.quotaResetWindow ?? runtime0.quotaResetWindow;
-        if (isQuota && qWindow) {
-          const quotaResetAt = poolResetAt(pool, qWindow, now());
-          if (quotaResetAt && quotaResetAt > now()) {
-            if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
-            pool.state.quotaWindows.set(targetRef, quotaResetAt);
-            pool.state.failedUntil.set(targetRef, quotaResetAt);
-          }
-        }
-
-        activePermit?.failure();
-        trace.addStep(targetRef, 'FAILED', errCode ?? 'UNKNOWN', errMsg);
-        pushEvent(pool, targetRef, errCode ?? 'UNKNOWN', _b);
-        if (typeof schedulePersist === 'function') {
-          try { schedulePersist(); } catch (_) { /* best-effort persistence */ }
-        }
-        if (runtime0.earlyWarningNotify && effectiveNotifyEarlyWarning) {
-          const total = pool.refs?.length || 1;
-          let available = 0;
-          const at = now();
-          for (const r of (pool.refs || [])) {
-            if (!isKeyRevoked(pool, r) && !isKeyPaused(pool, r) && !isKeyBroken(pool, r, at) && ((pool.state?.failedUntil?.get(r) ?? 0) <= at)) {
-              available++;
-            }
-          }
-          const threshold = Number.isFinite(runtime0.earlyWarningKeyThreshold) ? runtime0.earlyWarningKeyThreshold : 0.25;
-          if (total > 0 && (available / total) <= threshold) {
-            effectiveNotifyEarlyWarning(runtime0, pool, {
-              provider: options.provider,
-              availableCount: available,
-              totalCount: total,
-            });
-          }
-        }
-        if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
-        if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
-        const _cStr = String(errCode ?? '').toUpperCase();
-        const _isAuth = cls.reason === 'auth' || cls.code === '401' || cls.code === '403' ||
-          _cStr === 'AUTH' || _cStr === '401' || _cStr === '403' ||
-          _cStr === 'PERMISSION_DENIED' || _cStr === 'UNAUTHENTICATED' ||
-          /auth|unauthorized|invalid[_\s-]?api[_\s-]?key|unauthenticated|permission[_\s-]?denied/i.test(errMsg);
-
-        if (_isAuth) {
-          const _c2 = (pool.state.authFailCounts.get(targetRef) ?? 0) + 1;
-          pool.state.authFailCounts.set(targetRef, _c2);
-          if (!pool.state.revokedRefs) pool.state.revokedRefs = new Set();
-          pool.state.revokedRefs.add(targetRef);
-          if (!pool.state.pausedRefs) pool.state.pausedRefs = new Set();
-          if (pool.state.pausedRefs instanceof Set) pool.state.pausedRefs.add(targetRef);
-          else if (Array.isArray(pool.state.pausedRefs) && !pool.state.pausedRefs.includes(targetRef)) pool.state.pausedRefs.push(targetRef);
-          if (!pool.state.pauseReasons) pool.state.pauseReasons = new Map();
-          pool.state.pauseReasons.set(targetRef, 'AUTH_FAILURE_PAUSED');
-          pool.state.failedUntil.set(targetRef, Infinity);
-          pushEvent(pool, targetRef, 'KEY_AUTH_PAUSED', 0);
-          logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently PAUSED on auth failure (${errCode})`);
-          // Fan-out permanent pause across all pools sharing this physical ref (#408):
-          const rAll = buildRuntime();
-          const candidatePools = [
-            ...(Array.isArray(rAll.pools) ? rAll.pools : []),
-            ...(rAll.index?.allPools ? rAll.index.allPools.values() : []),
-            ...(rAll.providerToPool ? rAll.providerToPool.values() : []),
-          ];
-          if (rAll.modelPoolByProvider) {
-            for (const map of rAll.modelPoolByProvider.values()) { if (map) candidatePools.push(...map.values()); }
-          }
-          const seenPools = new Set();
-          for (const other of candidatePools) {
-            if (!other || seenPools.has(other)) continue;
-            seenPools.add(other);
-            if (other === pool) continue;
-            const hasRef = (Array.isArray(other.refs) && other.refs.includes(targetRef)) ||
-              (other.weightsMap instanceof Map && other.weightsMap.has(targetRef));
-            if (hasRef && other.state) {
-              if (!other.state.revokedRefs) other.state.revokedRefs = new Set();
-              other.state.revokedRefs.add(targetRef);
-              if (!other.state.pausedRefs) other.state.pausedRefs = new Set();
-              if (other.state.pausedRefs instanceof Set) other.state.pausedRefs.add(targetRef);
-              else if (Array.isArray(other.state.pausedRefs) && !other.state.pausedRefs.includes(targetRef)) other.state.pausedRefs.push(targetRef);
-              if (!other.state.pauseReasons) other.state.pauseReasons = new Map();
-              other.state.pauseReasons.set(targetRef, 'AUTH_FAILURE_PAUSED');
-              if (other.state.failedUntil) other.state.failedUntil.set(targetRef, Infinity);
-              pushEvent(other, targetRef, 'KEY_AUTH_PAUSED', 0);
-            }
-          }
-        } else {
-          pool.state.authFailCounts.delete(targetRef);
-          // Soft/transient errors (TIMEOUT, 5xx, socket) do not count toward quarantine (#473).
-          // recordFailure already maintains pool.state.failCounts accurately for non-soft failures.
-          if (!isQuota && !cls.soft) {
-            const _fc = pool.state.failCounts.get(targetRef) ?? 0;
-            if (_fc >= 3 && !pool.state.brokenUntil.has(targetRef)) {
-              const quarantineMs = 1800000;
-              pool.state.brokenUntil.set(targetRef, Date.now() + quarantineMs);
-              pushEvent(pool, targetRef, 'KEY_QUARANTINED', quarantineMs);
-              logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} put in recoverable quarantine after ${_fc} failures`);
-            }
-          }
-        }
+        penalizeKey(targetRef, errCode, errMsg, rawHeaders, {
+          pool,
+          options,
+          sessionId,
+          runtime0,
+          cooldownMs,
+          maxCooldownMs,
+          now,
+          activePermit,
+          trace,
+          pushEvent,
+          schedulePersist,
+          effectiveNotifyEarlyWarning,
+          logWarn,
+          buildRuntime,
+        });
       };
 
       for (let attempt = 0; attempt < attemptList.length; attempt++) {
@@ -420,13 +276,23 @@ export function createRotate(deps) {
             const curCandidate = reqStore.pickedRef ?? pool.state.lastUsed ?? attemptList[attempt];
             trace.addStep(curCandidate, 'DISPATCHED', 'ATTEMPT', `Attempt ${budget.getAttempts()}`);
 
-            // mark the internal dispatch so the interceptor does not re-rotate
+            // #526: Direct key resolution without relying on credentials.resolve monkey-patch
+            const directSecret = await resolveKeyDirectly(curCandidate, ctx);
+            const directOpts = { ...options, [MARKER]: true, signal: budget.signal, _rotationKeyRef: curCandidate };
+            if (directSecret) {
+              directOpts.apiKey = directSecret;
+              if (!directOpts.headers) directOpts.headers = {};
+              if (!directOpts.headers.Authorization && !directOpts.headers.authorization) {
+                directOpts.headers.Authorization = `Bearer ${directSecret}`;
+              }
+            }
+
             inner = dispatchStorage.run(reqStore, () => {
               const llm = ctx.get('llm');
               if (!llm || typeof llm.stream !== 'function') {
                 throw new Error('dsh-key-rotation: llm service unavailable');
               }
-              return llm.stream({ ...options, [MARKER]: true, signal: budget.signal });
+              return llm.stream(directOpts);
             });
           } catch (e) {
             if (e?.localQuota || e?.localExhausted || e?.code === LOCAL_QUOTA_CODE || e?.code === LOCAL_POOL_EXHAUSTED_CODE) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.80",
+  "version": "0.8.81",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [