@goodandready/dsh-key-rotation 0.8.65 → 0.8.67

package/lib/ops-config-io.js

@@ -0,0 +1,325 @@
+// lib/ops-config-io.js — full configuration export and import (#504).
+import crypto from 'node:crypto';
+import { json, readJson, isTrustedBridgeRequest, NS } from './http-bridge.js';
+import { CONFIG_EXPORT_PATH, CONFIG_IMPORT_PATH, IMPORT_PATH } from './ops-paths.js';
+import { keyTail } from './pool.js';
+import { plainConfig } from './index.js';
+
+export function encryptConfig(plainText, passphrase) {
+  const salt = crypto.randomBytes(16);
+  const key = crypto.pbkdf2Sync(passphrase, salt, 100000, 32, 'sha256');
+  const iv = crypto.randomBytes(12);
+  const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
+  const encrypted = Buffer.concat([cipher.update(plainText, 'utf8'), cipher.final()]);
+  const tag = cipher.getAuthTag();
+  return {
+    encrypted: true,
+    algorithm: 'aes-256-gcm',
+    kdf: 'pbkdf2',
+    iterations: 100000,
+    salt: salt.toString('hex'),
+    iv: iv.toString('hex'),
+    tag: tag.toString('hex'),
+    ciphertext: encrypted.toString('hex'),
+  };
+}
+
+export function decryptConfig(payload, passphrase) {
+  if (!payload || !payload.salt || !payload.iv || !payload.tag || !payload.ciphertext) {
+    throw new Error('Invalid encrypted config payload structure');
+  }
+  const salt = Buffer.from(payload.salt, 'hex');
+  const iv = Buffer.from(payload.iv, 'hex');
+  const tag = Buffer.from(payload.tag, 'hex');
+  const ciphertext = Buffer.from(payload.ciphertext, 'hex');
+  const key = crypto.pbkdf2Sync(passphrase, salt, payload.iterations || 100000, 32, 'sha256');
+  const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
+  decipher.setAuthTag(tag);
+  const decrypted = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+  return decrypted.toString('utf8');
+}
+
+export function maskConfig(config) {
+  if (!config || typeof config !== 'object') return config;
+  const clone = JSON.parse(JSON.stringify(config));
+  if (Array.isArray(clone.providers)) {
+    for (const prov of clone.providers) {
+      if (Array.isArray(prov.keys)) {
+        prov.keys = prov.keys.map((k) => {
+          if (typeof k === 'string') return k.length > 8 ? `sk-...${k.slice(-4)}` : '***';
+          if (k && typeof k === 'object') {
+            return {
+              ...k,
+              value: typeof k.value === 'string' ? (k.value.length > 8 ? `sk-...${k.value.slice(-4)}` : '***') : '***',
+            };
+          }
+          return '***';
+        });
+      }
+    }
+  }
+  if (clone.webhookUrl && typeof clone.webhookUrl === 'string') {
+    try {
+      const u = new URL(clone.webhookUrl);
+      const parts = u.pathname.split('/');
+      if (parts.length > 2) {
+        parts[parts.length - 1] = '***';
+        u.pathname = parts.join('/');
+      }
+      clone.webhookUrl = u.toString();
+    } catch (_) {
+      clone.webhookUrl = '***';
+    }
+  }
+  if (clone.webhookActionToken) clone.webhookActionToken = '***';
+  return clone;
+}
+
+export function diffConfigs(currentConfig, incomingConfig, mode = 'merge') {
+  const currentProviders = Array.isArray(currentConfig?.providers) ? currentConfig.providers : [];
+  const incomingProviders = Array.isArray(incomingConfig?.providers) ? incomingConfig.providers : [];
+
+  const currentMap = new Map();
+  for (const p of currentProviders) {
+    const name = p.provider || p.name || p.id;
+    if (name) currentMap.set(name, p);
+  }
+
+  const createdProviders = [];
+  const updatedProviders = [];
+  let keysAdded = 0;
+  let keysUpdated = 0;
+
+  for (const p of incomingProviders) {
+    const name = p.provider || p.name || p.id;
+    if (!name) continue;
+    const inKeys = Array.isArray(p.keys) ? p.keys : [];
+    if (!currentMap.has(name)) {
+      createdProviders.push(name);
+      keysAdded += inKeys.length;
+    } else {
+      updatedProviders.push(name);
+      const curP = currentMap.get(name);
+      const curKeys = new Set(Array.isArray(curP.keys) ? curP.keys.map((k) => (typeof k === 'string' ? k : k?.value)) : []);
+      for (const k of inKeys) {
+        const val = typeof k === 'string' ? k : k?.value;
+        if (!curKeys.has(val)) keysAdded++;
+        else keysUpdated++;
+      }
+    }
+  }
+
+  const removedProviders = [];
+  if (mode === 'replace') {
+    const incomingNames = new Set(incomingProviders.map((p) => p.provider || p.name || p.id).filter(Boolean));
+    for (const [name] of currentMap.entries()) {
+      if (!incomingNames.has(name)) removedProviders.push(name);
+    }
+  }
+
+  const totalKeys = incomingProviders.reduce((acc, p) => acc + (Array.isArray(p.keys) ? p.keys.length : 0), 0);
+
+  return {
+    mode,
+    createdProviders,
+    updatedProviders,
+    removedProviders,
+    keysAdded,
+    keysUpdated,
+    totalKeys,
+  };
+}
+
+export function registerConfigIoRoutes(ctx, deps) {
+  // ── GET /dsh-key-rotation/export ──
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: CONFIG_EXPORT_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'GET') {
+        json(res, 405, { error: { code: 'method', message: 'GET only' } });
+        return;
+      }
+      if (!isTrustedBridgeRequest(req)) {
+        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: export is local-only' } });
+        return;
+      }
+
+      const urlObj = new URL(req.url, 'http://127.0.0.1');
+      const maskedParam = urlObj.searchParams.get('masked');
+      const masked = maskedParam === null ? true : maskedParam !== 'false';
+      const passphrase = urlObj.searchParams.get('passphrase') || null;
+
+      const settings = ctx.get('settings');
+      let currentConfig = {};
+      try {
+        if (settings && typeof settings.get === 'function') {
+          currentConfig = settings.get(NS) || {};
+        } else if (settings && typeof settings.describe === 'function') {
+          const desc = settings.describe({ redactSecrets: false })?.find((c) => c.ns === NS);
+          currentConfig = desc?.value || {};
+        } else {
+          currentConfig = deps.buildRuntime().cfg0 || {};
+        }
+      } catch (_) {
+        currentConfig = deps.buildRuntime().cfg0 || {};
+      }
+
+      const rawConfig = plainConfig(currentConfig);
+      const exportData = masked ? maskConfig(rawConfig) : rawConfig;
+
+      if (passphrase) {
+        const encryptedPayload = encryptConfig(JSON.stringify(exportData), passphrase);
+        json(res, 200, {
+          ok: true,
+          version: '0.8.67',
+          exportedAt: new Date().toISOString(),
+          masked,
+          encrypted: true,
+          ...encryptedPayload,
+        });
+        return;
+      }
+
+      json(res, 200, {
+        ok: true,
+        version: '0.8.67',
+        exportedAt: new Date().toISOString(),
+        masked,
+        encrypted: false,
+        config: exportData,
+      });
+    },
+  }));
+
+  // ── POST /dsh-key-rotation/config-import ──
+  const handleImport = async (req, res) => {
+    if (req.method !== 'POST') {
+      json(res, 405, { error: { code: 'method', message: 'POST only' } });
+      return;
+    }
+    if (!isTrustedBridgeRequest(req)) {
+      json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: import is local-only' } });
+      return;
+    }
+
+    let body;
+    try {
+      body = await readJson(req);
+    } catch (e) {
+      json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } });
+      return;
+    }
+
+    let incomingConfig = null;
+    const passphrase = body.passphrase || null;
+
+    if (body.encrypted && body.ciphertext) {
+      if (!passphrase) {
+        json(res, 400, { error: { code: 'passphrase-required', message: 'Passphrase is required for encrypted configuration' } });
+        return;
+      }
+      try {
+        const decryptedStr = decryptConfig(body, passphrase);
+        incomingConfig = JSON.parse(decryptedStr);
+      } catch (err) {
+        json(res, 400, { error: { code: 'decryption-failed', message: `Decryption failed: ${err.message}` } });
+        return;
+      }
+    } else if (body.config) {
+      incomingConfig = body.config;
+    } else if (body.providers) {
+      incomingConfig = body;
+    } else {
+      json(res, 400, { error: { code: 'invalid-payload', message: 'Missing config or providers in request' } });
+      return;
+    }
+
+    if (!incomingConfig || typeof incomingConfig !== 'object') {
+      json(res, 400, { error: { code: 'invalid-config', message: 'Imported configuration must be an object' } });
+      return;
+    }
+
+    const mode = body.mode === 'replace' ? 'replace' : 'merge';
+    const dryRun = Boolean(body.dryRun);
+
+    const settings = ctx.get('settings');
+    let currentConfig = {};
+    try {
+      if (settings && typeof settings.get === 'function') {
+        currentConfig = settings.get(NS) || {};
+      } else if (settings && typeof settings.describe === 'function') {
+        const desc = settings.describe({ redactSecrets: false })?.find((c) => c.ns === NS);
+        currentConfig = desc?.value || {};
+      } else {
+        currentConfig = deps.buildRuntime().cfg0 || {};
+      }
+    } catch (_) {
+      currentConfig = deps.buildRuntime().cfg0 || {};
+    }
+    const rawCurrent = plainConfig(currentConfig);
+
+    const preview = diffConfigs(rawCurrent, incomingConfig, mode);
+
+    if (dryRun) {
+      json(res, 200, {
+        ok: true,
+        dryRun: true,
+        preview,
+      });
+      return;
+    }
+
+    // Apply configuration
+    let mergedConfig;
+    if (mode === 'replace') {
+      mergedConfig = { ...rawCurrent, ...incomingConfig };
+      if (Array.isArray(incomingConfig.providers)) {
+        mergedConfig.providers = incomingConfig.providers;
+      }
+    } else {
+      // Merge mode
+      mergedConfig = { ...rawCurrent, ...incomingConfig };
+      const curProviders = Array.isArray(rawCurrent.providers) ? [...rawCurrent.providers] : [];
+      const curMap = new Map();
+      for (let i = 0; i < curProviders.length; i++) {
+        const name = curProviders[i].provider || curProviders[i].name || curProviders[i].id;
+        if (name) curMap.set(name, i);
+      }
+      const inProviders = Array.isArray(incomingConfig.providers) ? incomingConfig.providers : [];
+      for (const p of inProviders) {
+        const name = p.provider || p.name || p.id;
+        if (!name) continue;
+        if (curMap.has(name)) {
+          const idx = curMap.get(name);
+          curProviders[idx] = { ...curProviders[idx], ...p };
+        } else {
+          curProviders.push(p);
+        }
+      }
+      mergedConfig.providers = curProviders;
+    }
+
+    if (settings && typeof settings.replace === 'function') {
+      try {
+        await settings.replace(NS, mergedConfig);
+      } catch (err) {
+        json(res, 500, { error: { code: 'settings-write-failed', message: err.message } });
+        return;
+      }
+    }
+
+    json(res, 200, {
+      ok: true,
+      applied: true,
+      mode,
+      preview,
+    });
+  };
+
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: CONFIG_IMPORT_PATH,
+    handler: handleImport,
+  }));
+}

package/lib/ops-paths.js

@@ -18,3 +18,7 @@ export const TEST_MATRIX_PATH = '/dsh-key-rotation/test-matrix';
 export const DISCOVER_PATH = '/dsh-key-rotation/discover';
 export const AUTO_IMPORT_PATH = '/dsh-key-rotation/auto-import';
 export const PRESET_LIMITS_PATH = '/dsh-key-rotation/preset-limits';
+export const BALANCE_PATH = '/dsh-key-rotation/balance';
+export const CONFIG_EXPORT_PATH = '/dsh-key-rotation/export';
+export const CONFIG_IMPORT_PATH = '/dsh-key-rotation/config-import';
+export const QUICK_ACTION_PATH = '/dsh-key-rotation/quick-action';

package/lib/ops-quick-action.js

@@ -0,0 +1,167 @@
+import { json, readJson, isTrustedBridgeRequest } from './http-bridge.js';
+import { QUICK_ACTION_PATH } from './ops-paths.js';
+import { keyTail, isKeyPaused } from './pool.js';
+
+/**
+ * @param {object} ctx cordis context
+ * @param {object} deps live dependencies from apply()
+ */
+export function registerQuickActionRoutes(ctx, deps) {
+  const { buildRuntime, pushEvent, circuitBreaker } = deps;
+
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: QUICK_ACTION_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'POST') {
+        json(res, 405, { error: { code: 'method', message: 'POST only' } });
+        return;
+      }
+      if (!isTrustedBridgeRequest(req)) {
+        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: quick-action is local-only' } });
+        return;
+      }
+
+      let body;
+      try {
+        body = await readJson(req);
+      } catch (e) {
+        json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } });
+        return;
+      }
+
+      const action = typeof body?.action === 'string' ? body.action.trim().toLowerCase() : '';
+      const provider = typeof body?.provider === 'string' ? body.provider.trim() : '';
+      const ref = typeof body?.ref === 'string' ? body.ref.trim() : null;
+
+      if (!action || !['skip', 'reset', 'pause'].includes(action)) {
+        json(res, 400, { error: { code: 'invalid-action', message: 'action must be skip, reset, or pause' } });
+        return;
+      }
+
+      if (!provider) {
+        json(res, 400, { error: { code: 'missing-provider', message: 'provider is required' } });
+        return;
+      }
+
+      const runtime = buildRuntime();
+      let targetPool = null;
+
+      const pools = runtime.pools ?? Array.from(runtime.poolByRef?.values() || []);
+      for (const pool of pools) {
+        if (pool.provider === provider || pool.base === provider) {
+          targetPool = pool;
+          break;
+        }
+      }
+
+      if (!targetPool) {
+        json(res, 404, { error: { code: 'pool-not-found', message: `Pool not found for provider: ${provider}` } });
+        return;
+      }
+
+      if (action === 'skip') {
+        const total = targetPool.refs?.length || 0;
+        if (total === 0) {
+          json(res, 400, { error: { code: 'no-keys', message: 'Pool has no keys to rotate' } });
+          return;
+        }
+        const currentCursor = targetPool.state.cursor ?? 0;
+        const nextCursor = (currentCursor + 1) % total;
+        targetPool.state.cursor = nextCursor;
+        const nextRef = targetPool.refs[nextCursor];
+        targetPool.state.lastUsed = nextRef;
+
+        if (typeof pushEvent === 'function') {
+          pushEvent(targetPool, nextRef, 'QUICK_ACTION_SKIP', 0);
+        }
+
+        json(res, 200, {
+          ok: true,
+          action: 'skip',
+          provider,
+          cursor: nextCursor,
+          activeRef: nextRef,
+          activeTail: keyTail(nextRef),
+        });
+        return;
+      }
+
+      if (action === 'reset') {
+        targetPool.state.failedUntil?.clear();
+        targetPool.state.failCounts?.clear();
+        targetPool.state.resetReasons?.clear();
+        targetPool.state.headerResetTimes?.clear();
+        targetPool.state.switches = 0;
+        targetPool.state.lastReason = null;
+
+        const breaker = circuitBreaker ?? runtime.breaker;
+        if (breaker) {
+          if (typeof breaker.reset === 'function') breaker.reset(provider);
+          else if (typeof breaker.onSuccess === 'function') breaker.onSuccess(provider);
+        }
+
+        if (typeof pushEvent === 'function') {
+          pushEvent(targetPool, '*', 'QUICK_ACTION_RESET', 0);
+        }
+
+        json(res, 200, {
+          ok: true,
+          action: 'reset',
+          provider,
+          healthyKeys: targetPool.refs?.length || 0,
+        });
+        return;
+      }
+
+      if (action === 'pause') {
+        const targetRef = ref || targetPool.state.lastUsed || targetPool.refs?.[0];
+        if (!targetRef) {
+          json(res, 400, { error: { code: 'no-ref', message: 'No key specified or active in pool' } });
+          return;
+        }
+
+        let isPaused = false;
+        if (!targetPool.state.pausedRefs) targetPool.state.pausedRefs = new Set();
+
+        if (targetPool.state.pausedRefs instanceof Set) {
+          if (targetPool.state.pausedRefs.has(targetRef)) {
+            targetPool.state.pausedRefs.delete(targetRef);
+            targetPool.state.pauseReasons?.delete(targetRef);
+            isPaused = false;
+          } else {
+            targetPool.state.pausedRefs.add(targetRef);
+            if (!targetPool.state.pauseReasons) targetPool.state.pauseReasons = new Map();
+            targetPool.state.pauseReasons.set(targetRef, 'MANUAL_QUICK_PAUSED');
+            isPaused = true;
+          }
+        } else if (Array.isArray(targetPool.state.pausedRefs)) {
+          const idx = targetPool.state.pausedRefs.indexOf(targetRef);
+          if (idx >= 0) {
+            targetPool.state.pausedRefs.splice(idx, 1);
+            targetPool.state.pauseReasons?.delete(targetRef);
+            isPaused = false;
+          } else {
+            targetPool.state.pausedRefs.push(targetRef);
+            if (!targetPool.state.pauseReasons) targetPool.state.pauseReasons = new Map();
+            targetPool.state.pauseReasons.set(targetRef, 'MANUAL_QUICK_PAUSED');
+            isPaused = true;
+          }
+        }
+
+        if (typeof pushEvent === 'function') {
+          pushEvent(targetPool, targetRef, isPaused ? 'KEY_PAUSED' : 'KEY_RESUMED', 0);
+        }
+
+        json(res, 200, {
+          ok: true,
+          action: 'pause',
+          provider,
+          ref: targetRef,
+          paused: isPaused,
+        });
+        return;
+      }
+    },
+  }));
+}

package/lib/ops-status.js

@@ -124,6 +124,9 @@ ctx.effect(() => ctx.webServer.register({
           revoked: isKeyRevoked(pool, ref),
           modelCapabilities: pool.state.modelCapabilities?.get?.(ref) ? [...pool.state.modelCapabilities.get(ref)] : null,
           detectedLimits: pool.state.detectedLimits?.get?.(ref) ?? null,
+          balance: pool.state.balances?.get?.(ref) ?? null,
+          resetReason: pool.state.resetReasons?.get?.(ref) ?? null,
+          headerResetMs: pool.state.headerResetTimes?.get?.(ref) ?? null,
           modelQuota,
           modelPools: activeModelPools.map(mp => mp.base),
         });
@@ -239,6 +242,8 @@ ctx.effect(() => ctx.webServer.register({
         providerName: pool.provider ?? providerOfBase(pool.base),
         basePool: basePoolH?.base ?? null,
         modelQuota: quotaAware ? { configured: true, exhaustedKeys: quotaExhausted } : null,
+        lastUsed: pool.state.lastUsed ?? null,
+        lastUsedTail: pool.state.lastUsed ? keyTail(pool.state.lastUsed) : null,
       };
     }
     let circuitOpenAny = false;

package/lib/persistence.js

@@ -138,6 +138,13 @@ export class StatePersistence {
         if (pausedRefs.length > 0) outPool.pausedRefs = pausedRefs;
         if (Object.keys(pauseReasons).length > 0) outPool.pauseReasons = pauseReasons;
         if (Object.keys(modelCapabilities).length > 0) outPool.modelCapabilities = modelCapabilities;
+        const balances = {};
+        if (st && st.balances && typeof st.balances.forEach === 'function') {
+          st.balances.forEach((bal, ref) => {
+            if (bal && typeof bal === 'object') balances[ref] = bal;
+          });
+        }
+        if (Object.keys(balances).length > 0) outPool.balances = balances;
         if (Object.keys(tokenUsage).length > 0) outPool.tokenUsage = tokenUsage;
         if (Object.keys(costDays).length > 0) outPool.costDays = costDays;
         if (Object.keys(costPerKey).length > 0) outPool.costPerKey = costPerKey;
@@ -241,6 +248,12 @@ export class StatePersistence {
             if (typeof reason === 'string') state.pauseReasons.set(ref, reason);
           }
         }
+        if (saved.balances && typeof saved.balances === 'object' && !Array.isArray(saved.balances)) {
+          if (!state.balances) state.balances = new Map();
+          for (const [ref, bal] of Object.entries(saved.balances)) {
+            if (bal && typeof bal === 'object') state.balances.set(ref, bal);
+          }
+        }
         if (saved.modelCapabilities && typeof saved.modelCapabilities === 'object' && !Array.isArray(saved.modelCapabilities)) {
           if (!state.modelCapabilities) state.modelCapabilities = new Map();
           for (const [ref, list] of Object.entries(saved.modelCapabilities)) {

package/lib/pool-state.js

@@ -3,7 +3,7 @@
 const MAP_FIELDS = [
   'failedUntil', 'failCounts', 'authFailCounts', 'brokenUntil', 'costPerKey',
   'lastUsedAt', 'usageCounts', 'byModel', 'usageDays', 'quotaWindows', 'costDays',
-  'tokenUsage', 'rpmWindows', 'tpmWindows', 'sessionSticky', 'modelCapabilities', 'pauseReasons', 'detectedLimits',
+  'tokenUsage', 'rpmWindows', 'tpmWindows', 'sessionSticky', 'modelCapabilities', 'pauseReasons', 'detectedLimits', 'balances', 'resetReasons', 'headerResetTimes',
 ];
 
 const SET_FIELDS = ['pausedRefs', 'revokedRefs'];

package/lib/rotate.js

@@ -197,16 +197,51 @@ export function createRotate(deps) {
         }
       }
 
-      const penalizeRef = (targetRef, errCode, errMsg) => {
+      const penalizeRef = (targetRef, errCode, errMsg, rawHeaders) => {
         if (!targetRef) return;
         if (sessionId) clearSessionSticky(pool, sessionId);
         if (errCode === LOCAL_QUOTA_CODE) return;
-        const _retry = parseRetryAfter(errMsg);
+        let headerResetMs = null;
+        if (rawHeaders) {
+          const limits = (typeof rawHeaders === 'object' && rawHeaders.rpmResetMs !== undefined)
+            ? rawHeaders
+            : extractProviderLimits(rawHeaders);
+          headerResetMs = limits?.retryAfterMs ?? limits?.rpmResetMs ?? null;
+        }
+        if (!headerResetMs) {
+          const pRetry = parseRetryAfter(errMsg);
+          if (pRetry) headerResetMs = pRetry;
+        }
+
         const _base = pool.cooldownMs ?? cooldownMs;
-        const _max = pool.maxCooldownMs ?? maxCooldownMs;
-        const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base;
+        const _max = pool.maxCooldownMs ?? maxCooldownMs ?? 3600000;
+        let _b = 0;
         const cls = classifyFailure({ status: Number(errCode) || 0, code: errCode, message: errMsg });
-        const _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft, true);
+
+        if (headerResetMs && headerResetMs > 0) {
+          const dynamicCooldownMs = Math.min(_max, Math.max(1000, Math.floor(headerResetMs) + 250));
+          pool.state.failedUntil.set(targetRef, now() + dynamicCooldownMs);
+          if (!pool.state.resetReasons) pool.state.resetReasons = new Map();
+          pool.state.resetReasons.set(targetRef, 'header');
+          if (!pool.state.headerResetTimes) pool.state.headerResetTimes = new Map();
+          pool.state.headerResetTimes.set(targetRef, dynamicCooldownMs);
+          _b = dynamicCooldownMs;
+          pushEvent(pool, targetRef, 'HEADER_RESET', dynamicCooldownMs);
+          logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} dynamic cooldown set to ${dynamicCooldownMs}ms via upstream headers`);
+
+          const recTimer = setTimeout(() => {
+            if ((pool.state.failedUntil?.get?.(targetRef) ?? 0) <= now()) {
+              pool.state.failedUntil.delete(targetRef);
+              pushEvent(pool, targetRef, 'DYNAMIC_UNBREAK', 0);
+            }
+          }, dynamicCooldownMs + 50);
+          if (typeof recTimer.unref === 'function') recTimer.unref();
+        } else {
+          const _effBase = _base;
+          _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft, true);
+          if (!pool.state.resetReasons) pool.state.resetReasons = new Map();
+          pool.state.resetReasons.set(targetRef, 'backoff');
+        }
 
         // #337 & #486 (GitHub #24): connect quota-window calendar reset for quota exhaustion
         // Transient rate limiting (RPM/TPM, requests/tokens per minute, concurrency)
@@ -409,7 +444,7 @@ export function createRotate(deps) {
               break;
             }
             const curRef = reqStore.pickedRef ?? pool.state.lastUsed;
-            penalizeRef(curRef, e?.code ?? 'TRANSPORT', String(e?.message ?? ''));
+            penalizeRef(curRef, e?.code ?? 'TRANSPORT', String(e?.message ?? ''), e?.headers ?? e?.response?.headers ?? pool.state.detectedLimits?.get?.(curRef));
             lastFailure = finishError(e?.code ?? 'TRANSPORT',
               `dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
             logWarn(`[dsh-key-rotation] ${options.provider}: key ${String(curRef ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
@@ -434,6 +469,7 @@ export function createRotate(deps) {
                   continue;
                 }
                 if (chunk && chunk.type === 'finish') {
+                  const rawHdrs = chunk?.metadata?.headers ?? chunk?.headers ?? chunk?.reason?.failure?.headers;
                   const kind = chunk.reason?.kind;
                   const failure = chunk.reason?.failure;
                   const code = failure?.code;
@@ -442,7 +478,7 @@ export function createRotate(deps) {
                   const switchable = !yielded && kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes);
                   const activeRef = reqStore.pickedRef ?? pool.state.lastUsed;
                   if (switchable) {
-                    penalizeRef(activeRef, code ?? 'UNKNOWN', message);
+                    penalizeRef(activeRef, code ?? 'UNKNOWN', message, rawHdrs ?? failure?.headers ?? pool.state.detectedLimits?.get?.(activeRef));
                     pool.state.switches = (pool.state.switches ?? 0) + 1;
                     pool.state.lastReason = String(code ?? 'UNKNOWN');
                     pool.state.lastSwitchAt = now();
@@ -464,7 +500,7 @@ export function createRotate(deps) {
                   // content delta we may penalize, but must never replay the request.
                   if (kind === 'error' || kind === 'aborted') {
                     if (kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes)) {
-                      penalizeRef(activeRef, code ?? 'UNKNOWN', message);
+                      penalizeRef(activeRef, code ?? 'UNKNOWN', message, rawHdrs ?? failure?.headers ?? pool.state.detectedLimits?.get?.(activeRef));
                     }
                     yield chunk;
                     return;
@@ -528,7 +564,6 @@ export function createRotate(deps) {
                   // its quota or has retry-after, cool it down so the NEXT request starts on a different key.
                   // We do NOT re-run this (already successful) request — that would double-send.
                   // #503: Extract provider limits and reset headers
-                  const rawHdrs = chunk?.metadata?.headers ?? chunk?.headers;
                   if (rawHdrs && activeRef) {
                     const detected = extractProviderLimits(rawHdrs);
                     if (detected) {
@@ -574,7 +609,7 @@ export function createRotate(deps) {
               const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
               const activeRef = _pickedRef ?? reqStore.pickedRef ?? pool.state.lastUsed;
               if (!yielded && isSwitchableError(e, effectiveSwitchCodes)) {
-                penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+                penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e), e?.headers ?? e?.response?.headers ?? pool.state.detectedLimits?.get?.(activeRef));
                 pool.state.switches = (pool.state.switches ?? 0) + 1;
                 pool.state.lastReason = String(e?.code ?? 'TRANSPORT');
                 pool.state.lastSwitchAt = now();
@@ -591,7 +626,7 @@ export function createRotate(deps) {
                 continue; // Failover to next key!
               }
               if (yielded && isSwitchableError(e, effectiveSwitchCodes)) {
-                penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+                penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e), e?.headers ?? e?.response?.headers ?? pool.state.detectedLimits?.get?.(activeRef));
               }
               yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
               return;