@goodandready/dsh-key-rotation 0.8.79 → 0.8.81

package/lib/key-state-machine.js

@@ -1,267 +1,2 @@
-// lib/key-state-machine.js — Encapsulated Key State Machine (#529).
-// Manages credential lifecycle across deterministic states and valid transitions.
-
-export const KEY_STATES = Object.freeze({
-  ACTIVE: 'ACTIVE',
-  COOLDOWN: 'COOLDOWN',
-  PROBING: 'PROBING',
-  EXHAUSTED: 'EXHAUSTED',
-  PAUSED: 'PAUSED',
-  REVOKED: 'REVOKED',
-});
-
-export const VALID_TRANSITIONS = Object.freeze({
-  [KEY_STATES.ACTIVE]: Object.freeze([
-    KEY_STATES.COOLDOWN,
-    KEY_STATES.EXHAUSTED,
-    KEY_STATES.PAUSED,
-    KEY_STATES.REVOKED,
-  ]),
-  [KEY_STATES.COOLDOWN]: Object.freeze([
-    KEY_STATES.PROBING,
-    KEY_STATES.ACTIVE,
-    KEY_STATES.PAUSED,
-    KEY_STATES.REVOKED,
-  ]),
-  [KEY_STATES.PROBING]: Object.freeze([
-    KEY_STATES.ACTIVE,
-    KEY_STATES.COOLDOWN,
-    KEY_STATES.PAUSED,
-    KEY_STATES.REVOKED,
-  ]),
-  [KEY_STATES.EXHAUSTED]: Object.freeze([
-    KEY_STATES.ACTIVE,
-    KEY_STATES.COOLDOWN,
-    KEY_STATES.PAUSED,
-    KEY_STATES.REVOKED,
-  ]),
-  [KEY_STATES.PAUSED]: Object.freeze([
-    KEY_STATES.ACTIVE,
-    KEY_STATES.COOLDOWN,
-    KEY_STATES.REVOKED,
-  ]),
-  [KEY_STATES.REVOKED]: Object.freeze([
-    KEY_STATES.ACTIVE,
-  ]),
-});
-
-export class KeyStateMachine {
-  /**
-   * @param {object} [options]
-   * @param {number} [options.maxHistory] Maximum transition history entries per pool
-   */
-  constructor(options = {}) {
-    /** @type {Map<string, { state: string, updatedAt: number, previousState?: string, reason?: string, failedUntil?: number }>} */
-    this.keyRecords = new Map();
-    /** @type {Array<{ ref: string, from: string, to: string, at: number, reason: string|null }>} */
-    this.history = [];
-    this.maxHistory = options.maxHistory ?? 100;
-  }
-
-  /**
-   * Computes the current deterministic state of a key reference.
-   *
-   * @param {string} keyRef
-   * @param {object} [context] Optional pool state context for backward compatibility
-   * @param {Set<string>} [context.revokedRefs]
-   * @param {Set<string>} [context.pausedRefs]
-   * @param {Map<string, number>} [context.failedUntil]
-   * @param {number} [now]
-   * @returns {string} Current KEY_STATES value
-   */
-  getState(keyRef, context = {}, now = Date.now()) {
-    if (!keyRef) return KEY_STATES.ACTIVE;
-
-    // 1. Revocation is absolute
-    if (context.revokedRefs && context.revokedRefs.has(keyRef)) {
-      return KEY_STATES.REVOKED;
-    }
-
-    // 2. Manual pause overrides operational state
-    if (context.pausedRefs && context.pausedRefs.has(keyRef)) {
-      return KEY_STATES.PAUSED;
-    }
-
-    const record = this.keyRecords.get(keyRef);
-    if (!record) {
-      const failedUntil = context.failedUntil?.get?.(keyRef);
-      if (typeof failedUntil === 'number' && failedUntil > now) {
-        return KEY_STATES.COOLDOWN;
-      }
-      return KEY_STATES.ACTIVE;
-    }
-
-    // Cooldown transition to probing when timer elapses
-    if (record.state === KEY_STATES.COOLDOWN) {
-      const until = record.failedUntil ?? context.failedUntil?.get?.(keyRef);
-      if (typeof until === 'number' && until <= now) {
-        return KEY_STATES.PROBING;
-      }
-    }
-
-    return record.state;
-  }
-
-  /**
-   * Transitions a key reference to a new state if valid.
-   *
-   * @param {string} keyRef
-   * @param {string} targetState Target KEY_STATES value
-   * @param {object} [metadata]
-   * @param {string} [metadata.reason]
-   * @param {number} [metadata.failedUntil]
-   * @param {object} [metadata.context]
-   * @param {number} [now]
-   * @returns {string|null} Resulting state or null on failure
-   */
-  transition(keyRef, targetState, metadata = {}, now = Date.now()) {
-    if (!keyRef || !targetState) return null;
-
-    const currentState = this.getState(keyRef, metadata.context, now);
-    if (currentState === targetState) return currentState;
-
-    const allowed = VALID_TRANSITIONS[currentState] || [];
-    if (!allowed.includes(targetState)) {
-      // In relaxed mode, force transition with logged anomaly
-      targetState = targetState in KEY_STATES ? targetState : KEY_STATES.ACTIVE;
-    }
-
-    const record = this.keyRecords.get(keyRef) ?? {
-      state: currentState,
-      updatedAt: now,
-    };
-
-    record.previousState = currentState;
-    record.state = targetState;
-    record.updatedAt = now;
-    if (metadata.reason !== undefined) record.reason = metadata.reason;
-    if (metadata.failedUntil !== undefined) record.failedUntil = metadata.failedUntil;
-
-    this.keyRecords.set(keyRef, record);
-
-    const event = {
-      ref: keyRef,
-      from: currentState,
-      to: targetState,
-      at: now,
-      reason: metadata.reason ?? null,
-    };
-
-    this.history.push(event);
-    if (this.history.length > this.maxHistory) {
-      this.history.shift();
-    }
-
-    // Synchronize external pool state if provided
-    if (metadata.context) {
-      if (targetState === KEY_STATES.REVOKED) {
-        metadata.context.revokedRefs?.add?.(keyRef);
-        metadata.context.pausedRefs?.delete?.(keyRef);
-      } else if (targetState === KEY_STATES.PAUSED) {
-        metadata.context.pausedRefs?.add?.(keyRef);
-      } else if (currentState === KEY_STATES.PAUSED && targetState !== KEY_STATES.PAUSED) {
-        metadata.context.pausedRefs?.delete?.(keyRef);
-      }
-    }
-
-    return targetState;
-  }
-
-  /**
-   * Puts key into COOLDOWN state until specified duration elapses.
-   */
-  transitionToCooldown(keyRef, options = {}, now = Date.now()) {
-    const cooldownMs = Number(options.cooldownMs) > 0 ? Number(options.cooldownMs) : 60000;
-    const failedUntil = now + cooldownMs;
-    return this.transition(keyRef, KEY_STATES.COOLDOWN, {
-      failedUntil,
-      reason: options.reason || 'COOLDOWN',
-      context: options.context,
-    }, now);
-  }
-
-  /** Marks key as ready for canary probe dispatch. */
-  markProbing(keyRef, metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.PROBING, {
-      reason: metadata.reason || 'CANARY_PROBE',
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Activates key following successful request or probe. */
-  activate(keyRef, metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.ACTIVE, {
-      reason: metadata.reason || 'SUCCESS',
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Marks key as exhausted due to local quota or TPM/RPM window. */
-  markExhausted(keyRef, metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.EXHAUSTED, {
-      reason: metadata.reason || 'QUOTA_EXHAUSTED',
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Manually pauses key from rotation. */
-  pause(keyRef, reason = 'USER_PAUSE', metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.PAUSED, {
-      reason,
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Unpauses key, returning it to active rotation. */
-  unpause(keyRef, metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.ACTIVE, {
-      reason: 'USER_RESUME',
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Revokes key permanently until credentials change. */
-  revoke(keyRef, reason = 'INVALID_CREDENTIAL', metadata = {}, now = Date.now()) {
-    return this.transition(keyRef, KEY_STATES.REVOKED, {
-      reason,
-      context: metadata.context,
-    }, now);
-  }
-
-  /** Checks if key is usable for dispatch (ACTIVE or PROBING). */
-  isUsable(keyRef, context = {}, now = Date.now()) {
-    const state = this.getState(keyRef, context, now);
-    return state === KEY_STATES.ACTIVE || state === KEY_STATES.PROBING;
-  }
-
-  /** Returns transition history for a specific key or all keys. */
-  getHistory(keyRef) {
-    if (!keyRef) return this.history.slice();
-    return this.history.filter((e) => e.ref === keyRef);
-  }
-
-  /** Returns state distribution statistics across a set of keys. */
-  getStats(refs = [], context = {}, now = Date.now()) {
-    const stats = {
-      total: refs.length,
-      [KEY_STATES.ACTIVE]: 0,
-      [KEY_STATES.COOLDOWN]: 0,
-      [KEY_STATES.PROBING]: 0,
-      [KEY_STATES.EXHAUSTED]: 0,
-      [KEY_STATES.PAUSED]: 0,
-      [KEY_STATES.REVOKED]: 0,
-    };
-    for (const ref of refs) {
-      const state = this.getState(ref, context, now);
-      if (state in stats) stats[state]++;
-    }
-    return stats;
-  }
-
-  clear() {
-    this.keyRecords.clear();
-    this.history = [];
-  }
-}
-
-export const defaultKeyStateMachine = new KeyStateMachine();
+// lib/key-state-machine.js — Backwards-compatible facade (#530).
+export * from './core/state-machine.js';

package/lib/persistence.js

@@ -1,8 +1,9 @@
-// lib/persistence.js — atomic snapshot of rotation state across restarts (#287).
-// Cooldowns, circuit-breaker entries and quota counters live in memory; a DSH
-// reload would otherwise forget cooldowns and stampede recovered keys.
+// lib/persistence.js — atomic snapshot and append-only event journal (#287, #525).
+// Cooldowns, circuit-breaker entries and quota counters live in memory; an append-only
+// journal records incremental events without rewriting the entire state tree on disk.
 
 import path from 'node:path';
+import fs from 'node:fs';
 import { initializePoolState, hasPoolActivity } from './pool-state.js';
 import { atomicWriteJson, safeReadJson } from './atomic-io.js';
 
@@ -18,6 +19,9 @@ export class StatePersistence {
       throw new Error('StatePersistence: filePath is required');
     }
     this.filePath = filePath;
+    this.journalPath = filePath.endsWith('.json')
+      ? filePath.slice(0, -5) + '-journal.jsonl'
+      : `${filePath}.jsonl`;
     this._now = now;
     this._timer = null;
     this._dirty = false;
@@ -112,8 +116,8 @@ export class StatePersistence {
             if (val instanceof Map || (typeof val === 'object' && !Array.isArray(val))) {
               const innerObj = {};
               const innerEntries = val instanceof Map ? val.entries() : Object.entries(val);
-              for (const [day, c] of innerEntries) {
-                const num = Number(c);
+              for (const [day, count] of innerEntries) {
+                const num = Number(count);
                 if (Number.isFinite(num) && num >= 0) innerObj[day] = Math.floor(num);
               }
               if (Object.keys(innerObj).length > 0) usageDays[k] = innerObj;
@@ -168,7 +172,22 @@ export class StatePersistence {
   }
 
   /**
-   * Schedule a debounced atomic write. Never throws to the caller.
+   * Append-only event journaling (#525). Records incremental event without full state rewrite.
+   * @param {object} event
+   */
+  async appendEvent(event) {
+    if (!event || typeof event !== 'object') return false;
+    const line = JSON.stringify({ ...event, at: event.at ?? this._nowSafe() }) + '\n';
+    try {
+      await fs.promises.appendFile(this.journalPath, line, 'utf-8');
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Schedule a debounced atomic snapshot write. Never throws to the caller.
    * @param {object} payload result of StatePersistence.serialize
    */
   save(payload) {
@@ -190,9 +209,11 @@ export class StatePersistence {
     try {
       await atomicWriteJson(this.filePath, payload);
       this._lastWrite = this._nowSafe();
+      try {
+        await fs.promises.writeFile(this.journalPath, '', 'utf-8');
+      } catch {}
       return true;
     } catch {
-      // Keep previous good file; mark dirty so a later save retries.
       this._dirty = true;
       this._pending = this._pending || payload;
       return false;
@@ -200,16 +221,56 @@ export class StatePersistence {
   }
 
   /**
-   * Load a snapshot. Corrupt/missing file → null (never wipes memory).
+   * Load snapshot and replay any journaled events (#525).
    * @returns {Promise<object|null>}
    */
   async load() {
     const raw = await safeReadJson(this.filePath, null);
     if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null;
     if (raw.version !== 1 && raw.version !== 2) return null;
+
+    if (fs.existsSync(this.journalPath)) {
+      try {
+        const content = await fs.promises.readFile(this.journalPath, 'utf-8');
+        const lines = content.split('\n');
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (trimmed) {
+            try { this.replayEvent(raw, JSON.parse(trimmed)); } catch {}
+          }
+        }
+      } catch {}
+    }
+
     return raw;
   }
 
+  /** Replay individual event into a snapshot object (#525). */
+  replayEvent(snapshot, event) {
+    if (!snapshot || !event?.pool || !event?.ref) return;
+    if (!snapshot.pools) snapshot.pools = {};
+    if (!snapshot.pools[event.pool]) {
+      snapshot.pools[event.pool] = { failedUntil: {}, pausedRefs: [], revokedRefs: [] };
+    }
+    const p = snapshot.pools[event.pool];
+    if (event.type === 'cooldown' || event.type === 'failure') {
+      if (!p.failedUntil) p.failedUntil = {};
+      p.failedUntil[event.ref] = event.until ?? (event.at + (event.cooldownMs || 60000));
+    } else if (event.type === 'success' || event.type === 'reset') {
+      if (p.failedUntil) delete p.failedUntil[event.ref];
+    } else if (event.type === 'pause') {
+      if (!Array.isArray(p.pausedRefs)) p.pausedRefs = [];
+      if (!p.pausedRefs.includes(event.ref)) p.pausedRefs.push(event.ref);
+    } else if (event.type === 'unpause') {
+      if (Array.isArray(p.pausedRefs)) {
+        p.pausedRefs = p.pausedRefs.filter((r) => r !== event.ref);
+      }
+    } else if (event.type === 'revoke') {
+      if (!Array.isArray(p.revokedRefs)) p.revokedRefs = [];
+      if (!p.revokedRefs.includes(event.ref)) p.revokedRefs.push(event.ref);
+    }
+  }
+
   /** Restore poolState Map from a loaded snapshot. */
   static restorePools(poolState, snapshot) {
     if (!poolState || !snapshot?.pools || typeof snapshot.pools !== 'object' || Array.isArray(snapshot.pools)) return 0;
@@ -218,8 +279,6 @@ export class StatePersistence {
       if (!saved || typeof saved !== 'object' || Array.isArray(saved)) continue;
       const existing = poolState.get(base);
       const state = initializePoolState(existing);
-      // Never replace objects retained by buildRuntime(), nor roll back a pool
-      // already used while the asynchronous load was in flight.
       if (!existing || !hasPoolActivity(state)) {
         if (saved.failedUntil && typeof saved.failedUntil === 'object' && !Array.isArray(saved.failedUntil)) {
           for (const [ref, until] of Object.entries(saved.failedUntil)) {

package/lib/quota.js

@@ -1,39 +1,2 @@
-// lib/quota.js — quota-remaining persistence per ref.
-// ponytail: pure helpers, snapshot() returns shallow copies.
-
-export class QuotaStore {
-  constructor() {
-    this._data = new Map(); // ref -> { remaining, limit, reset, at }
-  }
-
-  set(ref, info) {
-    if (!ref) return;
-    if (!info || typeof info !== 'object') return;
-    const next = {
-      remaining: Number.isFinite(info.remaining) ? info.remaining : null,
-      limit: Number.isFinite(info.limit) ? info.limit : null,
-      reset: Number.isFinite(info.reset) ? info.reset : null,
-      at: Number.isFinite(info.at) ? info.at : Date.now(),
-    };
-    this._data.set(ref, next);
-  }
-
-  get(ref) {
-    return this._data.get(ref);
-  }
-
-  snapshot() {
-    const out = {};
-    for (const [k, v] of this._data) out[k] = v;
-    return out;
-  }
-
-  clear(ref) {
-    if (ref) this._data.delete(ref);
-    else this._data.clear();
-  }
-
-  get size() {
-    return this._data.size;
-  }
-}
+// lib/quota.js — Backwards-compatible facade (#530).
+export * from './core/quota.js';

package/lib/resolver.js

@@ -309,3 +309,39 @@ function settle(pool, candidate, now, onPicked, concurrencyRef, sessionId) {
     pool.state.quotaWindows.set(candidate, win);
   }
 }
+
+/**
+ * Resolves credential secret directly without requiring monkey-patching (#526).
+ * Reads from credentials service safely or falls back to environment variables.
+ * @param {string} candidateRef
+ * @param {object} [ctx]
+ * @returns {Promise<string|null>}
+ */
+export async function resolveKeyDirectly(candidateRef, ctx) {
+  if (!candidateRef || typeof candidateRef !== 'string') return null;
+  try {
+    const creds = ctx?.get?.('credentials') ?? ctx?.credentials;
+    if (creds) {
+      if (typeof creds.get === 'function') {
+        const val = await creds.get(candidateRef);
+        if (val) return decryptSecret(typeof val === 'object' ? val.value : val);
+      }
+      if (typeof creds.resolve === 'function') {
+        const orig = creds.__dshKeyRotationOriginalResolve ?? creds.resolve;
+        const hit = await orig.call(creds, candidateRef);
+        if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
+          return decryptSecret(hit.value);
+        }
+      }
+    }
+  } catch {
+    /* credentials miss or access error falls through to env */
+  }
+
+  const envVal = envValue(candidateRef);
+  if (envVal !== undefined && typeof envVal === 'string' && envVal.length > 0) {
+    return decryptSecret(envVal);
+  }
+
+  return null;
+}

package/lib/rotate-penalty.js

@@ -0,0 +1,193 @@
+// lib/rotate-penalty.js — Key penalty, backoff, dynamic headers, and failure tracking (#526).
+import { classifyFailure } from './error-taxonomy.js';
+import {
+  recordFailure,
+  parseRetryAfter,
+  isKeyPaused,
+  isKeyRevoked,
+  isKeyBroken,
+  clearSessionSticky,
+  unstickFailedKey,
+} from './pool.js';
+import { extractProviderLimits } from './provider-limits-detector.js';
+import { poolResetAt } from './quota-window.js';
+
+export const LOCAL_QUOTA_CODE = 'LOCAL_MODEL_QUOTA_EXHAUSTED';
+
+/**
+ * Penalize a key upon error or upstream failure during rotation.
+ */
+export function penalizeKey(targetRef, errCode, errMsg, rawHeaders, {
+  pool,
+  options,
+  sessionId,
+  runtime0,
+  cooldownMs,
+  maxCooldownMs,
+  now,
+  activePermit,
+  trace,
+  pushEvent,
+  schedulePersist,
+  effectiveNotifyEarlyWarning,
+  logWarn,
+  buildRuntime,
+}) {
+  if (!targetRef) return;
+  if (sessionId) clearSessionSticky(pool, sessionId);
+  unstickFailedKey(pool, targetRef);
+  if (errCode === LOCAL_QUOTA_CODE) return;
+
+  let headerResetMs = null;
+  if (rawHeaders) {
+    const limits = (typeof rawHeaders === 'object' && rawHeaders.rpmResetMs !== undefined)
+      ? rawHeaders
+      : extractProviderLimits(rawHeaders);
+    headerResetMs = limits?.retryAfterMs ?? limits?.rpmResetMs ?? null;
+  }
+  if (!headerResetMs) {
+    const pRetry = parseRetryAfter(errMsg);
+    if (pRetry) headerResetMs = pRetry;
+  }
+
+  const _base = pool.cooldownMs ?? cooldownMs;
+  const _max = pool.maxCooldownMs ?? maxCooldownMs ?? 3600000;
+  let _b = 0;
+  const cls = classifyFailure({ status: Number(errCode) || 0, code: errCode, message: errMsg });
+
+  if (headerResetMs && headerResetMs > 0) {
+    const dynamicCooldownMs = Math.min(_max, Math.max(1000, Math.floor(headerResetMs) + 250));
+    pool.state.failedUntil.set(targetRef, now() + dynamicCooldownMs);
+    if (!pool.state.resetReasons) pool.state.resetReasons = new Map();
+    pool.state.resetReasons.set(targetRef, 'header');
+    if (!pool.state.headerResetTimes) pool.state.headerResetTimes = new Map();
+    pool.state.headerResetTimes.set(targetRef, dynamicCooldownMs);
+    _b = dynamicCooldownMs;
+    pushEvent(pool, targetRef, 'HEADER_RESET', dynamicCooldownMs);
+    if (logWarn) logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} dynamic cooldown set to ${dynamicCooldownMs}ms via upstream headers`);
+
+    const recTimer = setTimeout(() => {
+      if ((pool.state.failedUntil?.get?.(targetRef) ?? 0) <= now()) {
+        pool.state.failedUntil.delete(targetRef);
+        pushEvent(pool, targetRef, 'DYNAMIC_UNBREAK', 0);
+      }
+    }, dynamicCooldownMs + 50);
+    if (typeof recTimer.unref === 'function') recTimer.unref();
+  } else {
+    const _effBase = _base;
+    _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft, true);
+    if (!pool.state.resetReasons) pool.state.resetReasons = new Map();
+    pool.state.resetReasons.set(targetRef, 'backoff');
+  }
+
+  // #337 & #486 (GitHub #24): connect quota-window calendar reset for quota exhaustion
+  // Transient rate limiting (RPM/TPM, requests/tokens per minute, concurrency)
+  // must NOT be locked until midnight. Keep those on ordinary cooldowns.
+  const isRateThrottling = /\b(?:rpm|tpm|requests?\s+per\s+minute|tokens?\s+per\s+minute|per[-_\s]*minute|per[-_\s]*second|concurrency|requests?\s+limit|tokens?\s+limit|traffic)\b/i.test(errMsg) ||
+    (/\b(?:rate[-_\s]*limit)\b/i.test(errMsg) && !/\b(?:insufficient[_\s-]+quota|exceeded\s+(?:your\s+)?(?:current\s+)?quota|credit\s+balance|billing)\b/i.test(errMsg));
+
+  const isQuota = !isRateThrottling && (
+    (errCode === 'QUOTA' && !/\b(?:rpm|tpm|rate|traffic|throttle)\b/i.test(errMsg)) ||
+    /\b(?:quota(?:\s+is)?\s+(?:exhausted|exceeded)|exceeded\s+(?:your\s+)?(?:current\s+)?quota|insufficient[_\s-]+quota|credit\s+balance|billing\s+quota|out\s+of\s+credits?)\b/i.test(errMsg)
+  );
+  const qWindow = pool.quotaResetWindow ?? runtime0.quotaResetWindow;
+  if (isQuota && qWindow) {
+    const quotaResetAt = poolResetAt(pool, qWindow, now());
+    if (quotaResetAt && quotaResetAt > now()) {
+      if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+      pool.state.quotaWindows.set(targetRef, quotaResetAt);
+      pool.state.failedUntil.set(targetRef, quotaResetAt);
+    }
+  }
+
+  activePermit?.failure();
+  trace.addStep(targetRef, 'FAILED', errCode ?? 'UNKNOWN', errMsg);
+  pushEvent(pool, targetRef, errCode ?? 'UNKNOWN', _b);
+  if (typeof schedulePersist === 'function') {
+    try { schedulePersist(); } catch (_) { /* best-effort persistence */ }
+  }
+  if (runtime0.earlyWarningNotify && effectiveNotifyEarlyWarning) {
+    const total = pool.refs?.length || 1;
+    let available = 0;
+    const at = now();
+    for (const r of (pool.refs || [])) {
+      if (!isKeyRevoked(pool, r) && !isKeyPaused(pool, r) && !isKeyBroken(pool, r, at) && ((pool.state?.failedUntil?.get(r) ?? 0) <= at)) {
+        available++;
+      }
+    }
+    const threshold = Number.isFinite(runtime0.earlyWarningKeyThreshold) ? runtime0.earlyWarningKeyThreshold : 0.25;
+    if (total > 0 && (available / total) <= threshold) {
+      effectiveNotifyEarlyWarning(runtime0, pool, {
+        provider: options.provider,
+        availableCount: available,
+        totalCount: total,
+      });
+    }
+  }
+  if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
+  if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
+  const _cStr = String(errCode ?? '').toUpperCase();
+  const _isAuth = cls.reason === 'auth' || cls.code === '401' || cls.code === '403' ||
+    _cStr === 'AUTH' || _cStr === '401' || _cStr === '403' ||
+    _cStr === 'PERMISSION_DENIED' || _cStr === 'UNAUTHENTICATED' ||
+    /auth|unauthorized|invalid[_\s-]?api[_\s-]?key|unauthenticated|permission[_\s-]?denied/i.test(errMsg);
+
+  if (_isAuth) {
+    const _c2 = (pool.state.authFailCounts.get(targetRef) ?? 0) + 1;
+    pool.state.authFailCounts.set(targetRef, _c2);
+    if (!pool.state.revokedRefs) pool.state.revokedRefs = new Set();
+    pool.state.revokedRefs.add(targetRef);
+    if (!pool.state.pausedRefs) pool.state.pausedRefs = new Set();
+    if (pool.state.pausedRefs instanceof Set) pool.state.pausedRefs.add(targetRef);
+    else if (Array.isArray(pool.state.pausedRefs) && !pool.state.pausedRefs.includes(targetRef)) pool.state.pausedRefs.push(targetRef);
+    if (!pool.state.pauseReasons) pool.state.pauseReasons = new Map();
+    pool.state.pauseReasons.set(targetRef, 'AUTH_FAILURE_PAUSED');
+    pool.state.failedUntil.set(targetRef, Infinity);
+    pushEvent(pool, targetRef, 'KEY_AUTH_PAUSED', 0);
+    if (logWarn) logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently PAUSED on auth failure (${errCode})`);
+    // Fan-out permanent pause across all pools sharing this physical ref (#408):
+    if (typeof buildRuntime === 'function') {
+      const rAll = buildRuntime();
+      const candidatePools = [
+        ...(Array.isArray(rAll.pools) ? rAll.pools : []),
+        ...(rAll.index?.allPools ? rAll.index.allPools.values() : []),
+        ...(rAll.providerToPool ? rAll.providerToPool.values() : []),
+      ];
+      if (rAll.modelPoolByProvider) {
+        for (const map of rAll.modelPoolByProvider.values()) { if (map) candidatePools.push(...map.values()); }
+      }
+      const seenPools = new Set();
+      for (const other of candidatePools) {
+        if (!other || seenPools.has(other)) continue;
+        seenPools.add(other);
+        if (other === pool) continue;
+        const hasRef = (Array.isArray(other.refs) && other.refs.includes(targetRef)) ||
+          (other.weightsMap instanceof Map && other.weightsMap.has(targetRef));
+        if (hasRef && other.state) {
+          if (!other.state.revokedRefs) other.state.revokedRefs = new Set();
+          other.state.revokedRefs.add(targetRef);
+          if (!other.state.pausedRefs) other.state.pausedRefs = new Set();
+          if (other.state.pausedRefs instanceof Set) other.state.pausedRefs.add(targetRef);
+          else if (Array.isArray(other.state.pausedRefs) && !other.state.pausedRefs.includes(targetRef)) other.state.pausedRefs.push(targetRef);
+          if (!other.state.pauseReasons) other.state.pauseReasons = new Map();
+          other.state.pauseReasons.set(targetRef, 'AUTH_FAILURE_PAUSED');
+          if (other.state.failedUntil) other.state.failedUntil.set(targetRef, Infinity);
+          pushEvent(other, targetRef, 'KEY_AUTH_PAUSED', 0);
+        }
+      }
+    }
+  } else {
+    pool.state.authFailCounts.delete(targetRef);
+    // Soft/transient errors (TIMEOUT, 5xx, socket) do not count toward quarantine (#473).
+    // recordFailure already maintains pool.state.failCounts accurately for non-soft failures.
+    if (!isQuota && !cls.soft) {
+      const _fc = pool.state.failCounts.get(targetRef) ?? 0;
+      if (_fc >= 3 && !pool.state.brokenUntil.has(targetRef)) {
+        const quarantineMs = 1800000;
+        pool.state.brokenUntil.set(targetRef, Date.now() + quarantineMs);
+        pushEvent(pool, targetRef, 'KEY_QUARANTINED', quarantineMs);
+        if (logWarn) logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} put in recoverable quarantine after ${_fc} failures`);
+      }
+    }
+  }
+}