@goodandready/dsh-key-rotation 0.8.53 → 0.8.54

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.8.54 - 2026-10-11
+
+### Added
+- **Shared Request Deadline and Attempt Budget Across Rotation & Cascade (#464)**:
+  - Added `createRequestBudget` in `lib/request-budget.js` with monotonic timer (`nowMono`) and `AbortSignal` merging.
+  - Added `maxDurationMs` and `maxAttempts` configuration options.
+  - Budget threads through rotation and cross-provider cascade recursively without resetting counters, terminating early with `BUDGET_DEADLINE_EXCEEDED` or `BUDGET_ATTEMPTS_EXCEEDED`.
+- **Key Selection Explanation & Request Attempt Trace (#461)**:
+  - Added `createSelectionTrace` in `lib/selection-trace.js` tracking candidate evaluation decisions, cooldown reasons, and dispatches.
+  - Added loopback-guarded endpoints `GET /dsh-key-rotation/trace` and `DELETE /dsh-key-rotation/trace` in `lib/ops-telemetry.js` with automatic secret redaction.
+- **Offline Rotation & Cascade Simulator (#462)**:
+  - Added `simulateRouting` engine in `lib/simulator.js` running over isolated in-memory state without HTTP requests, token charges, or credential resolution.
+  - Added `POST /dsh-key-rotation/simulate` endpoint in `lib/ops-telemetry.js` to preview failover decisions.
+- **Managed Key Draining and Graceful Replacement (#463)**:
+  - Added `lib/key-drain.js` tracking draining keys and in-flight stream concurrency.
+  - `rotate.js` excludes draining keys from new candidate dispatches while active in-flight streams finish cleanly.
+  - Added `GET / POST / DELETE /dsh-key-rotation/drain` endpoints in `lib/ops-keys.js`.
+
 ## 0.8.53 - 2026-10-09
 
 ### Fixed

package/lib/index.js

@@ -105,6 +105,8 @@ const SettingsSchema = Schema.object({
   circuitBreakerThreshold: Schema.number().default(5),
   circuitBreakerOpenMs: Schema.number().default(30000),
   circuitBreakerHalfOpenProbes: Schema.number().default(1),
+  maxDurationMs: Schema.number().default(0),
+  maxAttempts: Schema.number().default(0),
   persistenceEnabled: Schema.boolean().default(true),
   persistencePath: Schema.string().default(''),
   switchNotifyThrottleMs: Schema.number().default(60000),

package/lib/key-drain.js

@@ -0,0 +1,54 @@
+// lib/key-drain.js — Managed key draining and graceful replacement (#463).
+// Allows draining an old key ref (no new dispatches assigned) while letting
+// in-flight active streams complete cleanly before permanent removal.
+
+const drainingKeys = new Map(); // ref -> { provider, startedAt, inFlightCount }
+
+export function markKeyDraining(ref, provider = '') {
+  if (!ref) return false;
+  drainingKeys.set(ref, {
+    ref,
+    provider,
+    startedAt: Date.now(),
+  });
+  return true;
+}
+
+export function isKeyDraining(ref) {
+  if (!ref) return false;
+  return drainingKeys.has(ref);
+}
+
+export function cancelKeyDrain(ref) {
+  if (!ref) return false;
+  return drainingKeys.delete(ref);
+}
+
+export function getDrainingStatus(ref, concurrencyTracker) {
+  if (!ref || !drainingKeys.has(ref)) return null;
+  const entry = drainingKeys.get(ref);
+  const inFlight = concurrencyTracker && typeof concurrencyTracker.getInFlight === 'function'
+    ? concurrencyTracker.getInFlight(ref)
+    : 0;
+
+  return {
+    ref,
+    provider: entry.provider,
+    startedAt: entry.startedAt,
+    elapsedMs: Date.now() - entry.startedAt,
+    inFlight,
+    canComplete: inFlight === 0,
+  };
+}
+
+export function getAllDrainingKeys(concurrencyTracker) {
+  const result = [];
+  for (const ref of drainingKeys.keys()) {
+    result.push(getDrainingStatus(ref, concurrencyTracker));
+  }
+  return result;
+}
+
+export function clearAllDraining() {
+  drainingKeys.clear();
+}

package/lib/ops-keys.js

@@ -19,13 +19,64 @@ import {
   KEY_PATH,
   RESET_PATH,
   IMPORT_PATH,
+  DRAIN_PATH,
 } from './ops-paths.js';
+import { markKeyDraining, isKeyDraining, cancelKeyDrain, getDrainingStatus, getAllDrainingKeys } from './key-drain.js';
 
 /**
  * @param {object} ctx cordis context
  * @param {object} deps live dependencies from apply()
  */
 export function registerKeyRoutes(ctx, deps) {
+
+ctx.effect(() => ctx.webServer.register({
+  kind: 'exact',
+  path: DRAIN_PATH,
+  handler: async (req, res) => {
+    if (!isTrustedBridgeRequest(req)) {
+      json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: drain is local-only' } });
+      return;
+    }
+    const concurrency = deps.concurrencyTracker;
+
+    if (req.method === 'GET') {
+      const url = new URL(req.url ?? DRAIN_PATH, 'http://localhost');
+      const ref = url.searchParams.get('ref');
+      if (ref) {
+        const status = getDrainingStatus(ref, concurrency);
+        if (!status) { json(res, 404, { error: { code: 'not-found', message: `Key ${ref} is not currently draining` } }); return; }
+        json(res, 200, { ok: true, draining: status });
+        return;
+      }
+      json(res, 200, { ok: true, draining: getAllDrainingKeys(concurrency) });
+      return;
+    }
+
+    if (req.method === 'POST') {
+      let body;
+      try { body = await readJson(req); } catch (e) {
+        json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } });
+        return;
+      }
+      const ref = body?.ref;
+      if (!ref) { json(res, 400, { error: { code: 'bad-request', message: 'Missing ref' } }); return; }
+      markKeyDraining(ref, body?.provider || '');
+      json(res, 200, { ok: true, status: getDrainingStatus(ref, concurrency) });
+      return;
+    }
+
+    if (req.method === 'DELETE') {
+      const url = new URL(req.url ?? DRAIN_PATH, 'http://localhost');
+      const ref = url.searchParams.get('ref');
+      if (!ref) { json(res, 400, { error: { code: 'bad-request', message: 'Missing ref query parameter' } }); return; }
+      const cancelled = cancelKeyDrain(ref);
+      json(res, 200, { ok: true, cancelled });
+      return;
+    }
+
+    json(res, 405, { error: { code: 'method', message: 'GET, POST, or DELETE only' } });
+  },
+}), 'dsh-key-rotation: drain route');
   const {
     lastTestCache,
     poolState,

package/lib/ops-paths.js

@@ -11,3 +11,6 @@ export const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
 export const WEBHOOK_ACTION_PATH = '/dsh-key-rotation/webhook-action';
 export const METRICS_PATH = '/dsh-key-rotation/metrics';
 export const DIAG_PATH = '/dsh-key-rotation/diag';
+export const TRACE_PATH = '/dsh-key-rotation/trace';
+export const SIMULATE_PATH = '/dsh-key-rotation/simulate';
+export const DRAIN_PATH = '/dsh-key-rotation/drain';

package/lib/ops-telemetry.js

@@ -14,7 +14,11 @@ import {
   USAGE_PATH,
   SNAPSHOT_PATH,
   DIAG_PATH,
+  TRACE_PATH,
+  SIMULATE_PATH,
 } from './ops-paths.js';
+import { simulateRouting } from './simulator.js';
+import { getSelectionTraces, clearSelectionTraces } from './selection-trace.js';
 import { findSecrets } from './keycheck.js';
 import { mutateSettings, sectionOps } from './settings-write.js';
 
@@ -130,6 +134,73 @@ ctx.effect(() => ctx.webServer.register({
   },
 }), 'dsh-key-rotation: snapshot route');
 
+
+ctx.effect(() => ctx.webServer.register({
+  kind: 'exact',
+  path: TRACE_PATH,
+  handler: (req, res) => {
+    if (req.method !== 'GET' && req.method !== 'DELETE') {
+      json(res, 405, { error: { code: 'method', message: 'GET or DELETE only' } });
+      return;
+    }
+    if (!isTrustedBridgeRequest(req)) {
+      json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: trace is local-only' } });
+      return;
+    }
+    if (req.method === 'DELETE') {
+      clearSelectionTraces();
+      json(res, 200, { ok: true, message: 'trace history cleared' });
+      return;
+    }
+    const url = new URL(req.url ?? TRACE_PATH, 'http://localhost');
+    const limit = Math.min(100, Math.max(1, Number(url.searchParams.get('limit')) || 50));
+    json(res, 200, { ok: true, traces: getSelectionTraces(limit) });
+  },
+}), 'dsh-key-rotation: trace route');
+
+
+ctx.effect(() => ctx.webServer.register({
+  kind: 'exact',
+  path: SIMULATE_PATH,
+  handler: async (req, res) => {
+    if (req.method !== 'POST') {
+      json(res, 405, { error: { code: 'method', message: 'POST only' } });
+      return;
+    }
+    if (!isTrustedBridgeRequest(req)) {
+      json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: simulate is local-only' } });
+      return;
+    }
+    let body;
+    try { body = await readJson(req); } catch (e) {
+      json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } });
+      return;
+    }
+
+    const runtime = buildRuntime();
+    const provider = body?.provider || (runtime.pools?.[0]?.base ?? '');
+    const pool = runtime.providerToPool?.[provider] || runtime.pools?.[0];
+
+    if (!pool) {
+      json(res, 404, { error: { code: 'not-found', message: `Provider '${provider}' not found` } });
+      return;
+    }
+
+    try {
+      const result = simulateRouting({
+        pool,
+        runtime,
+        events: Array.isArray(body?.events) ? body.events : [],
+        model: body?.model || '',
+        maxAttempts: Number(body?.maxAttempts) || 0,
+      });
+      json(res, 200, { ok: true, simulation: result });
+    } catch (e) {
+      json(res, 500, { error: { code: 'sim-error', message: String(e?.message ?? e) } });
+    }
+  },
+}), 'dsh-key-rotation: simulate route');
+
 ctx.effect(() => ctx.webServer.register({
   kind: 'exact',
   path: DIAG_PATH,

package/lib/request-budget.js

@@ -0,0 +1,112 @@
+// lib/request-budget.js — per-request deadline & attempt budget management (#464).
+// Monotonic clock timing via nowMono(), abort signal chaining, and dispatch attempt tracking.
+import { nowMono } from './clock.js';
+
+export const BUDGET_EXHAUSTED_TIMEOUT = 'BUDGET_DEADLINE_EXCEEDED';
+export const BUDGET_EXHAUSTED_ATTEMPTS = 'BUDGET_ATTEMPTS_EXCEEDED';
+
+/**
+ * Creates or normalizes a RequestBudget for rotation and cascade.
+ * @param {object} [options]
+ * @param {number} [options.maxDurationMs] - Max total wall duration in ms (0 = unlimited).
+ * @param {number} [options.maxAttempts] - Max total upstream dispatch attempts (0 = unlimited).
+ * @param {AbortSignal} [options.signal] - Caller-provided abort signal.
+ * @returns {object}
+ */
+export function createRequestBudget(options = {}) {
+  const maxDurationMs = Math.max(0, Number(options.maxDurationMs) || 0);
+  const maxAttempts = Math.max(0, Number(options.maxAttempts) || 0);
+  const startTime = nowMono();
+  const deadline = maxDurationMs > 0 ? startTime + maxDurationMs : Infinity;
+  let attempts = 0;
+
+  const controller = new AbortController();
+  let timerId = null;
+
+  // Chain caller signal if provided
+  if (options.signal) {
+    if (options.signal.aborted) {
+      controller.abort(options.signal.reason);
+    } else {
+      const onCallerAbort = () => {
+        controller.abort(options.signal.reason);
+        cleanup();
+      };
+      options.signal.addEventListener('abort', onCallerAbort, { once: true });
+    }
+  }
+
+  // Setup deadline timeout if finite
+  if (Number.isFinite(deadline)) {
+    const remaining = Math.max(0, deadline - startTime);
+    timerId = setTimeout(() => {
+      const err = new Error(`Request exceeded maximum deadline of ${maxDurationMs}ms`);
+      err.code = BUDGET_EXHAUSTED_TIMEOUT;
+      controller.abort(err);
+    }, remaining);
+    if (timerId.unref) timerId.unref();
+  }
+
+  function cleanup() {
+    if (timerId) {
+      clearTimeout(timerId);
+      timerId = null;
+    }
+  }
+
+  return {
+    startTime,
+    maxDurationMs,
+    maxAttempts,
+    deadline,
+    signal: controller.signal,
+
+    /** Check if deadline is expired */
+    isExpired() {
+      if (controller.signal.aborted) return true;
+      if (Number.isFinite(deadline) && nowMono() >= deadline) {
+        return true;
+      }
+      return false;
+    },
+
+    /** Remaining time in ms before deadline (or Infinity if none) */
+    remainingMs() {
+      if (controller.signal.aborted) return 0;
+      if (!Number.isFinite(deadline)) return Infinity;
+      return Math.max(0, Math.round(deadline - nowMono()));
+    },
+
+    /** Current count of attempted dispatches */
+    getAttempts() {
+      return attempts;
+    },
+
+    /** Check whether another attempt is permitted */
+    canAttempt() {
+      if (this.isExpired()) return false;
+      if (maxAttempts > 0 && attempts >= maxAttempts) return false;
+      return true;
+    },
+
+    /** Record a real upstream dispatch attempt */
+    recordAttempt() {
+      if (!this.canAttempt()) {
+        const err = new Error(
+          this.isExpired()
+            ? `Request exceeded maximum duration (${maxDurationMs}ms)`
+            : `Request exceeded maximum attempts (${maxAttempts})`
+        );
+        err.code = this.isExpired() ? BUDGET_EXHAUSTED_TIMEOUT : BUDGET_EXHAUSTED_ATTEMPTS;
+        throw err;
+      }
+      attempts += 1;
+      return attempts;
+    },
+
+    /** Cleanup timers / listeners */
+    dispose() {
+      cleanup();
+    }
+  };
+}

package/lib/rotate.js

@@ -16,6 +16,9 @@ import {
   sortAttemptList,
 } from './pool.js';
 import { pickCascadeFallback, CASCADE_MAX_DEPTH } from './cascade.js';
+import { createRequestBudget, BUDGET_EXHAUSTED_TIMEOUT, BUDGET_EXHAUSTED_ATTEMPTS } from './request-budget.js';
+import { createSelectionTrace } from './selection-trace.js';
+import { isKeyDraining } from './key-drain.js';
 import { nextQuotaReset, poolResetAt } from './quota-window.js';
 import { tpmRecord } from './bucket.js';
 import {
@@ -79,6 +82,21 @@ export function createRotate(deps) {
   function rotate(options, pool) {
     return (async function* () {
       const runtime0 = buildRuntime();
+      // #464: Per-request budget (deadline and max attempts) shared across rotation and cascade
+      const existingBudget = options.__requestBudget;
+      const budget = existingBudget || createRequestBudget({
+        maxDurationMs: options.maxDurationMs ?? pool.maxDurationMs ?? runtime0.maxDurationMs ?? 0,
+        maxAttempts: options.maxAttempts ?? pool.maxAttempts ?? runtime0.maxAttempts ?? 0,
+        signal: options.signal,
+      });
+      const isBudgetOwner = !existingBudget;
+
+      // #461: Track key selection and failover explanation
+      const trace = options.__selectionTrace || createSelectionTrace(options.requestId, {
+        provider: options.provider,
+        model: options.model,
+      });
+      const isTraceOwner = !options.__selectionTrace;
       const { switchCodes, cooldownMs, maxCooldownMs, switchNotify, rateLimitThreshold } = runtime0;
       let lastFailure = null;
       const breaker = runtime0.circuitBreakerEnabled === false ? null : circuitBreaker;
@@ -88,7 +106,7 @@ export function createRotate(deps) {
       const wallNow = Date.now();
       const quotaManaged = hasModelQuotaConfig(pool);
       let attemptList = (pool.weightedRefs ?? pool.refs).slice();
-      attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r) && !isKeyBroken(pool, r, wallNow));
+      attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r) && !isKeyBroken(pool, r, wallNow) && !isKeyDraining(r));
       if (pool.pinnedRef && attemptList.includes(pool.pinnedRef)) {
         const pIdx = attemptList.indexOf(pool.pinnedRef);
         if (pIdx > 0) {
@@ -119,6 +137,8 @@ export function createRotate(deps) {
                 model: fb.model || options.model,
                 __isCascade: true,
                 __cascadeDepth: currentDepth + 1,
+                __requestBudget: budget,
+                __selectionTrace: trace,
               }, fb.pool);
               for await (const chunk of innerCascade) {
                 yield chunk;
@@ -179,6 +199,7 @@ export function createRotate(deps) {
         }
 
         activePermit?.failure();
+        trace.addStep(targetRef, 'FAILED', errCode ?? 'UNKNOWN', errMsg);
         pushEvent(pool, targetRef, errCode ?? 'UNKNOWN', _b);
         if (typeof schedulePersist === 'function') {
           try { schedulePersist(); } catch (_) { /* best-effort persistence */ }
@@ -235,6 +256,19 @@ export function createRotate(deps) {
       };
 
       for (let attempt = 0; attempt < attemptList.length; attempt++) {
+        // #464: Check deadline & attempt budget before dispatch
+        if (!budget.canAttempt()) {
+          const isTimeout = budget.isExpired();
+          const code = isTimeout ? BUDGET_EXHAUSTED_TIMEOUT : BUDGET_EXHAUSTED_ATTEMPTS;
+          const msg = isTimeout
+            ? `[dsh-key-rotation] request deadline exceeded (${budget.maxDurationMs}ms)`
+            : `[dsh-key-rotation] request attempt budget exceeded (${budget.maxAttempts} attempts)`;
+          logWarn(msg);
+          yield finishError(code, msg);
+          if (isBudgetOwner) budget.dispose();
+          return;
+        }
+
         activePermit = breaker ? breaker.acquire(options.provider) : null;
         if (breaker && !activePermit) {
           logWarn(`[dsh-key-rotation] ${options.provider}: circuit open — trying cascade fallback`);
@@ -254,6 +288,8 @@ export function createRotate(deps) {
                 model: fb.model || options.model,
                 __isCascade: true,
                 __cascadeDepth: currentDepth + 1,
+                __requestBudget: budget,
+                __selectionTrace: trace,
               }, fb.pool);
               for await (const chunk of innerCascade) {
                 yield chunk;
@@ -269,13 +305,18 @@ export function createRotate(deps) {
           let switching = false;
           let inner;
           try {
+            // #464 & #461: Record attempt & trace dispatch
+            budget.recordAttempt();
+            const curCandidate = reqStore.pickedRef ?? pool.state.lastUsed ?? attemptList[attempt];
+            trace.addStep(curCandidate, 'DISPATCHED', 'ATTEMPT', `Attempt ${budget.getAttempts()}`);
+
             // mark the internal dispatch so the interceptor does not re-rotate
             inner = dispatchStorage.run(reqStore, () => {
               const llm = ctx.get('llm');
               if (!llm || typeof llm.stream !== 'function') {
                 throw new Error('dsh-key-rotation: llm service unavailable');
               }
-              return llm.stream({ ...options, [MARKER]: true });
+              return llm.stream({ ...options, [MARKER]: true, signal: budget.signal });
             });
           } catch (e) {
             if (e?.localQuota || e?.localExhausted || e?.code === LOCAL_QUOTA_CODE || e?.code === LOCAL_POOL_EXHAUSTED_CODE) {
@@ -430,6 +471,7 @@ export function createRotate(deps) {
                   }
                   recordLatency(pool, reqStore);
                   activePermit?.success();
+                  if (isTraceOwner) trace.finish('SUCCESS', activeRef);
                   yield chunk;
                   return;
                 }
@@ -468,6 +510,9 @@ export function createRotate(deps) {
               if (activeRef) recordSuccess(pool, activeRef, now());
               recordLatency(pool, reqStore);
               activePermit?.success();
+              if (isTraceOwner) trace.finish('SUCCESS', activeRef);
+            } else if (isTraceOwner) {
+              trace.finish('COMPLETED', reqStore.pickedRef ?? _pickedRef ?? pool.state.lastUsed);
             }
             return; // clean end — served
         } finally {
@@ -481,6 +526,13 @@ export function createRotate(deps) {
         }
       }
 
+      if (isBudgetOwner) {
+        budget.dispose();
+      }
+      if (isTraceOwner) {
+        trace.finish('TERMINATED', reqStore.pickedRef ?? pool.state.lastUsed);
+      }
+
       // pool exhausted — all keys cooling or missing
       pool.state.lastExhaustionAt = now();
       pool.state.exhaustionCount = (pool.state.exhaustionCount ?? 0) + 1;

package/lib/selection-trace.js

@@ -0,0 +1,80 @@
+// lib/selection-trace.js — Request-scoped key selection explanation and attempt history (#461).
+// Provides auditable reasoning for key skips, selection, failovers, and cascade with strict secret redaction.
+
+import { keyTail } from './pool.js';
+
+const TRACE_BUFFER_LIMIT = 100;
+const traces = [];
+
+/**
+ * Redacts any sensitive tokens from string / object representation.
+ */
+export function redactSecret(val) {
+  if (typeof val !== 'string') return val;
+  // Key refs (env names, vault refs) or tails (sk-...1234) are safe.
+  // Full raw tokens (sk-..., key-..., etc.) are masked.
+  if (/^(?:sk-|key-|Bearer\s+|ghp_)[a-zA-Z0-9_\-\.]{12,}$/i.test(val)) {
+    return "..." + keyTail(val);
+  }
+  return val;
+}
+
+export function createSelectionTrace(requestId, options = {}) {
+  const reqId = requestId || `req-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
+  const startTime = Date.now();
+  const events = [];
+
+  return {
+    requestId: reqId,
+    provider: options.provider || '',
+    model: options.model || '',
+    startTime,
+
+    /**
+     * Record a candidate evaluation (selected, skipped, cooling, quota, etc.)
+     * @param {string} ref - Key identifier
+     * @param {string} decision - 'SELECTED' | 'SKIPPED' | 'DISPATCHED' | 'FAILED' | 'CASCADE'
+     * @param {string} reasonCode - Machine code e.g. 'COOLDOWN' | 'QUOTA' | 'PINNED_PRIORITY'
+     * @param {string} [details] - User-readable explanation
+     */
+    addStep(ref, decision, reasonCode, details = '') {
+      events.push({
+        at: Date.now(),
+        ref: redactSecret(ref),
+        decision,
+        reasonCode,
+        details: String(details).slice(0, 300),
+      });
+    },
+
+    /**
+     * Complete the trace and store into bounded history
+     */
+    finish(outcome, finalRef = null) {
+      const summary = {
+        requestId: reqId,
+        provider: options.provider || '',
+        model: options.model || '',
+        startTime,
+        durationMs: Date.now() - startTime,
+        outcome, // 'SUCCESS' | 'EXHAUSTED' | 'TIMEOUT' | 'CASCADE' | 'ERROR'
+        selectedRef: finalRef ? redactSecret(finalRef) : null,
+        steps: events,
+      };
+
+      if (traces.length >= TRACE_BUFFER_LIMIT) {
+        traces.shift();
+      }
+      traces.push(summary);
+      return summary;
+    }
+  };
+}
+
+export function getSelectionTraces(limit = 50) {
+  return traces.slice(-Math.max(1, Math.min(100, limit)));
+}
+
+export function clearSelectionTraces() {
+  traces.length = 0;
+}

package/lib/simulator.js

@@ -0,0 +1,156 @@
+// lib/simulator.js — Offline rotation and cascade simulator (#462).
+// Simulates key selection, failovers, cooldowns, and cross-provider cascade
+// without network requests, credential resolution, token expenditure, or modifying production state.
+
+import {
+  isKeyPaused,
+  isKeyRevoked,
+  isKeyBroken,
+  sortAttemptList,
+} from './pool.js';
+import { isModelQuotaAvailable, hasModelQuotaConfig } from './model-quota.js';
+import { pickCascadeFallback, CASCADE_MAX_DEPTH } from './cascade.js';
+import { createSelectionTrace } from './selection-trace.js';
+
+/**
+ * Runs an offline simulation of rotation and cascade.
+ * @param {object} params
+ * @param {object} params.pool - Simulated pool definition
+ * @param {object} [params.runtime] - Simulated runtime with providerToPool, cascade, circuitBreaker
+ * @param {Array<object>} [params.events] - Simulated step events
+ * @param {number} [params.nowMs] - Simulated current timestamp
+ * @param {string} [params.model] - Optional model identifier
+ * @param {number} [params.maxAttempts] - Max allowed attempts (from #464)
+ * @returns {object} Simulation trace and result
+ */
+export function simulateRouting(params = {}) {
+  const {
+    pool,
+    runtime = {},
+    events = [],
+    nowMs = Date.now(),
+    model = '',
+    maxAttempts = 0,
+  } = params;
+
+  if (!pool || !Array.isArray(pool.refs)) {
+    throw new Error('simulateRouting: valid pool with refs is required');
+  }
+
+  // Clone pool state into isolated mock state so real runtime is never touched
+  const mockFailedUntil = new Map(pool.state?.failedUntil || []);
+  const mockFailCounts = new Map(pool.state?.failCounts || []);
+  const mockRevokedRefs = new Set(pool.state?.revokedRefs || []);
+
+  const simPool = {
+    ...pool,
+    state: {
+      failedUntil: mockFailedUntil,
+      failCounts: mockFailCounts,
+      revokedRefs: mockRevokedRefs,
+      brokenUntil: new Map(),
+    },
+  };
+
+  const trace = createSelectionTrace(`sim-${nowMs}`, {
+    provider: pool.provider || pool.base || 'sim-provider',
+    model,
+  });
+
+  const quotaManaged = hasModelQuotaConfig(simPool);
+  let attemptList = (simPool.weightedRefs ?? simPool.refs).slice();
+
+  // Filter paused/revoked/broken
+  attemptList = attemptList.filter((r) => {
+    if (isKeyPaused(simPool, r)) {
+      trace.addStep(r, 'SKIPPED', 'PAUSED', 'Key is marked paused');
+      return false;
+    }
+    if (isKeyRevoked(simPool, r)) {
+      trace.addStep(r, 'SKIPPED', 'REVOKED', 'Key is revoked');
+      return false;
+    }
+    if (isKeyBroken(simPool, r, nowMs)) {
+      trace.addStep(r, 'SKIPPED', 'BROKEN', 'Key is in broken quarantine');
+      return false;
+    }
+    const fu = simPool.state.failedUntil.get(r) ?? 0;
+    if (fu > nowMs) {
+      trace.addStep(r, 'SKIPPED', 'COOLDOWN', `In cooldown for ${Math.round((fu - nowMs) / 1000)}s`);
+      return false;
+    }
+    return true;
+  });
+
+  // Prioritize pinnedRef
+  if (simPool.pinnedRef && attemptList.includes(simPool.pinnedRef)) {
+    const pIdx = attemptList.indexOf(simPool.pinnedRef);
+    if (pIdx > 0) {
+      attemptList.splice(pIdx, 1);
+      attemptList.unshift(simPool.pinnedRef);
+      trace.addStep(simPool.pinnedRef, 'PRIORITIZED', 'PINNED', 'Key is pinned');
+    }
+  }
+
+  // Model quota filter
+  if (quotaManaged) {
+    attemptList = attemptList.filter((ref) => {
+      const ok = isModelQuotaAvailable(simPool, ref, nowMs);
+      if (!ok) {
+        trace.addStep(ref, 'SKIPPED', 'QUOTA_EXHAUSTED', 'Local model token quota reached');
+      }
+      return ok;
+    });
+  }
+
+  let attemptCount = 0;
+  let finalOutcome = 'EXHAUSTED';
+  let finalRef = null;
+
+  for (let i = 0; i < attemptList.length; i++) {
+    const candidate = attemptList[i];
+    if (maxAttempts > 0 && attemptCount >= maxAttempts) {
+      trace.addStep(candidate, 'BLOCKED', 'BUDGET_ATTEMPTS_EXCEEDED', `Max attempts (${maxAttempts}) reached`);
+      finalOutcome = 'BUDGET_ATTEMPTS_EXCEEDED';
+      break;
+    }
+
+    attemptCount++;
+    trace.addStep(candidate, 'DISPATCHED', 'ATTEMPT', `Attempt ${attemptCount}`);
+
+    // Check if user specified a synthetic event/error for this candidate
+    const ev = events.find((e) => e.ref === candidate || e.attempt === attemptCount);
+    if (ev && ev.error) {
+      const errCode = ev.error.code || 'SIM_ERROR';
+      const errMsg = ev.error.message || 'Simulated failure';
+      trace.addStep(candidate, 'FAILED', errCode, errMsg);
+      // Apply simulated cooldown
+      simPool.state.failedUntil.set(candidate, nowMs + (ev.cooldownMs || 60000));
+      continue;
+    }
+
+    // Success simulated!
+    trace.addStep(candidate, 'SUCCESS', 'OK', 'Candidate served response');
+    finalOutcome = 'SUCCESS';
+    finalRef = candidate;
+    break;
+  }
+
+  // Cascade check if exhausted
+  if (finalOutcome === 'EXHAUSTED' && runtime.cascade && Array.isArray(runtime.cascade) && runtime.cascade.length > 0) {
+    const fb = pickCascadeFallback(simPool.provider || simPool.base, runtime, runtime.providerToPool, model, runtime.modelPoolByProvider);
+    if (fb && fb.pool) {
+      trace.addStep(fb.provider, 'CASCADE', 'TRIGGERED', `Cascading to fallback provider ${fb.provider}`);
+      finalOutcome = 'CASCADE_TRIGGERED';
+      finalRef = fb.provider;
+    }
+  }
+
+  const result = trace.finish(finalOutcome, finalRef);
+  return {
+    outcome: finalOutcome,
+    selectedRef: finalRef,
+    attempts: attemptCount,
+    trace: result,
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.53",
+  "version": "0.8.54",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [