@goodandready/dsh-key-rotation 0.7.39 → 0.8.0

package/lib/index.js

@@ -1,37 +1,11 @@
-// ─────────────────────────────────────────────────────────────────────────────
-//  dsh-key-rotation — per-provider API key rotation for DeepSeek Harness.
+// dsh-key-rotation — per-provider API key rotation for DeepSeek Harness.
 //
-//  Transparent key rotation, Hermes-style: every configured provider has a KEY
-//  POOL (env refs). The plugin patches `ctx.credentials.resolve` so a pool ref
-//  resolves to the next available key (round-robin, skipping keys in cooldown),
-//  and intercepts `llm/stream` to retry a request on the next key when the
-//  current one fails with a switchable error (QUOTA, RATE_LIMIT, ...) before
-//  any content chunk.
-//
-//  The PROVIDER IDENTITY NEVER CHANGES: requests always go out with the
-//  provider the user selected, only the resolved API key differs. This keeps
-//  pi-ai's replay state consistent across multi-call turns and multi-turn
-//  sessions (the earlier clone-provider approach broke it with
-//  INVALID_REPLAY_STATE).
-//
-//  Config is a KEY POOL PER PROVIDER: you list a real provider id and the env
-//  names of its API keys (e.g. <PROVIDER>_API_KEY, <PROVIDER>_API_KEY_2, ...).
-//  When a key's limit is exhausted, the request retries on the next key in the
-//  list; exhausted keys stay in cooldown for cooldownMs. Clone provider routes
-//  (named `<base>-2`, `<base>-3`, ...) are no longer used for rotation but
-//  remain registered, so selecting them also rotates (their apiKeyEnv ref
-//  belongs to the same pool).
-//
-//  The Settings section ("Key Rotation") edits the provider key pools as a
-//  simple list: pick a provider from the dropdown of every provider registered
-//  with ctx.llm (clone routes are hidden from the dropdown), then add/remove key
-//  env names. Plus cooldown and switch codes.
-//
-//  Config (all optional, sane defaults):
-//    switchCodes: string[]   failure codes eligible to switch
-//    cooldownMs: number      key cooldown after a switchable failure
-//    providers:  array       [{ provider, keys: [envName, ...] }]
-// ─────────────────────────────────────────────────────────────────────────────
+// Transparent key pool rotation: credentials.resolve patch + llm/stream
+// interceptor. Provider identity never changes (keeps pi-ai replay state).
+// See README.md and docs/design/DESIGN.md for the full contract.
+// Config (all optional): switchCodes, cooldownMs, providers[{provider,keys}],
+// verboseLogging (default false) gates per-request rotation logs.
+
 import { AsyncLocalStorage } from 'node:async_hooks';
 import Schema from '@deepseek-ai/schemastery';
 import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter, computeHealthScore, extractRateLimit, isRateLimited, selectPool, isSwitchableError, formatExhaustionMessage, expiringSoon, shouldNotifyDaily, costForDay, costForWeek, budgetVerdict } from './pool.js';
@@ -66,6 +40,15 @@ import { usageRows, usageCsv, compactUsage } from './usage-report.js';
 
 const dispatchStorage = new AsyncLocalStorage();
 import { findSecrets, looksLikeApiSecret } from './keycheck.js';
+import { json, readJson, descriptorOf, viewOf, writeSection, providerCatalog, handleConfigBridge, NS as BRIDGE_NS } from './http-bridge.js';
+import { createRotate } from './rotate.js';
+import { nowMono } from './clock.js';
+import { CircuitBreaker } from './circuit-breaker.js';
+import { NotifyQueue } from './notify-queue.js';
+import { BoundedMap } from './bounded-map.js';
+import { classifyFailure } from './error-taxonomy.js';
+import { safeParseJson } from './atomic-io.js';
+import { registerOpsRoutes } from './routes-ops.js';
 
 /** The llm-pi-ai namespace whose provider profiles map providers to pools. */
 const PIAI_NS = 'llm-pi-ai';
@@ -90,7 +73,11 @@ export function notifySwitch(runtime, pool, info, hooks = { webhookSender, now:
   const now = hooks.now();
   if (now - last < throttle) return;
   switchNotifiedAt.set(info.provider, now);
-  hooks.webhookSender.send(runtime.notifyWebhook, {
+  // #263: non-blocking — enqueue never awaits webhook I/O
+  const send = hooks.notifyQueue
+    ? (url, payload) => { hooks.notifyQueue.enqueue(url, payload); return { sent: true, queued: true }; }
+    : (url, payload) => hooks.webhookSender.send(url, payload);
+  send(runtime.notifyWebhook, {
     title: `Key switched: ${info.provider}`,
     text: `${info.from} failed (${info.code}) - next key in pool`,
     provider: info.provider,
@@ -121,11 +108,18 @@ let lastTestCacheRunnerCtx = null;
 const lastTestCache = new LastTestCache();
 const latencyHistogram = new LatencyHistogram();
 const quotaStore = new QuotaStore();
+// #260/#263 module-scope infra (process-wide, reset on reload via buildRuntime)
+let moduleBreaker = null;
+let moduleNotifyQueue = null;
 // Global config accessor safe against early initialization
 let getConfig = () => null;
 let getRuntime = () => null;
 let sandboxRunner = null;
 const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
+// #263: queue webhook I/O off the hot path
+moduleNotifyQueue = new NotifyQueue({ send: (url, payload) => webhookSender.send(url, payload) });
+// #260: process-wide breaker; thresholds re-read from runtime when dispatching
+moduleBreaker = new CircuitBreaker({ threshold: 5, openMs: 30000, halfOpenProbes: 1, now: nowMono });
 const concurrencyTracker = new ConcurrencyTracker();
 function ensureSandboxRunner(ctx) {
   if (sandboxRunner) return sandboxRunner;
@@ -204,6 +198,12 @@ export const Config = Schema.object({
   webhookActionToken: Schema.string().role('secret').default(''),
   expiryWarnDays: Schema.number().default(7),
   switchNotify: Schema.boolean().default(false),
+  verboseLogging: Schema.boolean().default(false),
+  // #260 circuit breaker
+  circuitBreakerEnabled: Schema.boolean().default(true),
+  circuitBreakerThreshold: Schema.number().default(5),
+  circuitBreakerOpenMs: Schema.number().default(30000),
+  circuitBreakerHalfOpenProbes: Schema.number().default(1),
   switchNotifyThrottleMs: Schema.number().default(60000),
   warnBelowHealthy: Schema.number().default(0),
   latencySloMs: Schema.number().default(0),
@@ -228,163 +228,6 @@ export const Config = Schema.object({
 // ── config bridge (GET/PUT/DELETE on CONFIG_PATH), mirroring llm-fallback ──
 
 
-function json(res, status, obj) {
-  res.writeHead(status, { 'content-type': 'application/json' });
-  res.end(JSON.stringify(obj));
-}
-
-function readJson(request) {
-  return new Promise((resolve, reject) => {
-    let raw = '';
-    request.on('data', (c) => { raw += c; });
-    request.on('end', () => {
-      try {
-        resolve(JSON.parse(raw || '{}'));
-      } catch (e) {
-        reject(e);
-      }
-    });
-    request.on('error', reject);
-  });
-}
-
-function descriptorOf(ctx, ns) {
-  const settings = ctx.get('settings');
-  if (settings === void 0) return void 0;
-  return settings.describe({ redactSecrets: true }).find((candidate) => candidate.ns === ns);
-}
-
-function viewOf(descriptor, settings) {
-  return {
-    available: true,
-    writable: settings.writable,
-    hasDocument: settings.hasDocument,
-    value: descriptor.value,
-    ...descriptor.base === void 0 ? {} : { base: descriptor.base },
-    ...descriptor.user === void 0 || Object.keys(descriptor.user).length === 0 ? {} : { user: descriptor.user },
-    revision: descriptor.revision,
-  };
-}
-
-async function writeSection(ctx, ns, section, expectedRevision, res) {
-  const settings = ctx.get('settings');
-  if (settings === void 0) {
-    json(res, 503, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: no settings provider is mounted' } });
-    return;
-  }
-  try {
-    await settings.replace(ns, section, expectedRevision);
-  } catch (error) {
-    if (error?.code === 'SETTINGS_CONFLICT') {
-      json(res, 409, { error: { code: 'settings-conflict', message: `dsh-key-rotation: changed elsewhere (expected revision ${String(error.expected)}, current ${String(error.actual)}); reload and retry` } });
-      return;
-    }
-    json(res, 400, { error: { code: 'settings-rejected', message: error instanceof Error ? error.message : String(error) } });
-    return;
-  }
-  const descriptor = descriptorOf(ctx, ns);
-  if (descriptor === void 0) {
-    json(res, 500, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: namespace vanished after write' } });
-    return;
-  }
-  json(res, 200, viewOf(descriptor, { writable: settings.writable, hasDocument: settings.documentPath !== void 0 }));
-}
-
-/** Provider catalog for the GUI dropdown, minus clone routes of configured chains. */
-function providerCatalog(ctx, cloneIds) {
-  const seen = new Set();
-  const out = [];
-  for (const info of ctx.llm.listProviders()) {
-    if (seen.has(info.id) || cloneIds.has(info.id)) continue;
-    seen.add(info.id);
-    out.push({ id: info.id, name: info.name ?? info.id });
-  }
-  return out;
-}
-
-async function handleConfigBridge(ctx, request, res, getCloneIds) {
-  if (!isTrustedBridgeRequest(request)) {
-    res.writeHead(403);
-    res.end();
-    return;
-  }
-  const method = request.method ?? 'GET';
-  if (method === 'GET') {
-    const settings = ctx.get('settings');
-    const descriptor = descriptorOf(ctx, NS);
-    const body = {
-      providers: providerCatalog(ctx, getCloneIds()),
-    };
-    if (descriptor === void 0) {
-      json(res, 200, {
-        ...body,
-        available: false,
-        writable: settings?.writable ?? false,
-        hasDocument: settings?.documentPath !== void 0,
-        value: void 0,
-        revision: 0,
-      });
-      return;
-    }
-    json(res, 200, {
-      ...body,
-      ...viewOf(descriptor, {
-        writable: settings?.writable ?? false,
-        hasDocument: settings?.documentPath !== void 0,
-      }),
-    });
-    return;
-  }
-  if (method === 'PUT' || method === 'DELETE') {
-    let section;
-    let expectedRevision;
-    if (method === 'PUT') {
-      let body;
-      try {
-        body = await readJson(request);
-      } catch (error) {
-        json(res, 400, { error: { code: 'settings-rejected', message: `dsh-key-rotation: invalid request body: ${error instanceof Error ? error.message : String(error)}` } });
-        return;
-      }
-      if (typeof body !== 'object' || body === null || typeof body.section !== 'object' || body.section === null || Array.isArray(body.section)) {
-        json(res, 400, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: PUT requires {"section": {...}}' } });
-        return;
-      }
-      section = body.section;
-      expectedRevision = typeof body.expectedRevision === 'number' ? body.expectedRevision : void 0;
-      // #200 leak detector: a live secret pasted into the config section is
-      // almost always a mistake (real key values belong in PUT /key). The two
-      // fields that legitimately hold tokens are masked before scanning.
-      try {
-        const masked = structuredClone(section);
-        if (masked.webhookActionToken) masked.webhookActionToken = '***';
-        // notifyWebhook legitimately carries bot tokens inside URLs
-        // (api.telegram.org/bot<token>/...) - scan it for nothing.
-        if (masked.notifyWebhook) masked.notifyWebhook = '***';
-        const findings = findSecrets(JSON.stringify(masked));
-        if (findings.length > 0) {
-          json(res, 400, {
-            error: {
-              code: 'secret-in-config',
-              message: `dsh-key-rotation: value looks like a live credential (${findings[0].type}); store key values via the key field, not the config section`,
-              findings,
-            },
-          });
-          return;
-        }
-      } catch {
-        /* scanning must never block a valid save */
-      }
-    } else {
-      section = {};
-    }
-    await writeSection(ctx, NS, section, expectedRevision, res);
-    return;
-  }
-  res.writeHead(405);
-  res.end();
-}
-
 function registerConfigBridge(ctx, getCloneIds) {
   return ctx.webServer.register({
     kind: 'exact',
@@ -622,6 +465,8 @@ export function apply(ctx, config = {}) {
           byModel: new Map(),
           usageDays: new Map(),
           quotaWindows: new Map(),
+          // #260 per-provider circuit breaker (lazy)
+          breaker: null,
           pointer: 0,
           lastUsed: undefined,
           switches: 0,
@@ -716,7 +561,30 @@ export function apply(ctx, config = {}) {
       const weekly = typeof p.costBudgetWeekly === 'number' ? p.costBudgetWeekly : 0;
       if (daily > 0 || weekly > 0) providerBudgets.set(p.provider, { costBudgetDaily: daily, costBudgetWeekly: weekly, pauseOnBudget: p.pauseOnBudget ?? false });
     }
-    cachedRuntime = { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, concurrencyLimit, cascade, quotaResetWindow, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
+    // #266: orphan clone-route GC — expected clones only for live multi-key providers
+    const expectedClones = new Set();
+    for (const p of cfg.providers ?? []) {
+      const n = (p.keys ?? []).filter((k) => typeof k === 'string' && k.length > 0).length;
+      for (let i = 1; i < n; i++) expectedClones.add(`${p.provider}-${i + 1}`);
+    }
+    // drop breaker entries for removed providers
+    if (moduleBreaker) {
+      for (const key of Object.keys(moduleBreaker.snapshot())) {
+        if (![...providerToPool.keys()].includes(key) && !expectedClones.has(key)) {
+          // keep until TTL; only reset if provider gone from config entirely
+          const still = (cfg.providers ?? []).some((p) => p.provider === key);
+          if (!still) moduleBreaker.reset(key);
+        }
+      }
+    }
+    cachedRuntime = { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, concurrencyLimit, cascade, quotaResetWindow, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, verboseLogging: cfg.verboseLogging ?? false, switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds, expectedClones,
+      circuitBreakerEnabled: cfg.circuitBreakerEnabled ?? true,
+      circuitBreakerThreshold: cfg.circuitBreakerThreshold ?? 5,
+      circuitBreakerOpenMs: cfg.circuitBreakerOpenMs ?? 30000,
+      circuitBreakerHalfOpenProbes: cfg.circuitBreakerHalfOpenProbes ?? 1,
+      breaker: moduleBreaker,
+      notifyQueue: moduleNotifyQueue,
+    };
     lastConfigRef = rawConfig;
     lastProfilesRef = currentProfiles;
     return cachedRuntime;
@@ -849,757 +717,37 @@ export function apply(ctx, config = {}) {
     } catch (_) { /* ponytail: never crash */ }
   }
 
+  // rotate() factory (#253): dependencies injected for testability
+    const rotate = createRotate({
+    ctx,
+    dispatchStorage,
+    buildRuntime,
+    pushEvent,
+    notifySwitch: (runtime, pool, info) => notifySwitch(runtime, pool, info, { webhookSender, notifyQueue: moduleNotifyQueue, now: () => Date.now() }),
+    notifyExhaustion,
+    recordLatency,
+    concurrencyTracker,
+    MARKER,
+    finishError,
+    setRotateStartMs: (v) => { _rotateStartMs = v; },
+    quotaStore,
+    circuitBreaker: moduleBreaker,
+    now: nowMono,
+  });
+
   // Retry one request on the next pool key when the current key fails with a
   // switchable error before any content chunk. The provider never changes —
   // the resolve patch hands out the next key on each dispatch.
-  function rotate(options, pool) {
-    return (async function* () {
-      const runtime0 = buildRuntime();
-      const { switchCodes, cooldownMs, maxCooldownMs, switchNotify, rateLimitThreshold } = runtime0;
-      let lastFailure = null;
-      const reqStore = { pool, pickedRef: undefined, startMs: Date.now() };
-      _rotateStartMs = reqStore.startMs;
-      let attemptList = (pool.weightedRefs ?? pool.refs).slice();
-      if (runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
-        // #193: prefer least-loaded key within limit
-        const available = attemptList.filter((r) => {
-          const fu = pool.state.failedUntil.get(r) ?? 0;
-          if (fu > Date.now()) return false;
-          const exp = pool.expiresAt ? pool.expiresAt[r] : undefined;
-          if (exp !== undefined && Date.now() >= exp) return false;
-          return true;
-        });
-        const preferred = concurrencyTracker.pickLeastLoaded(available);
-        if (preferred && attemptList[0] !== preferred) {
-          const list = attemptList.slice();
-          const i = list.indexOf(preferred);
-          if (i > 0) { list.splice(i, 1); list.unshift(preferred); }
-          attemptList = list;
-        }
-      }
-
-      const penalizeRef = (targetRef, errCode, errMsg) => {
-        if (!targetRef) return;
-        const _retry = parseRetryAfter(errMsg);
-        const _base = pool.cooldownMs ?? cooldownMs;
-        const _max = pool.maxCooldownMs ?? maxCooldownMs;
-        const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base;
-        const _b = recordFailure(pool, targetRef, Date.now(), _effBase, _max);
-        pushEvent(pool, targetRef, errCode ?? 'UNKNOWN', _b);
-        if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
-        if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
-        const _cStr = String(errCode ?? '');
-        if (_cStr === 'AUTH' || /auth/i.test(errMsg)) {
-          const _c2 = (pool.state.authFailCounts.get(targetRef) ?? 0) + 1;
-          pool.state.authFailCounts.set(targetRef, _c2);
-          if (_c2 >= 3) {
-            pool.state.brokenUntil.set(targetRef, Date.now() + 86400000 * 30);
-            pool.state.failedUntil.set(targetRef, Date.now() + 86400000 * 30);
-          }
-        } else {
-          pool.state.authFailCounts.delete(targetRef);
-        }
-      };
-
-      for (let attempt = 0; attempt < attemptList.length; attempt++) {
-        let yielded = false;
-        let switching = false;
-        let inner;
-        try {
-          // mark the internal dispatch so the interceptor does not re-rotate
-          inner = dispatchStorage.run(reqStore, () => ctx.llm.stream({ ...options, [MARKER]: true }));
-        } catch (e) {
-          const curRef = reqStore.pickedRef ?? pool.state.lastUsed;
-          penalizeRef(curRef, e?.code ?? 'TRANSPORT', String(e?.message ?? ''));
-          lastFailure = finishError(e?.code ?? 'TRANSPORT',
-            `dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
-          console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(curRef ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
-          continue;
-        }
-
-        const _pickedRef = reqStore.pickedRef ?? pool.state.lastUsed;
-        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.acquire(_pickedRef);
-        try {
-          for await (const chunk of inner) {
-            // Only actual content deltas lock the stream (no more rotation).
-            // Structural/metadata chunks (block-start/end, usage) do not.
-            if (chunk && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta')) {
-              yielded = true;
-              yield chunk;
-              continue;
-            }
-            if (chunk && chunk.type === 'finish') {
-              const kind = chunk.reason?.kind;
-              const failure = chunk.reason?.failure;
-              const code = failure?.code;
-              const message = failure?.message ?? '';
-              const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-              const switchable = !yielded && kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes);
-              const activeRef = reqStore.pickedRef ?? pool.state.lastUsed;
-              if (switchable) {
-                penalizeRef(activeRef, code ?? 'UNKNOWN', message);
-                pool.state.switches = (pool.state.switches ?? 0) + 1;
-                pool.state.lastReason = String(code ?? 'UNKNOWN');
-                pool.state.lastSwitchAt = Date.now();
-                lastFailure = chunk;
-                console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) - next key`);
-                // #216: per-switch webhook (opt-in switchNotify), deduped per provider
-                if (switchNotify && activeRef) {
-                  notifySwitch(runtime0, pool, {
-                    provider: options.provider,
-                    from: activeRef,
-                    code: String(code ?? 'UNKNOWN'),
-                    at: pool.state.lastSwitchAt,
-                  });
-                }
-                switching = true;
-                break;
-              }
-              // cost tracking if provider returns usage.cost
-              const todayIso = activeRef ? new Date().toISOString().slice(0, 10) : undefined;
-              if (chunk.usage?.cost != null && activeRef) {
-                const c = Number(chunk.usage.cost);
-                if (!isNaN(c)) {
-                  if (!pool.state.costPerKey) pool.state.costPerKey = new Map();
-                  pool.state.costPerKey.set(activeRef, (pool.state.costPerKey.get(activeRef) ?? 0) + c);
-                  // #208: cost per day per key (mirrors usageDays) for budget checks
-                  if (!pool.state.costDays) pool.state.costDays = new Map();
-                  const cMap = pool.state.costDays.get(activeRef) || new Map();
-                  cMap.set(todayIso, (cMap.get(todayIso) ?? 0) + c);
-                  pool.state.costDays.set(activeRef, cMap);
-                }
-              }
-              // Usage by day (#119)
-              if (activeRef) {
-                if (!pool.state.usageDays) pool.state.usageDays = new Map();
-                const dayMap = pool.state.usageDays.get(activeRef) || new Map();
-                dayMap.set(todayIso, (dayMap.get(todayIso) ?? 0) + 1);
-                pool.state.usageDays.set(activeRef, dayMap);
-              }
-              // Per-model request detail (#121)
-              if (activeRef && options.model) {
-                if (!pool.state.byModel) pool.state.byModel = new Map();
-                let byRef = pool.state.byModel.get(activeRef);
-                if (!byRef) { byRef = new Map(); pool.state.byModel.set(activeRef, byRef); }
-                byRef.set(options.model, (byRef.get(options.model) ?? 0) + 1);
-              }
-              // Proactive rate-limit (#115): if response headers say this key is near
-              // its quota, cool it down so the NEXT request starts on a different key.
-              // We do NOT re-run this (already successful) request — that would double-send.
-              const rate = extractRateLimit(chunk?.metadata?.headers ?? chunk?.headers);
-              if (rate && activeRef) {
-                if (isRateLimited(rate, rateLimitThreshold ?? 0.1)) {
-                  const cool = rate.reset && rate.reset > Date.now() ? (rate.reset - Date.now()) : pool.cooldownMs;
-                  recordFailure(pool, activeRef, Date.now(), cool, pool.maxCooldownMs);
-                  pushEvent(pool, activeRef, 'RATE_LIMIT', cool);
-                  console.warn(`[dsh-key-rotation] ${options.provider}: key ${activeRef} near quota (remaining ${String(rate.remaining)}/${String(rate.limit)}) — next request will rotate`);
-                }
-              }
-              // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
-              if (rate && activeRef && Number.isFinite(rate.remaining)) {
-                quotaStore.set(activeRef, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: Date.now() });
-              }
-              yield chunk;
-              recordLatency(pool, reqStore);
-              return;
-            }
-            yield chunk;
-          }
-        } catch (e) {
-          if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
-          const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-          const activeRef = _pickedRef ?? reqStore.pickedRef ?? pool.state.lastUsed;
-          if (!yielded && isSwitchableError(e, effectiveSwitchCodes)) {
-            penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e));
-            pool.state.switches = (pool.state.switches ?? 0) + 1;
-            pool.state.lastReason = String(e?.code ?? 'TRANSPORT');
-            pool.state.lastSwitchAt = Date.now();
-            lastFailure = finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
-            console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} stream threw ${String(e?.code ?? e?.message ?? e)} - failover to next key`);
-            if (switchNotify && activeRef) {
-              notifySwitch(runtime0, pool, {
-                provider: options.provider,
-                from: activeRef,
-                code: String(e?.code ?? 'TRANSPORT'),
-                at: pool.state.lastSwitchAt,
-              });
-            }
-            continue; // Failover to next key!
-          }
-          yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
-          return;
-        }
-
-        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
-        if (switching) continue; // try the next key
-        return; // clean end — served
-      }
-
-      // pool exhausted — all keys cooling or missing
-      pool.state.lastExhaustionAt = Date.now();
-      pool.state.exhaustionCount = (pool.state.exhaustionCount ?? 0) + 1;
-      console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — all ${pool.refs.length} keys cooling`);
-      const runtime = buildRuntime();
-      // notify via extracted helper (see notifyExhaustion above)
-      notifyExhaustion(runtime, pool, { provider: options.provider });
-
-      // #194: cross-provider cascade failover (guarded against infinite recursion)
-      if (!options.__isCascade && Array.isArray(runtime.cascade) && runtime.cascade.length > 0) {
-        const pools = runtime.providerToPool;
-        const fb = pickCascadeFallback(options.provider, runtime, pools);
-        if (fb && fb.pool && fb.pool !== pool) {
-          console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — cascading to ${fb.provider}`);
-          pool.state.lastReason = 'CASCADE';
-          pool.state.lastSwitchAt = Date.now();
-          // Re-dispatch on the fallback pool (depth-1 via __isCascade guard)
-          const innerCascade = rotate({ ...options, provider: fb.provider, __isCascade: true }, fb.pool);
-          for await (const chunk of innerCascade) {
-            yield chunk;
-          }
-          return;
-        }
-      }
-
-      const exhaustionMsg = formatExhaustionMessage(options.provider, pool);
-      yield lastFailure ?? finishError('QUOTA', exhaustionMsg);
-    })();
-  }
-
-  // ── status route: what the settings card cannot know on its own ──
-  //
-  // Reports, per configured provider, which key is in use, which are cooling
-  // down and until when, whether an env name resolves to a credential at all
-  // (a typo is otherwise silent), and how often rotation has fired.
-  //
-  // Key VALUES never leave the host — only the boolean fact that one exists.
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: STATUS_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'GET') {
-        json(res, 405, { error: { code: 'method', message: 'GET only' } });
-        return;
-      }
-      if (!isTrustedBridgeRequest(req)) {
-        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: status is local-only' } });
-        return;
-      }
-      const runtime = buildRuntime();
-      const { poolByRef, providerTags, providerBudgets, latencySloMs } = runtime;
-      const base = ctx.get('credentials');
-      const now = Date.now();
-      const seen = new Set();
-      const providers = [];
-      for (const pool of poolByRef.values()) {
-        if (seen.has(pool.base)) continue;
-        seen.add(pool.base);
-        try {
-        const keys = [];
-        for (const ref of pool.refs) {
-          let present = false;
-          let tail = '';
-          let source = null;
-          let writable = true;
-          try {
-            // The resolve patch is installed on this same service, so ask for
-            // the exact ref: a pool ref would otherwise round-robin to another
-            // key and report a missing name as present.
-            let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
-            present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
-            if (present) tail = keyTail(hit.value);
-            // fallback: env var bootstrapping (issue #7)
-            if (!present) {
-              const ev = envValue(ref);
-              if (ev !== undefined) { present = true; tail = keyTail(ev); source = 'env'; writable = false; }
-            }
-          } catch {
-            present = false;
-          }
-          try {
-            const described = await base?.describe?.(ref);
-            source = described?.source ?? null;
-            writable = described?.writable !== false;
-          } catch {
-            /* describe is optional — the card falls back to editable */
-          }
-          const until = pool.state.failedUntil.get(ref);
-          keys.push({
-            ref,
-            present,
-            tail,
-            source,
-            writable,
-            active: pool.state.lastUsed === ref,
-            cooldownMsLeft: until !== undefined && until > now ? until - now : 0,
-            // #210: RPM capacity snapshot (null when rpmLimit is off)
-            rpm: bucketInfo(pool.state.rpmWindows, ref, pool.rpmLimit, now),
-            // #215: effective round-robin weight of this key
-            weight: pool.weights?.[pool.refs.indexOf(ref)] ?? 1,
-            usage: pool.state.usageCounts?.get(ref) ?? 0,
-            byModel: pool.state.byModel?.get(ref) ? Object.fromEntries(pool.state.byModel.get(ref)) : {},
-            usageDays: pool.state.usageDays?.get(ref) ? Object.fromEntries(pool.state.usageDays.get(ref)) : {},
-            cost: pool.state.costPerKey?.get(ref) ?? 0,
-            lastUsedAt: pool.state.lastUsedAt?.get(ref) ?? null,
-            expiresAt: pool.expiresAt?.[ref] ?? null,
-            expired: pool.expiresAt?.[ref] !== undefined && now >= pool.expiresAt[ref],
-            broken: pool.state.brokenUntil?.has(ref) ?? false,
-          });
-        }
-        providers.push({
-          provider: pool.base,
-          keys,
-          tags: providerTags.get(pool.base) ?? [],
-          switches: pool.state.switches ?? 0,
-          lastReason: pool.state.lastReason ?? null,
-          lastSwitchAt: pool.state.lastSwitchAt ?? null,
-          lastExhaustionAt: pool.state.lastExhaustionAt ?? null,
-          exhaustionCount: pool.state.exhaustionCount ?? 0,
-          totalUsage: (() => { let s = 0; if (pool.state.usageCounts) for (const v of pool.state.usageCounts.values()) s += v; return s; })(),
-          // #225: aggregate p95 across the pool's keys
-          p95: (() => {
-            const vals = (pool.refs ?? []).map((r) => latencyHistogram.snapshot(r)).filter((s) => s && s.p95 != null).map((s) => s.p95);
-            return vals.length ? Math.round(Math.max(...vals)) : null;
-          })(),
-          latencySloMs,
-          events: (pool.state.events ?? []).slice(-50),
-          healthScore: computeHealthScore(pool.state),
-          // #208: today/week spend + configured budget for the card
-          todayCost: costForDay(pool.state.costDays),
-          weeklyCost: costForWeek(pool.state.costDays, now),
-          budgetDaily: providerBudgets.get(pool.base)?.costBudgetDaily ?? 0,
-          budgetWeekly: providerBudgets.get(pool.base)?.costBudgetWeekly ?? 0,
-          pauseOnBudget: providerBudgets.get(pool.base)?.pauseOnBudget ?? false,
-        });
-        } catch (e) {
-          console.warn(`[dsh-key-rotation] status: pool ${pool.base} failed: ${String(e?.message ?? e)} ${e?.stack ?? ''}`);
-          providers.push({ provider: pool.base, keys: [], statusError: String(e?.message ?? e) });
-        }
-      }
-      json(res, 200, { providers });
-    },
-  }), 'dsh-key-rotation: status route');
-
-  // #209: usage report - per-key requests/cost over the last N days.
-  // ?format=csv returns text/csv; ?days=N window (1..90, default 7).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: USAGE_PATH,
-    handler: (req, res) => {
-      if (req.method !== 'GET') { json(res, 405, { error: { code: 'method', message: 'GET only' } }); return; }
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: usage is local-only' } }); return; }
-      const url = new URL(req.url ?? USAGE_PATH, 'http://localhost');
-      const days = Math.min(90, Math.max(1, Number(url.searchParams.get('days')) || 7));
-      const csv = url.searchParams.get('format') === 'csv';
-      const provider = url.searchParams.get('provider') ?? '';
-      const runtime = buildRuntime();
-      const now = Date.now();
-      const seen = new Set();
-      const report = [];
-      for (const pool of runtime.poolByRef.values()) {
-        if (seen.has(pool.base)) continue;
-        seen.add(pool.base);
-        if (provider && pool.base !== provider) continue;
-        report.push({ provider: pool.base, rows: usageRows(pool, days, now) });
-      }
-      if (csv) {
-        res.writeHead(200, { 'content-type': 'text/csv; charset=utf-8', 'content-disposition': 'attachment; filename="dsh-key-rotation-usage.csv"' });
-        const parts = [];
-        for (const p of report) {
-          if (parts.length > 0) parts.push('');
-          parts.push('# ' + p.provider);
-          parts.push(usageCsv(p.rows));
-        }
-        res.end(parts.join('\n') + '\n');
-        return;
-      }
-      json(res, 200, { at: now, days, providers: report });
-    },
-  }), 'dsh-key-rotation: usage route');
-
-  // #218: full config snapshot - one JSON file to move between machines.
-  // Secret values never travel: only credential/env names. Token fields are
-  // exported as empty strings; on import they keep existing values when empty.
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: SNAPSHOT_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'GET' && req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'GET (export) or POST (import) only' } }); return; }
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: snapshot is local-only' } }); return; }
-      if (req.method === 'GET') {
-        const descriptor = descriptorOf(ctx, NS);
-        const value = descriptor?.value ?? {};
-        const exportable = { ...value };
-        // token-shaped fields stay empty in the file; refs are names, not secrets
-        exportable.webhookActionToken = '';
-        json(res, 200, { at: Date.now(), version: 1, snapshot: exportable });
-        return;
-      }
-      // POST = import: { snapshot } -> merge with current section, PUT semantics
-      let body;
-      try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
-      const snap = body?.snapshot;
-      if (!snap || typeof snap !== 'object' || Array.isArray(snap)) { json(res, 400, { error: { code: 'bad-format', message: 'dsh-key-rotation: POST requires {"snapshot": {...}}' } }); return; }
-      // #200 leak guard applies to imported content too
-      try {
-        const masked = structuredClone(snap);
-        if (masked.webhookActionToken) masked.webhookActionToken = '***';
-        if (masked.notifyWebhook) masked.notifyWebhook = '***';
-        const findings = findSecrets(JSON.stringify(masked));
-        if (findings.length > 0) { json(res, 400, { error: { code: 'secret-in-snapshot', message: 'dsh-key-rotation: snapshot carries a live-looking credential', findings } }); return; }
-      } catch { /* scanning must never block a valid import */ }
-      const settings = ctx.get('settings');
-      if (!settings) { json(res, 503, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: no settings provider' } }); return; }
-      const desc = descriptorOf(ctx, NS);
-      if (desc === void 0) { json(res, 500, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: namespace missing' } }); return; }
-      const cur = desc.value ?? {};
-      // empty token fields in the file keep the current values (never wipe a secret)
-      const merged = { ...cur, ...snap };
-      if (!snap.webhookActionToken) merged.webhookActionToken = cur.webhookActionToken ?? '';
-      try {
-        await settings.replace(NS, merged, desc.revision);
-        const after = descriptorOf(ctx, NS);
-        json(res, 200, { ok: true, revision: after?.revision });
-      } catch (e) {
-        json(res, e?.code === 'SETTINGS_CONFLICT' ? 409 : 400, { error: { code: 'settings-rejected', message: String(e?.message ?? e) } });
-      }
-    },
-  }), 'dsh-key-rotation: snapshot route');
-
-  // ── key route: store a key value without leaving the rotation card ──
-  //
-  // Adding a key used to mean two screens: create the credential elsewhere,
-  // then type its env name here. The value is write-only from the browser —
-  // it is never sent back, only its last few characters are (see the status
-  // route) — and the route is loopback- and same-origin-gated like the config
-  // bridge next to it.
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: KEY_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'PUT' && req.method !== 'DELETE') {
-        json(res, 405, { error: { code: 'method', message: 'PUT or DELETE only' } });
-        return;
-      }
-      if (!isTrustedBridgeRequest(req)) {
-        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: keys are local-only' } });
-        return;
-      }
-      const credentialsService = ctx.get('credentials');
-      if (!credentialsService || typeof credentialsService.set !== 'function') {
-        json(res, 503, { error: { code: 'no-credentials', message: 'dsh-key-rotation: no credentials service is mounted' } });
-        return;
-      }
-      let body;
-      try {
-        body = await readJson(req);
-      } catch (error) {
-        json(res, 400, { error: { code: 'bad-request', message: String(error?.message ?? error) } });
-        return;
-      }
-      const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
-      if (!isValidRef(ref)) {
-        json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } });
-        return;
-      }
-      try {
-        if (req.method === 'DELETE') {
-          await credentialsService.unset(ref);
-          json(res, 200, { ok: true, ref });
-          return;
-        }
-        const value = typeof body?.value === 'string' ? body.value.trim() : '';
-        if (value.length === 0) {
-          json(res, 400, { error: { code: 'empty-value', message: 'dsh-key-rotation: an empty key cannot be stored' } });
-          return;
-        }
-        await credentialsService.set(ref, value);
-        // #200: leak-detector hint - stored value should look like a credential
-        const secretShape = looksLikeApiSecret(value);
-        json(res, 200, { ok: true, ref, tail: keyTail(value), looksLikeSecret: secretShape });
-      } catch (error) {
-        // A ref supplied by the launching environment is read-only, and the
-        // service says so in plain words — pass that through to the card.
-        json(res, 409, { error: { code: 'write-rejected', message: String(error?.message ?? error) } });
-      }
-    },
-  }), 'dsh-key-rotation: key route');
-
-  // ── reset route: clear cooldown for a provider (or a single ref) ──
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: RESET_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'POST') {
-        json(res, 405, { error: { code: 'method', message: 'POST only' } });
-        return;
-      }
-      if (!isTrustedBridgeRequest(req)) {
-        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: reset is local-only' } });
-        return;
-      }
-      let body;
-      try { body = await readJson(req); } catch (e) {
-        json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } });
-        return;
-      }
-      const provider = typeof body?.provider === 'string' ? body.provider.trim() : '';
-      const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
-      if (provider) {
-        const st = poolState.get(provider);
-        if (!st) { json(res, 404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }); return; }
-        const cleared = st.failedUntil.size;
-        st.failedUntil.clear();
-        st.failCounts?.clear();
-        st.authFailCounts?.clear();
-        st.brokenUntil?.clear();
-        st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
-        json(res, 200, { ok: true, provider, cleared });
-        return;
-      }
-      if (ref) {
-        let found = false;
-        for (const st of poolState.values()) {
-          if (st.failedUntil.has(ref) || st.failCounts?.has(ref)) {
-            st.failedUntil.delete(ref);
-            st.failCounts?.delete(ref);
-            st.authFailCounts?.delete(ref);
-            st.brokenUntil?.delete(ref);
-            if (st.lastUsed === ref) st.lastUsed = undefined;
-            found = true; break;
-          }
-        }
-        // idempotent: even if ref was not cooling, report ok if it looks like a valid ref name
-        if (!found && !isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
-        json(res, 200, { ok: true, ref });
-        return;
-      }
-      json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: POST requires {"provider": "..."} or {"ref": "..."}' } });
-    },
-  }), 'dsh-key-rotation: reset route');
-
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: IMPORT_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: import is local-only' } }); return; }
-      let body; try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
-      const url = typeof body?.url === 'string' ? body.url.trim() : '';
-      if (!url || !url.startsWith('https://')) { json(res, 400, { error: { code: 'bad-url', message: 'dsh-key-rotation: only HTTPS URLs are allowed' } }); return; }
-      try {
-        const resp = await fetch(url);
-        if (!resp.ok) { json(res, 400, { error: { code: 'fetch-failed', message: 'dsh-key-rotation: fetch returned ' + resp.status } }); return; }
-        const data = await resp.json();
-        if (!Array.isArray(data)) { json(res, 400, { error: { code: 'bad-format', message: 'dsh-key-rotation: expected JSON array of providers' } }); return; }
-        const settings = ctx.get('settings');
-        if (!settings) { json(res, 503, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: no settings provider' } }); return; }
-        const desc = settings.describe({ redactSecrets: true }).find((c) => c.ns === NS);
-        const cur = desc?.value?.providers ?? [];
-        const merged = new Map();
-        for (const p of cur) if (p && p.provider) merged.set(p.provider, p);
-        for (const p of data) if (p && p.provider && typeof p.provider === 'string') merged.set(p.provider, p);
-        const mergedArr = [...merged.values()];
-        await settings.replace(NS, { ...(desc?.value ?? {}), providers: mergedArr }, desc?.revision);
-        json(res, 200, { ok: true, providersImported: data.length, total: mergedArr.length });
-      } catch (e) { json(res, 400, { error: { code: 'import-failed', message: String(e?.message ?? e) } }); }
-    },
-  }), 'dsh-key-rotation: import route');
-
-  // Health for external panels (Beszel/Uptime)
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: HEALTH_PATH,
-    handler: async (req, res) => {
-      if (!isTrustedBridgeRequest(req) && req.socket?.remoteAddress !== '127.0.0.1' && req.socket?.remoteAddress !== '::1') { } // allow same-origin already checked
-      if (!isTrustedBridgeRequest(req)) {
-        // also allow plain loopback without Origin
-        if (!isLoopbackAddress(req.socket?.remoteAddress)) { res.writeHead(403); res.end(); return; }
-        if (req.headers['sec-fetch-site'] === 'cross-site') { res.writeHead(403); res.end(); return; }
-      }
-      if (req.method !== 'GET') { json(res, 405, { error: { code: 'method', message: 'GET only' } }); return; }
-      const now = Date.now();
-      const pools = {};
-      let exhaustedAny = false;
-      const { poolByRef: pr, providerTags } = buildRuntime();
-      const seenH = new Set();
-      for (const pool of pr.values()) {
-        if (seenH.has(pool.base)) continue;
-        seenH.add(pool.base);
-        let healthy = 0;
-        for (const ref of pool.refs) {
-          const until = pool.state.failedUntil.get(ref);
-          if (until !== undefined && until > now) continue;
-          const exp = pool.expiresAt?.[ref];
-          if (exp !== undefined && now >= exp) continue;
-          healthy++;
-        }
-        const total = pool.refs.length;
-        const exhausted = healthy === 0 && total > 0;
-        if (exhausted) exhaustedAny = true;
-        pools[pool.base] = { healthy, total, exhausted, healthScore: computeHealthScore(pool.state) };
-      }
-      json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: quotaStore.snapshot() });
-    },
-  }), 'dsh-key-rotation: health');
-
-  // ── test route: dry-run a single key without rotation ──
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: TEST_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: test is local-only' } }); return; }
-      let body; try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
-      const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
-      if (!isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
-      // Optional value for pre-save validation (issue #118)
-      const testValue = typeof body?.value === 'string' && body.value.length > 0 ? body.value : undefined;
-      const probe = body?.probe === 'models' || body?.probe === 'chat' ? body.probe : undefined;
-      const base = ctx.get('credentials');
-      try {
-        let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
-        let present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
-        const effectiveValue = testValue || hit?.value;
-        const valid = present ? Boolean(effectiveValue && typeof effectiveValue === 'string' && effectiveValue.length > 0) : Boolean(testValue);
-        const tail = valid ? keyTail(effectiveValue) : '';
-        let source = null;
-        try { const d = await base?.describe?.(ref); source = d?.source ?? null; } catch {}
-        if (!present && !testValue) { json(res, 200, { ok: false, ref, code: 'no-credential', message: 'no such credential' }); return; }
-        if (!present && testValue) { source = 'pre-save'; }
-        else if (!present) {
-          const ev = envValue(ref);
-          if (ev !== undefined) { present = true; json(res, 200, { ok: true, ref, tail: keyTail(ev), source: 'env' }); return; }
-        }
-        // sandbox probe (models is free; chat is hook-only, see sandbox.js)
-        if (probe) {
-          const keyForProbe = effectiveValue;
-          const runner = ensureSandboxRunner(ctx);
-          const result = probe === 'chat' ? await runner.probeChat(ref, keyForProbe) : await runner.probeModels(ref, keyForProbe);
-          const cached = { ...result, at: Date.now() };
-          lastTestCache.set(ref, cached);
-          if (cached.ok) {
-            for (const st of poolState.values()) {
-              if (st.failedUntil?.has(ref) || st.failCounts?.has(ref) || st.brokenUntil?.has(ref)) {
-                st.failedUntil?.delete(ref);
-                st.failCounts?.delete(ref);
-                st.authFailCounts?.delete(ref);
-                st.brokenUntil?.delete(ref);
-              }
-            }
-          }
-          json(res, 200, { ok: cached.ok, ref, tail, source, probe, code: cached.code, latencyMs: cached.latencyMs, modelsCount: cached.modelsCount });
-          return;
-        }
-        json(res, 200, { ok: true, ref, tail, source });
-      } catch (e) {
-        json(res, 200, { ok: false, ref, code: 'error', message: String(e?.message ?? e) });
-      }
-    },
-  }), 'dsh-key-rotation: test route');
-
-  // Intercept the llm/stream waterfall: rotate any request whose provider maps
-  // to a configured key pool; pass everything else (and internal dispatches)
-  // straight through.
-  // Read-only cache snapshot for clients (badge polling).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: SANDBOX_CACHE_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: cache is local-only' } }); return; }
-      json(res, 200, lastTestCache.snapshot());
-    },
-  }), 'dsh-key-rotation: sandbox cache');
-
-
-
-  // #199 webhook-action: interactive webhook buttons call back here.
-  // Auth: bearer token from Config (external services like Telegram/Discord
-  // cannot be same-origin, so a shared secret is the gate).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: '/dsh-key-rotation/webhook-action',
-    handler: async (req, res) => {
-      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-      const runtime = buildRuntime();
-      const expected = runtime.webhookActionToken;
-      if (!expected) { json(res, 503, { error: { code: 'no-token', message: 'dsh-key-rotation: webhookActionToken is not configured' } }); return; }
-      const auth = String(req.headers.authorization ?? '');
-      if (auth !== `Bearer ${expected}`) { json(res, 401, { error: { code: 'unauthorized', message: 'dsh-key-rotation: bad webhook action token' } }); return; }
-      let body;
-      try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
-      // Accept callback payloads from formatInteractive (Telegram/Discord/Slack) or plain {action}
-      let action = typeof body?.action === 'string' ? body.action : '';
-      if (!action && typeof body?.data === 'string') {
-        try { action = String(JSON.parse(body.data)?.id ?? ''); } catch { action = ''; }
-      }
-      if (!action && typeof body?.callback_data === 'string') {
-        try { action = String(JSON.parse(body.callback_data)?.id ?? ''); } catch { action = ''; }
-      }
-      // #222: Telegram update envelope {update_id, callback_query:{data}}
-      if (!action && typeof body?.callback_query?.data === 'string') {
-        try { action = String(JSON.parse(body.callback_query.data)?.id ?? ''); } catch { action = ''; }
-      }
-      // #222: Telegram setWebhook registration helper
-      if (typeof body?.setWebhook === 'object' && body.setWebhook) {
-        const botToken = typeof body.setWebhook.botToken === 'string' ? body.setWebhook.botToken : '';
-        if (!botToken) { json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: setWebhook.botToken required' } }); return; }
-        // derive the public URL from request headers; explicit URL wins
-        const url = typeof body.setWebhook.url === 'string' && body.setWebhook.url ? body.setWebhook.url : `https://${String(req.headers.host ?? '')}/dsh-key-rotation/webhook-action`;
-        try {
-          const hookRes = await fetch(`https://api.telegram.org/bot${botToken}/setWebhook`, {
-            method: 'POST', headers: { 'content-type': 'application/json' },
-            body: JSON.stringify({ url, allowed_updates: ['callback_query'] }),
-          });
-          const hookData = await hookRes.json().catch(() => ({}));
-          json(res, 200, { ok: hookRes.ok, url, telegram: hookData });
-        } catch (e) {
-          json(res, 502, { error: { code: 'telegram-failed', message: String(e?.message ?? e) } });
-        }
-        return;
-      }
-      if (!action) { json(res, 400, { error: { code: 'bad-action', message: 'dsh-key-rotation: no action in payload' } }); return; }
-      const provider = action.startsWith('pause-') || action.startsWith('reset-') ? action.replace(/^(pause|reset)-/, '') : '';
-      try {
-        if (action === 'disable-rotation') {
-          rotationDisabled = true;
-          console.warn('[dsh-key-rotation] rotation DISABLED via webhook action');
-          json(res, 200, { ok: true, action });
-          return;
-        }
-        if (action === 'enable-rotation') {
-          rotationDisabled = false;
-          json(res, 200, { ok: true, action });
-          return;
-        }
-        if (action.startsWith('pause-') || action.startsWith('reset-')) {
-          const st = poolState.get(provider);
-          if (!st) { json(res, 404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }); return; }
-          if (action.startsWith('pause-')) {
-            const until = Date.now() + 3600000; // 1h pause
-            for (const ref of (st.failedUntil ? [...st.failedUntil.keys()] : [])) st.failedUntil.set(ref, Math.max(st.failedUntil.get(ref) ?? 0, until));
-            // also pause every key currently healthy
-            for (const p of buildRuntime().poolByRef.values()) {
-              if (p.base !== provider) continue;
-              for (const ref of p.refs) st.failedUntil.set(ref, Math.max(st.failedUntil.get(ref) ?? 0, until));
-            }
-            console.warn(`[dsh-key-rotation] pool ${provider} PAUSED 1h via webhook action`);
-            json(res, 200, { ok: true, action, provider, until: Date.now() + 3600000 });
-            return;
-          }
-          const cleared = st.failedUntil.size;
-          st.failedUntil.clear(); st.failCounts?.clear(); st.brokenUntil?.clear();
-          console.warn(`[dsh-key-rotation] pool ${provider} RESET via webhook action`);
-          json(res, 200, { ok: true, action, provider, cleared });
-          return;
-        }
-        json(res, 400, { error: { code: 'unknown-action', message: `dsh-key-rotation: unknown action '${action}'` } });
-      } catch (e) {
-        json(res, 500, { error: { code: 'action-failed', message: String(e?.message ?? e) } });
-      }
-    },
-  }), 'dsh-key-rotation: webhook-action');
-
-
+  // Operational routes (status/usage/snapshot/key/import/test/health/webhook) (#253)
+  registerOpsRoutes(ctx, {
+    buildRuntime,
+    latencyHistogram,
+    lastTestCache,
+    ensureSandboxRunner,
+    poolState,
+    getRotationDisabled: () => rotationDisabled,
+    setRotationDisabled: (v) => { rotationDisabled = v; },
+  });
 
   ctx.effect(() => ctx.on('llm/stream', (options, next) => {
     if (options[MARKER]) return next();
@@ -1608,7 +756,9 @@ export function apply(ctx, config = {}) {
     // #195: exact model pool -> longest model-family prefix -> provider pool
     const pool = selectPool(modelPoolByProvider, providerToPool, options.provider, options.model);
     if (!pool) return next();
-    console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${(pool.weightedRefs ?? pool.refs).length} slots (${pool.refs.length} keys)`);
+    if (buildRuntime()?.verboseLogging) {
+      console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${(pool.weightedRefs ?? pool.refs).length} slots (${pool.refs.length} keys)`);
+    }
     return rotate(options, pool);
   }), 'dsh-key-rotation: llm/stream');
 
@@ -1627,11 +777,13 @@ export function apply(ctx, config = {}) {
     const code = String(payload?.failure?.code ?? payload?.code ?? '');
     const message = String(payload?.failure?.message ?? payload?.message ?? '');
     const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-    const switchable = isSwitchableError(payload, effectiveSwitchCodes);
+    const cls = classifyFailure(payload);
+    const switchable = isSwitchableError(payload, effectiveSwitchCodes) || cls.action === 'switch';
     if (!switchable) return next();
+    if (moduleBreaker) moduleBreaker.onFailure(provider);
     const ref = pool.state.lastUsed;
     if (ref) {
-      const backoff = recordFailure(pool, ref, Date.now(), pool.cooldownMs ?? 60000);
+      const backoff = recordFailure(pool, ref, Date.now(), pool.cooldownMs ?? 60000, undefined, cls.soft);
       pushEvent(pool, ref, code || 'UNKNOWN', backoff);
       pool.state.switches = (pool.state.switches ?? 0) + 1;
       pool.state.lastReason = code || 'UNKNOWN';
@@ -1673,7 +825,8 @@ export function notifyExhaustion(runtime, pool, options, hooks = { webhookSender
           { id: `pause-${options.provider}`, label: 'Pause 1h' },
         ] : undefined,
       };
-      hooks.webhookSender.send(runtime.notifyWebhook, payload);
+      if (hooks.notifyQueue) hooks.notifyQueue.enqueue(runtime.notifyWebhook, payload);
+      else hooks.webhookSender.send(runtime.notifyWebhook, payload);
     }
   } catch (_) { /* ponytail: never crash rotate() */ }
 }