@goodandready/dsh-key-rotation 0.8.11 → 0.8.13

package/lib/index.js

@@ -3,33 +3,17 @@
 // Transparent key pool rotation: credentials.resolve patch + llm/stream
 // interceptor. Provider identity never changes (keeps pi-ai replay state).
 // See README.md and docs/design/DESIGN.md for the full contract.
-// Config (all optional): switchCodes, cooldownMs, providers[{provider,keys}],
-// verboseLogging (default false) gates per-request rotation logs.
 
 import { AsyncLocalStorage } from 'node:async_hooks';
 import Schema from '@deepseek-ai/schemastery';
-import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter, computeHealthScore, extractRateLimit, isRateLimited, selectPool, isSwitchableError, formatExhaustionMessage, expiringSoon, shouldNotifyDaily, costForDay, costForWeek, budgetVerdict } from './pool.js';
-
-export const name = 'dsh-key-rotation';
-export const inject = ['llm', 'webServer', 'settings', 'credentials'];
-export { keyTail, isLoopbackAddress, isTrustedBridgeRequest, DEFAULT_SWITCH_CODES, isSwitchableError, formatExhaustionMessage, getRuntime };
-
-/** Settings namespace owning the GUI-editable section (settingsNamespace-valid). */
-const NS = 'dsh-key-rotation';
-/** Config bridge route (GET / PUT / DELETE), loopback-fenced like llm-fallback. */
-const CONFIG_PATH = '/dsh-key-rotation/config';
-const STATUS_PATH = '/dsh-key-rotation/status';
-const SNAPSHOT_PATH = '/dsh-key-rotation/snapshot';
-const KEY_PATH = '/dsh-key-rotation/key';
-const RESET_PATH = '/dsh-key-rotation/reset';
-const IMPORT_PATH = '/dsh-key-rotation/import';
-const HEALTH_PATH = '/dsh-key-rotation/health';
-const USAGE_PATH = '/dsh-key-rotation/usage';
-const TEST_PATH = '/dsh-key-rotation/test';
-const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
-import { sortAttemptList } from './pool.js';
-import { LastTestCache, SandboxRunner } from './sandbox.js';
-import { healIdleCooldowns, autoUnbreakBrokenKeys } from './heal.js';
+import {
+  keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN,
+  DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure,
+  recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter,
+  computeHealthScore, extractRateLimit, isRateLimited, selectPool, isSwitchableError,
+  formatExhaustionMessage, expiringSoon, shouldNotifyDaily, costForDay, costForWeek,
+  budgetVerdict, sortAttemptList,
+} from './pool.js';
 import { LatencyHistogram } from './histogram.js';
 import { pickCascadeFallback } from './cascade.js';
 import { ConcurrencyTracker } from './concurrency.js';
@@ -38,8 +22,6 @@ import { QuotaStore } from './quota.js';
 import { WebhookSender } from './webhook.js';
 import { bucketAllow, bucketRetryMs, bucketSweep, bucketInfo } from './bucket.js';
 import { usageRows, usageCsv, compactUsage } from './usage-report.js';
-
-const dispatchStorage = new AsyncLocalStorage();
 import { findSecrets, looksLikeApiSecret } from './keycheck.js';
 import { json, readJson, descriptorOf, viewOf, writeSection, providerCatalog, handleConfigBridge, NS as BRIDGE_NS } from './http-bridge.js';
 import { createRotate } from './rotate.js';
@@ -50,135 +32,49 @@ import { BoundedMap } from './bounded-map.js';
 import { classifyFailure } from './error-taxonomy.js';
 import { safeParseJson } from './atomic-io.js';
 import { registerOpsRoutes } from './routes-ops.js';
-import { StatePersistence, resolveStatePath } from './persistence.js';
-import path from 'node:path';
-import { registerPluginUpdater } from './plugin-updater.js'
+import { registerPluginUpdater } from './plugin-updater.js';
+import { notifySwitch, notifyExhaustion, pushEvent } from './notify-events.js';
+import { getLogger } from './logger.js';
+import { createSandboxService } from './sandbox-service.js';
+import { checkBudgetAndHealthAlerts } from './budget-monitor.js';
+import { buildPoolItem, cleanupRemovedProviders } from './pool-builder.js';
+import { setupIdleHealEffect, setupAutoUnbreakEffect, setupPersistence } from './lifecycle.js';
+
+export const name = 'dsh-key-rotation';
+export const inject = ['llm', 'webServer', 'settings', 'credentials'];
+export { keyTail, isLoopbackAddress, isTrustedBridgeRequest, DEFAULT_SWITCH_CODES, isSwitchableError, formatExhaustionMessage, getRuntime };
+export { notifySwitch, notifyExhaustion };
 
-/** The llm-pi-ai namespace whose provider profiles map providers to pools. */
+const NS = 'dsh-key-rotation';
+const CONFIG_PATH = '/dsh-key-rotation/config';
 const PIAI_NS = 'llm-pi-ai';
-/** Marker on internally re-dispatched requests so the interceptor does not loop. */
 const MARKER = '__dshKeyRotation';
-/** #199: set true via webhook action; checked in the llm/stream interceptor. */
+
 let rotationDisabled = false;
-// #207/#208 dedupe maps: one notification per key/window per day.
 const expiryNotifiedAt = new Map();
 const budgetNotifiedAt = new Map();
-const switchNotifiedAt = new Map();
 const lowHealthNotifiedAt = new Map();
 const sloNotifiedAt = new Map();
-const DAY_MS = 86400000;
-
-// #216: one webhook per switch, deduped to at most one message per provider
-// per switchNotifyThrottleMs. Extracted for testability.
-export function notifySwitch(runtime, pool, info, hooks = { webhookSender, now: () => Date.now() }) {
-  if (!runtime?.notifyWebhook) return;
-  const throttle = Math.max(0, runtime.switchNotifyThrottleMs ?? 60000);
-  const last = switchNotifiedAt.get(info.provider) ?? 0;
-  const now = hooks.now();
-  if (now - last < throttle) return;
-  switchNotifiedAt.set(info.provider, now);
-  // #263: non-blocking — enqueue never awaits webhook I/O
-  const send = hooks.notifyQueue
-    ? (url, payload) => { hooks.notifyQueue.enqueue(url, payload); return { sent: true, queued: true }; }
-    : (url, payload) => hooks.webhookSender.send(url, payload);
-  send(runtime.notifyWebhook, {
-    title: `Key switched: ${info.provider}`,
-    text: `${info.from} failed (${info.code}) - next key in pool`,
-    provider: info.provider,
-    kind: 'switch',
-    from: info.from,
-    code: info.code,
-    at: info.at,
-  });
-}
-const MAX_EVENTS = 50;
-function pushEvent(pool, ref, reason, cooldownMs, type) {
-  const ev = { at: Date.now(), ref, reason: String(reason ?? 'UNKNOWN'), cooldownMs, type: type ?? 'fail' };
-  pool.state.events.push(ev);
-  if (pool.state.events.length > MAX_EVENTS) pool.state.events.shift();
-}
-
 
-// Fallback classification by failure message. pi-ai surfaces many real quota /
-// rate-limit / transport failures as thrown exceptions (e.g. the OpenAI SDK
-// throws on HTTP 429 before the stream starts), and dsh-llm then normalizes
-// them to finish chunks with code "UNKNOWN". The message still carries the
-// provider's own text ("429: ...", "Weekly usage limit reached", ...), so we
-// treat pre-content failures whose message matches these patterns as
-// switchable even when the code is not in `switchCodes`.
-
-// Sandbox-test infrastructure (sandbox.js): in-memory cache + runner.
-let lastTestCacheRunnerCtx = null;
-const lastTestCache = new LastTestCache();
+const dispatchStorage = new AsyncLocalStorage();
 const latencyHistogram = new LatencyHistogram();
 const quotaStore = new QuotaStore();
-// #260/#263 module-scope infra (process-wide, reset on reload via buildRuntime)
-let moduleBreaker = null;
-let moduleNotifyQueue = null;
-// Global config accessor safe against early initialization
+const concurrencyTracker = new ConcurrencyTracker();
+
+let moduleBreaker = new CircuitBreaker({ threshold: 5, openMs: 30000, halfOpenProbes: 1, now: nowMono });
+const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
+let moduleNotifyQueue = new NotifyQueue({ send: (url, payload) => webhookSender.send(url, payload) });
+
 let getConfig = () => null;
 function verboseLoggingOn() {
   try { return Boolean(getConfig()?.verboseLogging); } catch (_) { return false; }
 }
+
 let getRuntime = () => null;
-let sandboxRunner = null;
-const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
-// #263: queue webhook I/O off the hot path
-moduleNotifyQueue = new NotifyQueue({ send: (url, payload) => webhookSender.send(url, payload) });
-// #260: process-wide breaker; thresholds re-read from runtime when dispatching
-moduleBreaker = new CircuitBreaker({ threshold: 5, openMs: 30000, halfOpenProbes: 1, now: nowMono });
-const concurrencyTracker = new ConcurrencyTracker();
-function ensureSandboxRunner(ctx) {
-  if (sandboxRunner) return sandboxRunner;
-  // provider id or key ref -> baseUrl (stripped of trailing /) for fetch /models probe
-  function resolveBaseUrl(providerOrRef) {
-    try {
-      let provider = providerOrRef;
-      const rt = typeof getRuntime === 'function' ? getRuntime() : null;
-      const pool = rt?.poolByRef?.get(providerOrRef);
-      if (pool?.base) provider = pool.base;
-      else if (pool?.provider) provider = pool.provider;
-
-      const c = ctx || lastTestCacheRunnerCtx;
-      const pInfo = c?.llm?.getProvider?.(provider);
-      if (pInfo && (pInfo.baseUrl || pInfo.endpoint || pInfo.url)) {
-        return String(pInfo.baseUrl || pInfo.endpoint || pInfo.url);
-      }
-      for (const info of (c?.llm?.listProviders?.() || [])) {
-        if (info && (info.id === provider || info.name === provider)) {
-          const u = info.baseUrl || info.endpoint || info.url;
-          if (u) return String(u);
-        }
-      }
-      const ns = c?.get ? c.get(PIAI_NS) : null;
-      const list = ns && (ns.providers || (ns.config && ns.config.providers) || []);
-      if (Array.isArray(list)) {
-        const hit = list.find((p) => p && (p.id === provider || p.name === provider || (Array.isArray(p.aliases) && p.aliases.includes(provider))));
-        const base = hit && (hit.baseUrl || hit.endpoint || hit.url);
-        if (base) return String(base);
-      }
-      return null;
-    } catch (_) {
-      return null;
-    }
-  }
-  sandboxRunner = new SandboxRunner({ fetchImpl: globalThis.fetch, resolveBaseUrl });
-  return sandboxRunner;
-}
-async function probeRef(ref, key) {
-  // ref may be like "PROVIDER/KEY_NAME" — for sandbox we only care about the credential ref
-  // (the resolveBaseUrl uses the full provider id; ref can carry any string)
-  const runner = ensureSandboxRunner(lastTestCacheRunnerCtx);
-  const result = await runner.probeModels(ref, key);
-  lastTestCache.set(ref, { ...result, at: Date.now() });
-  return result;
-}
 
-// // Bootstrap key pools. The user configures them in the Settings GUI or via
-// the dsh profile bundle config; the plugin itself ships no provider defaults
-// so it does not bind to any specific installation. Empty array means: until
-// the user adds a pool, no rotation happens, and every provider falls back to
-// its single configured credential exactly as before this plugin was installed.
+const sandboxService = createSandboxService({ getRuntime: () => getRuntime() });
+const { ensureSandboxRunner, probeRef, lastTestCache } = sandboxService;
+
 const DEFAULT_PROVIDERS = [];
 
 export const Config = Schema.object({
@@ -209,12 +105,10 @@ export const Config = Schema.object({
   expiryWarnDays: Schema.number().default(7),
   switchNotify: Schema.boolean().default(false),
   verboseLogging: Schema.boolean().default(false),
-  // #260 circuit breaker
   circuitBreakerEnabled: Schema.boolean().default(true),
   circuitBreakerThreshold: Schema.number().default(5),
   circuitBreakerOpenMs: Schema.number().default(30000),
   circuitBreakerHalfOpenProbes: Schema.number().default(1),
-  // #287 persistence across restarts
   persistenceEnabled: Schema.boolean().default(true),
   persistencePath: Schema.string().default(''),
   switchNotifyThrottleMs: Schema.number().default(60000),
@@ -240,163 +134,44 @@ export const Config = Schema.object({
   })).default([...DEFAULT_PROVIDERS]),
 });
 
-// ── config bridge (GET/PUT/DELETE on CONFIG_PATH), mirroring llm-fallback ──
-
-
 function registerConfigBridge(ctx, getCloneIds) {
   return ctx.webServer.register({
     kind: 'exact',
     path: CONFIG_PATH,
-    handler: (req, res) => void handleConfigBridge(ctx, req, res, getCloneIds),
+    handler: (req, res) => {
+      if (!['GET', 'PUT', 'DELETE', 'OPTIONS'].includes(req.method)) {
+        res.writeHead(405, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ error: 'Method Not Allowed' }));
+        return;
+      }
+      handleConfigBridge(ctx, req, res, getCloneIds);
+    },
   });
 }
 
-// ── plugin ──
-
 export function apply(ctx, config = {}) {
-  // GUI section: defaults -> cordis row config -> saved user section.
-  // (installSettingsSection inlined: no @deepseek-ai/dsh-settings import, so the
-  // profile does not need a second copy of that package.)
+  const logger = getLogger(ctx);
   getConfig = () => config;
   registerConfigBridge(ctx, () => buildRuntime().cloneIds);
-  lastTestCacheRunnerCtx = ctx;
-  // Cache should not survive profile restarts (apply is called per reload).
-  // We deliberately do NOT clear on every apply — that would wipe badges when
-  // the user is just typing in the settings card. Re-init only on true reload.
   ensureSandboxRunner(ctx);
 
-  // Self-healing idle cooldowns: every 60s, lift expired cooldowns for keys
-  // that have been idle for selfHealIdleMs (default 1h). ponytail: small
-  // interval, low cost; skipped when selfHealCooldown is disabled in config.
-  // ponytail: keep handle on the same ctx via closure so buildRuntime() reads
-  // fresh config on every tick. Naive but correct: 60s cadence is cheap.
-  // Self-healing idle cooldowns lifecycle effect
-  ctx.effect(() => {
-    const cfg = getConfig();
-    if (!cfg || cfg.selfHealCooldown === false) return () => {};
-    const timer = setInterval(() => {
-      try {
-        const c = getConfig();
-        if (!c || c.selfHealCooldown === false) return;
-        const idle = Number.isFinite(c.selfHealIdleMs) && c.selfHealIdleMs > 0 ? c.selfHealIdleMs : 3600000;
-        const providers = Array.isArray(c.providers) ? c.providers : [];
-        const pools = providers
-          .map((p) => buildRuntime().providerToPool.get(p.provider))
-          .filter(Boolean);
-        healIdleCooldowns(pools, idle);
-      } catch (_) { /* ponytail: never crash the timer */ }
-    }, 60000);
-    if (typeof timer.unref === 'function') timer.unref();
-    return () => clearInterval(timer);
-  }, 'dsh-key-rotation: self-healing idle');
-
-  // ── key-pool state, persisted across config reloads ──
-  // base provider -> { failedUntil: Map<ref, epochMs>, pointer: number, lastUsed: ref }
-  const poolState = new Map();
+  setupIdleHealEffect(ctx, getConfig, buildRuntime);
 
-  // #287: persist cooldowns / circuit / quota across DSH reloads.
-  // Path resolution: explicit config -> host data dir -> DSH_HOME -> cwd.
-  // Disabled (with a single warn) when no writable directory can be found.
-  let statePersistence = null;
-  try {
-    const cfg0 = getConfig() ?? config ?? {};
-    // Cordis ctx properties require inject; bare getters throw.
-    // Use only host env/cwd so apply() never trips the injector.
-    const hostDirs = [
-      process.env.DSH_HOME,
-      process.cwd(),
-    ].filter((d) => typeof d === 'string' && d.length > 0);
-    const resolvedPath = resolveStatePath({
-      configuredPath: cfg0.persistencePath,
-      dataDir: hostDirs[0],
-    });
-    if (cfg0.persistenceEnabled !== false && resolvedPath) {
-      statePersistence = new StatePersistence({ filePath: resolvedPath });
-      statePersistence.load().then((snap) => {
-        if (!snap) return;
-        try {
-          StatePersistence.restorePools(poolState, snap);
-          if (moduleBreaker && snap.circuit) moduleBreaker.restore(snap.circuit);
-          if (verboseLoggingOn()) {
-            console.warn(`[dsh-key-rotation] restored ${Object.keys(snap.pools ?? {}).length} pool state(s) from ${path.basename(resolvedPath)}`);
-          }
-        } catch (e) {
-          console.warn('[dsh-key-rotation] persistence restore failed', e?.message ?? e);
-        }
-      }).catch(() => {});
-    } else if (cfg0.persistenceEnabled !== false && !resolvedPath) {
-      console.warn('[dsh-key-rotation] persistence disabled: no data directory');
-    }
-  } catch (e) {
-    console.warn('[dsh-key-rotation] persistence init failed', e?.message ?? e);
-  }
-
-  function persistenceSnapshot() {
-    if (!statePersistence) return null;
-    return StatePersistence.serialize({
-      poolState,
-      circuitSnapshot: moduleBreaker ? moduleBreaker.snapshot() : {},
-      quotaSnapshot: {},
-    });
-  }
-
-  function schedulePersist() {
-    if (!statePersistence) return;
-    const snap = persistenceSnapshot();
-    if (snap) statePersistence.save(snap);
-  }
+  const poolState = new Map();
+  const { schedulePersist, persistenceSnapshot } = setupPersistence(ctx, {
+    cfg0: getConfig() ?? config ?? {},
+    poolState,
+    moduleBreaker,
+    verboseLoggingOn,
+    logger,
+  });
 
-    // Auto-healing / auto-unbreak for broken keys (#303)
-  ctx.effect(() => {
-    const cfg = getConfig();
-    const intervalMin = cfg?.selfHealingIntervalMinutes ?? 30;
-    if (!intervalMin || intervalMin <= 0) return () => {};
-    const intervalMs = intervalMin * 60 * 1000;
-    const timer = setInterval(async () => {
-      try {
-        const c = getConfig();
-        if (!c || !c.selfHealingIntervalMinutes || c.selfHealingIntervalMinutes <= 0) return;
-        const { pools } = buildRuntime();
-        const runner = ensureSandboxRunner(ctx);
-        await autoUnbreakBrokenKeys(pools, async (ref) => {
-          let val = (await ctx.credentials?.resolve?.(ref))?.value;
-          if (!val) return { ok: false };
-          return runner.probeModels(ref, val);
-        });
-      } catch (e) { ctx.logger?.warn?.('[dsh-key-rotation] auto-unbreak failed', e); }
-    }, intervalMs);
-    if (typeof timer.unref === 'function') timer.unref();
-    return () => clearInterval(timer);
-  }, 'dsh-key-rotation: auto-unbreak');
+  setupAutoUnbreakEffect(ctx, getConfig, buildRuntime, ensureSandboxRunner, logger);
 
-  ctx.effect(() => {
-    const timer = setInterval(() => {
-      try { schedulePersist(); }
-      catch (e) { ctx.logger?.warn?.('[dsh-key-rotation] periodic persist failed', e); }
-    }, 15000);
-    if (typeof timer.unref === 'function') timer.unref();
-    return () => {
-      clearInterval(timer);
-      try {
-        if (statePersistence) {
-          const snap = persistenceSnapshot();
-          if (snap) {
-            statePersistence.save(snap);
-            // best-effort flush; dispose clears the debounce timer
-            void statePersistence.flush();
-          }
-          statePersistence.dispose();
-        }
-      } catch (e) {
-        ctx.logger?.warn?.('[dsh-key-rotation] dispose persist failed', e);
-      }
-    };
-  }, 'dsh-key-rotation: state persistence');
-  // Periodic sweep of expired cooldowns — keeps health probe cheap and avoids waiting for next user request
+  // Periodic sweep of expired cooldowns
   ctx.effect(() => {
     const id = setInterval(() => {
       const now = Date.now();
-      // probe events for keys whose cooldown just expired
       for (const st of poolState.values()) {
         for (const [ref, until] of [...(st.failedUntil?.entries() ?? [])]) {
           if (until <= now && !st.probedAt?.has(ref)) {
@@ -410,160 +185,60 @@ export function apply(ctx, config = {}) {
       const runtime = buildRuntime();
       const allActiveRefs = Array.from(runtime.poolByRef.keys());
       const n = sweepExpired(poolState, now, allActiveRefs);
-      if (n > 0) console.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
+      if (n > 0) logger.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
       for (const pool of runtime.poolByRef.values()) {
         compactUsage(pool, 30, now);
       }
-      // #207 expiry pre-warning + #208 cost budget - piggybacked on this timer,
-      // deduped to one notification per key/window per day (shouldNotifyDaily).
-      try {
-        const seen = new Set();
-        for (const pool of runtime.poolByRef.values()) {
-          if (seen.has(pool.base)) continue;
-          seen.add(pool.base);
-          // #207: keys expiring within expiryWarnDays -> one webhook per key/day
-          for (const { ref, expiresInDays } of expiringSoon(pool, runtime.expiryWarnDays, now)) {
-            if (!shouldNotifyDaily(expiryNotifiedAt, pool.base + ':' + ref, now)) continue;
-            console.warn(`[dsh-key-rotation] ${pool.base}: key ${ref} expires in ~${expiresInDays}d`);
-            if (runtime.notifyWebhook) {
-              webhookSender.send(runtime.notifyWebhook, {
-                title: `Key expiring soon: ${pool.base}`,
-                text: `${ref} expires in ~${expiresInDays} day(s)`,
-                provider: pool.base,
-                kind: 'expiry',
-                keys: [ref],
-              });
-            }
-          }
-          // #208: daily/weekly budget -> warn webhook, optional 1-day pause at 100%
-          const budget = runtime.providerBudgets.get(pool.base);
-          if (!budget) continue;
-          const daily = costForDay(pool.state.costDays);
-          const weekly = costForWeek(pool.state.costDays, now);
-          const verdict = budgetVerdict(daily, budget.costBudgetDaily);
-          const wVerdict = budgetVerdict(weekly, budget.costBudgetWeekly);
-          const hit = verdict.warn || wVerdict.warn;
-          if (hit && shouldNotifyDaily(budgetNotifiedAt, pool.base + ':budget', now)) {
-            console.warn(`[dsh-key-rotation] ${pool.base}: cost budget - day $${daily.toFixed(2)}/$${budget.costBudgetDaily} week $${weekly.toFixed(2)}/$${budget.costBudgetWeekly}`);
-            if (runtime.notifyWebhook) {
-              // #217: budget webhook gains action buttons when a callback token
-              // is configured (the /webhook-action route already knows these ids)
-              const token = runtime.webhookActionToken ?? '';
-              webhookSender.send(runtime.notifyWebhook, {
-                title: `Cost budget: ${pool.base}`,
-                text: `day $${daily.toFixed(2)} of $${budget.costBudgetDaily} · week $${weekly.toFixed(2)} of $${budget.costBudgetWeekly}` + (verdict.exceeded || wVerdict.exceeded ? ' · EXCEEDED' : ''),
-                provider: pool.base,
-                kind: 'budget',
-                spend: { daily, weekly },
-                actionToken: token || undefined,
-                actions: token ? [
-                  { id: `pause-${pool.base}`, label: 'Pause 1h' },
-                  { id: `reset-${pool.base}`, label: 'Reset cooldown' },
-                ] : undefined,
-              });
-            }
-          }
-          if ((verdict.exceeded || wVerdict.exceeded) && budget.pauseOnBudget) {
-            const until = now + DAY_MS;
-            for (const ref of pool.refs) {
-              if ((pool.state.failedUntil.get(ref) ?? 0) < until) pool.state.failedUntil.set(ref, until);
-            }
-          }
-          // #221: pool running low - webhook while healthy < warnBelowHealthy
-          const warnBelow = runtime.warnBelowHealthy ?? 0;
-          if (warnBelow > 0) {
-            let healthy = 0;
-            for (const ref of pool.refs) {
-              const fu = pool.state.failedUntil.get(ref);
-              if (fu !== undefined && fu > now) continue;
-              const exp = pool.expiresAt?.[ref];
-              if (exp !== undefined && now >= exp) continue;
-              healthy++;
-            }
-            if (healthy < warnBelow && shouldNotifyDaily(lowHealthNotifiedAt, pool.base, now)) {
-              console.warn(`[dsh-key-rotation] ${pool.base}: pool running low - ${healthy}/${pool.refs.length} healthy`);
-              if (runtime.notifyWebhook) {
-                const token = runtime.webhookActionToken ?? '';
-                webhookSender.send(runtime.notifyWebhook, {
-                  title: `Pool running low: ${pool.base}`,
-                  text: `${healthy}/${pool.refs.length} keys healthy (alert below ${warnBelow})`,
-                  provider: pool.base,
-                  kind: 'low-health',
-                  healthy,
-                  total: pool.refs.length,
-                  actionToken: token || undefined,
-                  actions: token ? [{ id: `reset-${pool.base}`, label: 'Reset cooldown' }] : undefined,
-                });
-              }
-            }
-          }
-          // #225: latency SLO - webhook when a key's p95 exceeds the threshold
-          const slo = runtime.latencySloMs ?? 0;
-          if (slo > 0) {
-            for (const ref of pool.refs) {
-              const snap = latencyHistogram.snapshot(ref);
-              if (!snap.p95 || snap.p95 <= slo) continue;
-              if (!shouldNotifyDaily(sloNotifiedAt, pool.base + ':' + ref + ':slo', now)) continue;
-              console.warn(`[dsh-key-rotation] ${pool.base}: ${ref} p95 ${Math.round(snap.p95)}ms > SLO ${slo}ms`);
-              if (runtime.notifyWebhook) {
-                webhookSender.send(runtime.notifyWebhook, {
-                  title: `Latency SLO exceeded: ${pool.base}`,
-                  text: `${ref} p95 ${Math.round(snap.p95)}ms > ${slo}ms (${snap.count} samples)`,
-                  provider: pool.base,
-                  kind: 'latency-slo',
-                  ref,
-                  p95: Math.round(snap.p95),
-                  slo,
-                });
-              }
-            }
-          }
-        }
-      } catch (_) { /* maintenance must never crash the sweep */ }
+      checkBudgetAndHealthAlerts({
+        runtime,
+        poolState,
+        now,
+        expiryNotifiedAt,
+        budgetNotifiedAt,
+        lowHealthNotifiedAt,
+        sloNotifiedAt,
+        latencyHistogram,
+        webhookSender,
+        logger,
+      });
     }, 30000);
     if (typeof id.unref === 'function') id.unref();
     return () => clearInterval(id);
   }, 'dsh-key-rotation: sweep expired cooldowns');
 
-  // ── runtime snapshot: config + llm-pi-ai profile mapping ──
   let cachedRuntime = null;
   let lastConfigRef = null;
   let lastProfilesRef = null;
 
-  getRuntime = buildRuntime;
   function buildRuntime() {
-    const rawConfig = getConfig() ?? {};
-    let currentProfiles = null;
+    const cfg = getConfig() ?? {};
+    let profilesSnapshot = null;
     try {
-      currentProfiles = ctx.get('settings')?.get(PIAI_NS)?.providers ?? null;
+      profilesSnapshot = ctx.get('settings')?.get(PIAI_NS)?.providers ?? null;
     } catch {
-      /* settings not mounted yet — empty mapping */
+      profilesSnapshot = null;
     }
-
-    if (cachedRuntime && lastConfigRef === rawConfig && lastProfilesRef === currentProfiles) {
+    if (cachedRuntime && cfg === lastConfigRef && profilesSnapshot === lastProfilesRef) {
       return cachedRuntime;
     }
+    lastConfigRef = cfg;
+    lastProfilesRef = profilesSnapshot;
 
-    const cfg = Config(rawConfig) ?? {};
-    const switchCodes = new Set(cfg.switchCodes ?? DEFAULT_SWITCH_CODES);
+    const switchCodes = cfg.switchCodes ?? DEFAULT_SWITCH_CODES;
     const cooldownMs = cfg.cooldownMs ?? 60000;
-    const maxCooldownMs = cfg.maxCooldownMs ?? undefined;
+    const maxCooldownMs = cfg.maxCooldownMs;
     const notifyWebhook = cfg.notifyWebhook ?? '';
     const notifyThreshold = cfg.notifyThreshold ?? 3;
+    const concurrencyLimit = cfg.concurrencyLimit ?? 0;
+    const cascade = cfg.cascade ?? [];
+    const quotaResetWindow = cfg.quotaResetWindow ?? { type: 'midnight_utc', hour: 0 };
     const rateLimitThreshold = cfg.rateLimitThreshold ?? 0.1;
     const rpmLimit = cfg.rpmLimit ?? 0;
     const webhookActionToken = cfg.webhookActionToken ?? '';
-    const concurrencyLimit = cfg.concurrencyLimit ?? 0;
-    const cascade = Array.isArray(cfg.cascade) ? cfg.cascade : [];
-    const quotaResetWindow = cfg.quotaResetWindow || null;
 
-    // ref -> pool (every key env of every configured provider)
     const poolByRef = new Map();
-    // provider route (from llm-pi-ai profiles) -> its key pool
     const providerToPool = new Map();
-    // per-model key pools: provider -> Map<model, pool>
     const modelPoolByProvider = new Map();
-    // clone route ids (for the settings dropdown filter)
     const cloneIds = new Set();
 
     const makeState = (base) => {
@@ -580,60 +255,31 @@ export function apply(ctx, config = {}) {
           byModel: new Map(),
           usageDays: new Map(),
           quotaWindows: new Map(),
-          // #260 per-provider circuit breaker (lazy)
-          breaker: null,
-          pointer: 0,
-          lastUsed: undefined,
-          switches: 0,
-          lastReason: undefined,
-          lastSwitchAt: undefined,
-          lastExhaustionAt: undefined,
-          exhaustionCount: 0,
+          costDays: new Map(),
           events: [],
+          pointer: 0,
+          lastUsed: null,
         };
         poolState.set(base, st);
       }
       return st;
     };
-    const parseExpiry = (v) => {
-      if (typeof v === 'number' && v > 0) return v;
-      if (typeof v === 'string' && v.length > 0) { const ts = Date.parse(v); return Number.isNaN(ts) ? undefined : ts; }
-      return undefined;
-    };
-    const buildPool = (base, keys, weights, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard) => {
-      const refs = (keys ?? []).filter((ref) => typeof ref === 'string' && ref.length > 0);
-      if (refs.length === 0) return null;
-      const w = Array.isArray(weights) ? weights : [];
-      const weightedRefs = [];
-      for (let i = 0; i < refs.length; i++) {
-        const ww = typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1;
-        for (let k = 0; k < ww; k++) weightedRefs.push(refs[i]);
-      }
-      const parsedExpiry = {};
-      if (Array.isArray(expiresAt)) {
-        for (let i = 0; i < refs.length; i++) {
-          const exp = parseExpiry(expiresAt[i]);
-          if (exp !== undefined) parsedExpiry[refs[i]] = exp;
-        }
-      }
-      return { base, refs, weights: refs.map((_, i) => (typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1)),
-               weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs,
-               state: makeState(base), cooldownMs: poolCooldown, maxCooldownMs: poolMax, expiresAt: parsedExpiry, rpmLimit, routingStrategy: poolStrategy, proactiveRateLimitGuard: poolGuard };
-    };
+
+    const buildPool = (base, keys, weights, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard) =>
+      buildPoolItem({ base, keys, weights, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, rpmLimit, makeState });
+
     for (const p of cfg.providers ?? []) {
       const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
       const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
-      // base provider pool (fallback)
-      const pool = buildPool(p.provider, p.keys, p.weights, poolCooldown, poolMax);
+      const pool = buildPool(p.provider, p.keys, p.weights, poolCooldown, poolMax, p.expiresAt, p.routingStrategy, p.proactiveRateLimitGuard);
       if (pool) {
         for (const ref of pool.refs) poolByRef.set(ref, pool);
         for (let i = 1; i < pool.refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
       }
-      // per-model pools
       const models = p.models ?? {};
       const byModel = new Map();
       for (const [model, mp] of Object.entries(models)) {
-        const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, poolCooldown, poolMax);
+        const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, poolCooldown, poolMax, undefined, p.routingStrategy, p.proactiveRateLimitGuard);
         if (mpool) {
           byModel.set(model, mpool);
           for (const ref of mpool.refs) poolByRef.set(ref, mpool);
@@ -645,30 +291,14 @@ export function apply(ctx, config = {}) {
     let profiles = {};
     try {
       profiles = ctx.get('settings')?.get(PIAI_NS)?.providers ?? {};
-    } catch {
-      /* settings not mounted yet — empty mapping */
-    }
+    } catch { /* settings not mounted yet */ }
     for (const [provider, profile] of Object.entries(profiles)) {
       if (profile?.apiKeyEnv && poolByRef.has(profile.apiKeyEnv)) {
         providerToPool.set(provider, poolByRef.get(profile.apiKeyEnv));
       }
     }
 
-    // auto-cleanup: remove poolState for providers that are now empty or removed
-    for (const key of [...poolState.keys()]) {
-      if (![...poolByRef.values()].some((p) => p.base === key)) {
-        poolState.delete(key);
-        lowHealthNotifiedAt.delete(key);
-        budgetNotifiedAt.delete(key + ':budget');
-      }
-    }
-    // #192: drop RPM windows for refs that no longer belong to any pool
-    for (const st of poolState.values()) {
-      if (st.rpmWindows) bucketSweep(st.rpmWindows, new Set(poolByRef.keys()));
-    }
-    // #195: provider -> tags (metadata, surfaced in status)
     const providerTags = new Map();
-    // #208: provider -> { costBudgetDaily, costBudgetWeekly, pauseOnBudget }
     const providerBudgets = new Map();
     for (const p of cfg.providers ?? []) {
       if (Array.isArray(p.tags) && p.tags.length > 0) providerTags.set(p.provider, p.tags);
@@ -676,136 +306,137 @@ export function apply(ctx, config = {}) {
       const weekly = typeof p.costBudgetWeekly === 'number' ? p.costBudgetWeekly : 0;
       if (daily > 0 || weekly > 0) providerBudgets.set(p.provider, { costBudgetDaily: daily, costBudgetWeekly: weekly, pauseOnBudget: p.pauseOnBudget ?? false });
     }
-    // #266: orphan clone-route GC — expected clones only for live multi-key providers
+
     const expectedClones = new Set();
     for (const p of cfg.providers ?? []) {
       const n = (p.keys ?? []).filter((k) => typeof k === 'string' && k.length > 0).length;
       for (let i = 1; i < n; i++) expectedClones.add(`${p.provider}-${i + 1}`);
     }
-    // drop breaker entries for removed providers
-    if (moduleBreaker) {
-      for (const key of Object.keys(moduleBreaker.snapshot())) {
-        if (![...providerToPool.keys()].includes(key) && !expectedClones.has(key)) {
-          // keep until TTL; only reset if provider gone from config entirely
-          const still = (cfg.providers ?? []).some((p) => p.provider === key);
-          if (!still) moduleBreaker.reset(key);
-        }
-      }
-    }
-    cachedRuntime = { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, concurrencyLimit, cascade, quotaResetWindow, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, verboseLogging: cfg.verboseLogging ?? false,
+
+    cleanupRemovedProviders({
+      cfg,
+      poolState,
+      poolByRef,
+      providerToPool,
+      expectedClones,
+      moduleBreaker,
+      lowHealthNotifiedAt,
+      budgetNotifiedAt,
+    });
+
+    cachedRuntime = {
+      switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold,
+      concurrencyLimit, cascade, quotaResetWindow, rateLimitThreshold, rpmLimit,
+      webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7,
+      switchNotify: cfg.switchNotify ?? false, verboseLogging: cfg.verboseLogging ?? false,
       proactiveRateLimitGuard: cfg.proactiveRateLimitGuard ?? true,
       selfHealingIntervalMinutes: cfg.selfHealingIntervalMinutes ?? 30,
-      routingStrategy: cfg.routingStrategy ?? 'round-robin', switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds, expectedClones,
+      routingStrategy: cfg.routingStrategy ?? 'round-robin',
+      switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000,
+      warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0,
+      providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider,
+      cloneIds, expectedClones,
       circuitBreakerEnabled: cfg.circuitBreakerEnabled ?? true,
       circuitBreakerThreshold: cfg.circuitBreakerThreshold ?? 5,
       circuitBreakerOpenMs: cfg.circuitBreakerOpenMs ?? 30000,
       circuitBreakerHalfOpenProbes: cfg.circuitBreakerHalfOpenProbes ?? 1,
-      breaker: moduleBreaker,
-      notifyQueue: moduleNotifyQueue,
+      pools: [...new Set(poolByRef.values())],
     };
-    lastConfigRef = rawConfig;
-    lastProfilesRef = currentProfiles;
     return cachedRuntime;
   }
 
-  // ── patch credentials.resolve: pool refs resolve to the next healthy key ──
-  // Round-robin over the pool, skipping keys in cooldown; the request's
-  // provider identity never changes, so pi-ai replay state stays consistent.
+  getRuntime = () => buildRuntime();
+
   ctx.effect(() => {
     const credentials = ctx.get('credentials');
-    if (credentials && typeof credentials.resolve === 'function' && !credentials.__dshKeyRotationPatched) {
+    if (credentials && typeof credentials.resolve === 'function') {
       const original = credentials.resolve.bind(credentials);
-      // Kept for the status route: it must ask about one exact ref instead of
-      // being rotated to a different key by the patch below.
       credentials.__dshKeyRotationOriginalResolve = original;
       credentials.resolve = async (ref) => {
-      const { poolByRef } = buildRuntime();
-      const pool = poolByRef.get(ref);
-      if (!pool) return original(ref);
-      const now = Date.now();
-      const strat = pool.routingStrategy ?? buildRuntime().routingStrategy ?? 'round-robin';
-      let list = pool.weightedRefs ?? pool.refs;
-      if (strat === 'lowest-latency' || strat === 'least-loaded') {
-        list = sortAttemptList(list, strat, { latencyHistogram, concurrencyTracker });
-      }
-      const start = (strat === 'round-robin') ? (pool.state.pointer ?? 0) : 0;
-      for (let i = 0; i < list.length; i++) {
-        const index = (start + i) % list.length;
-        const candidate = list[index];
-        const until = pool.state.failedUntil.get(candidate);
-        if (until !== undefined && until > now) continue;
-        if (pool.expiresAt?.[candidate] !== undefined && now >= pool.expiresAt[candidate]) continue;
-        // #192 RPM token bucket: skip a key that already hit its requests/min cap
-        const rpmLimit = pool.rpmLimit ?? 0;
-        if (rpmLimit > 0) {
-          if (!pool.state.rpmWindows) pool.state.rpmWindows = new Map();
-          if (!bucketAllow(pool.state.rpmWindows, candidate, rpmLimit, now)) {
-            const waitMs = bucketRetryMs(pool.state.rpmWindows, candidate, rpmLimit, now);
-            if ((pool.state.failedUntil.get(candidate) ?? 0) < now + waitMs) {
-              pool.state.failedUntil.set(candidate, now + waitMs);
-            }
-            continue;
-          }
+        const { poolByRef } = buildRuntime();
+        const pool = poolByRef.get(ref);
+        if (!pool) return original(ref);
+        const now = Date.now();
+        const strat = pool.routingStrategy ?? buildRuntime().routingStrategy ?? 'round-robin';
+        let list = pool.weightedRefs ?? pool.refs;
+        if (strat === 'lowest-latency' || strat === 'least-loaded') {
+          list = sortAttemptList(list, strat, { latencyHistogram, concurrencyTracker });
         }
-        if (pool.perHour) {
-          if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
-          let win = pool.state.quotaWindows.get(candidate);
-          if (!win || now - win.start >= 3600000) win = { count: 0, start: now };
-          if (win.count >= pool.perHour) {
-            const until = win.start + 3600000;
-            if ((pool.state.failedUntil.get(candidate) ?? 0) < until) pool.state.failedUntil.set(candidate, until);
-            continue;
+        const start = (strat === 'round-robin') ? (pool.state.pointer ?? 0) : 0;
+        for (let i = 0; i < list.length; i++) {
+          const index = (start + i) % list.length;
+          const candidate = list[index];
+          const until = pool.state.failedUntil.get(candidate);
+          if (until !== undefined && until > now) continue;
+          if (pool.expiresAt?.[candidate] !== undefined && now >= pool.expiresAt[candidate]) continue;
+          const rpmLimit = pool.rpmLimit ?? 0;
+          if (rpmLimit > 0) {
+            if (!pool.state.rpmWindows) pool.state.rpmWindows = new Map();
+            if (!bucketAllow(pool.state.rpmWindows, candidate, rpmLimit, now)) {
+              const waitMs = bucketRetryMs(pool.state.rpmWindows, candidate, rpmLimit, now);
+              if ((pool.state.failedUntil.get(candidate) ?? 0) < now + waitMs) {
+                pool.state.failedUntil.set(candidate, now + waitMs);
+              }
+              continue;
+            }
           }
-        }
-        // Advance pointer immediately so concurrent requests round-robin across distinct healthy keys
-        pool.state.pointer = (index + 1) % list.length;
-        let hit = await original(candidate);
-        if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
-          pool.state.lastUsed = candidate;
-          if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
-          pool.state.lastUsedAt.set(candidate, now);
-          const store = dispatchStorage.getStore();
-          if (store && store.pool === pool) store.pickedRef = candidate;
-          if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
-          pool.state.failedUntil.delete(candidate);
-          if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
-          if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
-          if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
-          pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
           if (pool.perHour) {
             if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
-            let win2 = pool.state.quotaWindows.get(candidate);
-            if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
-            win2.count++;
-            pool.state.quotaWindows.set(candidate, win2);
+            let win = pool.state.quotaWindows.get(candidate);
+            if (!win || now - win.start >= 3600000) win = { count: 0, start: now };
+            if (win.count >= pool.perHour) {
+              const until = win.start + 3600000;
+              if ((pool.state.failedUntil.get(candidate) ?? 0) < until) pool.state.failedUntil.set(candidate, until);
+              continue;
+            }
           }
-          return hit;
-        }
-        // fallback: env var (transient, not persisted)
-        const envVal = envValue(candidate);
-        if (envVal !== undefined) {
-          pool.state.lastUsed = candidate;
-          if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
-          pool.state.lastUsedAt.set(candidate, now);
-          const store = dispatchStorage.getStore();
-          if (store && store.pool === pool) store.pickedRef = candidate;
-          if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
-          pool.state.failedUntil.delete(candidate);
-          if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
-          if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
-          if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
-          pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
-          if (pool.perHour) {
-            if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
-            let win2 = pool.state.quotaWindows.get(candidate);
-            if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
-            win2.count++;
-            pool.state.quotaWindows.set(candidate, win2);
+          pool.state.pointer = (index + 1) % list.length;
+          let hit = await original(candidate);
+          if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
+            pool.state.lastUsed = candidate;
+            if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
+            pool.state.lastUsedAt.set(candidate, now);
+            const store = dispatchStorage.getStore();
+            if (store && store.pool === pool) store.pickedRef = candidate;
+            if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
+            pool.state.failedUntil.delete(candidate);
+            if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
+            if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
+            if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
+            pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
+            if (pool.perHour) {
+              if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+              let win2 = pool.state.quotaWindows.get(candidate);
+              if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
+              win2.count++;
+              pool.state.quotaWindows.set(candidate, win2);
+            }
+            return hit;
+          }
+          const envVal = envValue(candidate);
+          if (envVal !== undefined) {
+            pool.state.lastUsed = candidate;
+            if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
+            pool.state.lastUsedAt.set(candidate, now);
+            const store = dispatchStorage.getStore();
+            if (store && store.pool === pool) store.pickedRef = candidate;
+            if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
+            pool.state.failedUntil.delete(candidate);
+            if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
+            if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
+            if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
+            pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
+            if (pool.perHour) {
+              if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+              let win2 = pool.state.quotaWindows.get(candidate);
+              if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
+              win2.count++;
+              pool.state.quotaWindows.set(candidate, win2);
+            }
+            return { value: envVal, source: 'env' };
           }
-          return { value: envVal, source: 'env' };
         }
-      }
-      return original(ref); // everything cooled/missing — surface the base value
+        return original(ref);
       };
       credentials.__dshKeyRotationPatched = true;
       return () => {
@@ -823,8 +454,6 @@ export function apply(ctx, config = {}) {
     reason: { kind: 'error', failure: Object.freeze({ code, message }) },
   });
 
-  // Latency recording (#6): record successful llm/stream latency per ref.
-  // ponytail: only the true success path (finish-chunk). Failures are not recorded.
   let _rotateStartMs = Date.now();
   function recordLatency(pool, reqStore) {
     try {
@@ -839,8 +468,7 @@ export function apply(ctx, config = {}) {
     } catch (_) { /* ponytail: never crash */ }
   }
 
-  // rotate() factory (#253): dependencies injected for testability
-    const rotate = createRotate({
+  const rotate = createRotate({
     ctx,
     dispatchStorage,
     buildRuntime,
@@ -856,14 +484,10 @@ export function apply(ctx, config = {}) {
     quotaStore,
     circuitBreaker: moduleBreaker,
     now: nowMono,
+    logger,
   });
 
-  // Retry one request on the next pool key when the current key fails with a
-  // switchable error before any content chunk. The provider never changes —
-  // the resolve patch hands out the next key on each dispatch.
-  // Operational routes (status/usage/snapshot/key/import/test/health/webhook) (#253)
   registerOpsRoutes(ctx, {
-
     buildRuntime,
     latencyHistogram,
     lastTestCache,
@@ -875,7 +499,6 @@ export function apply(ctx, config = {}) {
     quotaStore,
   });
 
-  // One-click plugin updater (#307). GET status / POST install exact registry version.
   ctx.effect(() => {
     if (typeof ctx.webServer?.register !== 'function') return;
     registerPluginUpdater(ctx, {
@@ -887,27 +510,21 @@ export function apply(ctx, config = {}) {
 
   ctx.effect(() => ctx.on('llm/stream', (options, next) => {
     if (options[MARKER]) return next();
-    if (rotationDisabled) return next(); // #199: disabled via webhook action
+    if (rotationDisabled) return next();
     const { providerToPool, modelPoolByProvider } = buildRuntime();
-    // #195: exact model pool -> longest model-family prefix -> provider pool
     const pool = selectPool(modelPoolByProvider, providerToPool, options.provider, options.model);
     if (!pool) return next();
     if (buildRuntime()?.verboseLogging) {
-      console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${(pool.weightedRefs ?? pool.refs).length} slots (${pool.refs.length} keys)`);
+      logger.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${(pool.weightedRefs ?? pool.refs).length} slots (${pool.refs.length} keys)`);
     }
     return rotate(options, pool);
   }), 'dsh-key-rotation: llm/stream');
 
-  // Safety net for non-stream requests (agent/request-error waterfall).
-  // llm/stream covers streaming calls; sync calls (embeddings, batch) go
-  // through agent/request and surface errors here. If the error is
-  // switchable, mark the key and ask the agent loop to retry.
   ctx.effect(() => ctx.on('agent/request-error', async (payload, next) => {
     const provider = payload?.provider ?? payload?.failure?.provider ?? '';
     if (!provider) return next();
     const { providerToPool, modelPoolByProvider, switchCodes } = buildRuntime();
     const model = payload?.model || payload?.failure?.model || '';
-    // #195: same tier-aware selection as llm/stream
     const pool = selectPool(modelPoolByProvider, providerToPool, provider, model);
     if (!pool) return next();
     const code = String(payload?.failure?.code ?? payload?.code ?? '');
@@ -925,7 +542,7 @@ export function apply(ctx, config = {}) {
       pool.state.switches = (pool.state.switches ?? 0) + 1;
       pool.state.lastReason = code || 'UNKNOWN';
       pool.state.lastSwitchAt = Date.now();
-      console.warn(`[dsh-key-rotation] ${provider}: key ${String(ref)} failed via agent/request-error (${String(code)} ${String(message).slice(0, 80)}) — retry`);
+      logger.warn(`[dsh-key-rotation] ${provider}: key ${String(ref)} failed via agent/request-error (${String(code)} ${String(message).slice(0, 80)}) — retry`);
     }
     return { kind: 'retry' };
   }), 'dsh-key-rotation: agent/request-error');
@@ -933,7 +550,7 @@ export function apply(ctx, config = {}) {
   ctx.inject(['settings'], (sctx) => {
     const settingsSvc = sctx.get('settings');
     if (!settingsSvc || typeof settingsSvc.register !== 'function') {
-      console.warn('[dsh-key-rotation] settings service unavailable — config scope not registered');
+      logger.warn('[dsh-key-rotation] settings service unavailable — config scope not registered');
       return;
     }
     const scope = settingsSvc.register(NS, Config, { base: config });
@@ -943,33 +560,3 @@ export function apply(ctx, config = {}) {
     });
   });
 }
-
-// Notify on exhaustion: webhook notification.
-// Extracted at module scope for testability. No I/O outside the injected hooks.
-// ponytail: thresholds and URLs are runtime-resolved per call, so changing Config is reflected immediately.
-export function notifyExhaustion(runtime, pool, options, hooks = { webhookSender }) {
-  if (!runtime || !pool) return;
-  const count = pool.state ? (pool.state.exhaustionCount ?? 0) : 0;
-  if (count <= 0) return;
-  try {
-    if (runtime.notifyWebhook && count >= (runtime.notifyThreshold ?? 0)) {
-      const token = runtime.webhookActionToken ?? '';
-      const payload = {
-        title: `Key pool exhausted: ${options.provider}`,
-        text: `${count} exhaustion(s); keys: ${(pool.refs ?? []).join(', ')}`,
-        provider: options.provider,
-        exhaustionCount: count,
-        at: pool.state.lastExhaustionAt,
-        keys: pool.refs,
-        actionToken: token || undefined,
-        actions: token ? [
-          { id: `reset-${options.provider}`, label: 'Reset cooldown' },
-          { id: `pause-${options.provider}`, label: 'Pause 1h' },
-        ] : undefined,
-      };
-      if (hooks.notifyQueue) hooks.notifyQueue.enqueue(runtime.notifyWebhook, payload);
-      else hooks.webhookSender.send(runtime.notifyWebhook, payload);
-    }
-  } catch (_) { /* ponytail: never crash rotate() */ }
-}
-