@goodandready/dsh-key-rotation 0.7.26 → 0.7.28

package/lib/index.js

@@ -1,1402 +1,1658 @@
-// ─────────────────────────────────────────────────────────────────────────────
-//  dsh-key-rotation — per-provider API key rotation for DeepSeek Harness.
-//
-//  Transparent key rotation, Hermes-style: every configured provider has a KEY
-//  POOL (env refs). The plugin patches `ctx.credentials.resolve` so a pool ref
-//  resolves to the next available key (round-robin, skipping keys in cooldown),
-//  and intercepts `llm/stream` to retry a request on the next key when the
-//  current one fails with a switchable error (QUOTA, RATE_LIMIT, ...) before
-//  any content chunk.
-//
-//  The PROVIDER IDENTITY NEVER CHANGES: requests always go out with the
-//  provider the user selected, only the resolved API key differs. This keeps
-//  pi-ai's replay state consistent across multi-call turns and multi-turn
-//  sessions (the earlier clone-provider approach broke it with
-//  INVALID_REPLAY_STATE).
-//
-//  Config is a KEY POOL PER PROVIDER: you list a real provider id and the env
-//  names of its API keys (e.g. <PROVIDER>_API_KEY, <PROVIDER>_API_KEY_2, ...).
-//  When a key's limit is exhausted, the request retries on the next key in the
-//  list; exhausted keys stay in cooldown for cooldownMs. Clone provider routes
-//  (named `<base>-2`, `<base>-3`, ...) are no longer used for rotation but
-//  remain registered, so selecting them also rotates (their apiKeyEnv ref
-//  belongs to the same pool).
-//
-//  The Settings section ("Key Rotation") edits the provider key pools as a
-//  simple list: pick a provider from the dropdown of every provider registered
-//  with ctx.llm (clone routes are hidden from the dropdown), then add/remove key
-//  env names. Plus cooldown and switch codes.
-//
-//  Config (all optional, sane defaults):
-//    switchCodes: string[]   failure codes eligible to switch
-//    cooldownMs: number      key cooldown after a switchable failure
-//    providers:  array       [{ provider, keys: [envName, ...] }]
-// ─────────────────────────────────────────────────────────────────────────────
-import Schema from '@deepseek-ai/schemastery';
-import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter, computeHealthScore, extractRateLimit, isRateLimited } from './pool.js';
-
-export const name = 'dsh-key-rotation';
-export const inject = ['llm', 'webServer', 'settings', 'credentials'];
-export { keyTail, isLoopbackAddress, isTrustedBridgeRequest, DEFAULT_SWITCH_CODES };
-
-/** Settings namespace owning the GUI-editable section (settingsNamespace-valid). */
-const NS = 'dsh-key-rotation';
-/** Config bridge route (GET / PUT / DELETE), loopback-fenced like llm-fallback. */
-const CONFIG_PATH = '/dsh-key-rotation/config';
-const STATUS_PATH = '/dsh-key-rotation/status';
-const KEY_PATH = '/dsh-key-rotation/key';
-const RESET_PATH = '/dsh-key-rotation/reset';
-const IMPORT_PATH = '/dsh-key-rotation/import';
-const HEALTH_PATH = '/dsh-key-rotation/health';
-const TEST_PATH = '/dsh-key-rotation/test';
-const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
-const AGENT_BUDGET_PATH = '/dsh-key-rotation/agent-budget';
-const REGIONS_PATH = '/dsh-key-rotation/regions';
-const INCIDENT_RESET_PATH = '/dsh-key-rotation/incident-reset';
-const SHADOW_PATH = '/dsh-key-rotation/shadow';
-const WEBHOOK_TEST_PATH = '/dsh-key-rotation/webhook-test';
-const TEST_MATRIX_PATH = '/dsh-key-rotation/test-matrix';
-import { LastTestCache, SandboxRunner } from './sandbox.js';
-import { healIdleCooldowns } from './heal.js';
-import { LatencyHistogram } from './histogram.js';
-import { pickCascadeFallback } from './cascade.js';
-import { ConcurrencyTracker } from './concurrency.js';
-import { nextQuotaReset } from './quota-window.js';
-import { CanaryProber } from './canary.js';
-import { QuotaStore } from './quota.js';
-import { AgentBudget } from './agent-budget.js';
-import { RegionMap } from './region.js';
-import { IncidentReporter } from './incident.js';
-import { ShadowRouter } from './shadow.js';
-import { WebhookSender } from './webhook.js';
-
-/** The llm-pi-ai namespace whose provider profiles map providers to pools. */
-const PIAI_NS = 'llm-pi-ai';
-/** Marker on internally re-dispatched requests so the interceptor does not loop. */
-const MARKER = '__dshKeyRotation';
-const MAX_EVENTS = 50;
-function pushEvent(pool, ref, reason, cooldownMs, type) {
-  const ev = { at: Date.now(), ref, reason: String(reason ?? 'UNKNOWN'), cooldownMs, type: type ?? 'fail' };
-  pool.state.events.push(ev);
-  if (pool.state.events.length > MAX_EVENTS) pool.state.events.shift();
-}
-
-
-// Fallback classification by failure message. pi-ai surfaces many real quota /
-// rate-limit / transport failures as thrown exceptions (e.g. the OpenAI SDK
-// throws on HTTP 429 before the stream starts), and dsh-llm then normalizes
-// them to finish chunks with code "UNKNOWN". The message still carries the
-// provider's own text ("429: ...", "Weekly usage limit reached", ...), so we
-// treat pre-content failures whose message matches these patterns as
-// switchable even when the code is not in `switchCodes`.
-
-// Sandbox-test infrastructure (sandbox.js): in-memory cache + runner.
-let lastTestCacheRunnerCtx = null;
-const lastTestCache = new LastTestCache();
-const latencyHistogram = new LatencyHistogram();
-const quotaStore = new QuotaStore();
-const agentBudget = new AgentBudget();
-const regionMap = new RegionMap();
-// IncidentReporter: lazily built when Config provides incidentGitHubToken + incidentGitHubBaseUrl.
-// ponytail: never bake the token into source; repo is hardcoded (this plugin's home repo) but token is per-deploy.
-let incidentReporter = null;
-function ensureIncidentReporter() {
-  if (incidentReporter) return incidentReporter;
-  const cfg = getConfig();
-  const token = cfg ? cfg.incidentGitHubToken : '';
-  const baseUrl = cfg ? cfg.incidentGitHubBaseUrl : '';
-  if (!token || !baseUrl) return null;
-  incidentReporter = new IncidentReporter({ token, baseUrl, repo: 'goodandready/dsh-key-rotation', fetchImpl: globalThis.fetch });
-  return incidentReporter;
-}
-const shadowRouter = new ShadowRouter({ primary: '', secondary: '', percent: 0 });let sandboxRunner = null;
-const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
-const concurrencyTracker = new ConcurrencyTracker();
-let canaryProber = null;
-function ensureSandboxRunner(ctx) {
-  if (sandboxRunner) return sandboxRunner;
-  // provider id -> baseUrl (stripped of trailing /) for fetch /models probe
-  function resolveBaseUrl(provider) {
-    try {
-      const ns = ctx.get(PIAI_NS);
-      const list = ns && (ns.providers || (ns.config && ns.config.providers) || []);
-      if (!Array.isArray(list)) return null;
-      // ponytail: match by id OR name OR alias; pick first hit
-      const hit = list.find((p) => p && (p.id === provider || p.name === provider || (Array.isArray(p.aliases) && p.aliases.includes(provider))));
-      const base = hit && (hit.baseUrl || hit.endpoint || hit.url);
-      return base ? String(base) : null;
-    } catch (e) {
-      return null;
-    }
-  }
-  sandboxRunner = new SandboxRunner({ fetchImpl: globalThis.fetch, resolveBaseUrl });
-  return sandboxRunner;
-}
-async function probeRef(ref, key) {
-  // ref may be like "PROVIDER/KEY_NAME" — for sandbox we only care about the credential ref
-  // (the resolveBaseUrl uses the full provider id; ref can carry any string)
-  const runner = ensureSandboxRunner(lastTestCacheRunnerCtx);
-  const result = await runner.probeModels(ref, key);
-  lastTestCache.set(ref, { ...result, at: Date.now() });
-  return result;
-}
-
-// // Bootstrap key pools. The user configures them in the Settings GUI or via
-// the dsh profile bundle config; the plugin itself ships no provider defaults
-// so it does not bind to any specific installation. Empty array means: until
-// the user adds a pool, no rotation happens, and every provider falls back to
-// its single configured credential exactly as before this plugin was installed.
-const DEFAULT_PROVIDERS = [];
-
-export const Config = Schema.object({
-  switchCodes: Schema.array(Schema.string()).default([...DEFAULT_SWITCH_CODES]),
-  cooldownMs: Schema.number().default(60000),
-  maxCooldownMs: Schema.number(),
-  notifyWebhook: Schema.string().default(''),
-  notifyThreshold: Schema.number().default(3),
-  backupDir: Schema.string().default(''),
-  backupIntervalMs: Schema.number().default(86400000),
-  backupKeep: Schema.number().default(7),
-  rotationScheduleDays: Schema.number().default(0),
-  selfHealCooldown: Schema.boolean().default(true),
-  selfHealIdleMs: Schema.number().default(3600000),
-  latencyEnabled: Schema.boolean().default(true),
-  latencyWindow: Schema.number().default(200),
-  incidentGitHubToken: Schema.string().default(''),
-  incidentGitHubBaseUrl: Schema.string().default(''),
-  incidentThreshold: Schema.number().default(5),
-  concurrencyLimit: Schema.number().default(0),
-  canaryProbingEnabled: Schema.boolean().default(false),
-  canaryIntervalMs: Schema.number().default(30000),
-  cascade: Schema.array(Schema.object({
-    provider: Schema.string().required(),
-    model: Schema.string(),
-  })).default([]),
-  quotaResetWindow: Schema.object({
-    type: Schema.string().default('midnight_utc'),
-    hour: Schema.number().default(0),
-  }),
-  rateLimitThreshold: Schema.number().default(0.1),
-  providers: Schema.array(Schema.object({
-    provider: Schema.string().required(),
-    keys: Schema.array(Schema.string()).default([]),
-    weights: Schema.array(Schema.number()).default([]),
-    expiresAt: Schema.array(Schema.union([Schema.number(), Schema.string()])).default([]),
-    models: Schema.dict(Schema.object({
-      keys: Schema.array(Schema.string()).default([]),
-      weights: Schema.array(Schema.number()).default([]),
-    })).default({}),
-    cooldownMs: Schema.number(),
-    maxCooldownMs: Schema.number(),
-  })).default([...DEFAULT_PROVIDERS]),
-});
-
-// ── config bridge (GET/PUT/DELETE on CONFIG_PATH), mirroring llm-fallback ──
-
-
-function json(res, status, obj) {
-  res.writeHead(status, { 'content-type': 'application/json' });
-  res.end(JSON.stringify(obj));
-}
-
-function readJson(request) {
-  return new Promise((resolve, reject) => {
-    let raw = '';
-    request.on('data', (c) => { raw += c; });
-    request.on('end', () => {
-      try {
-        resolve(JSON.parse(raw || '{}'));
-      } catch (e) {
-        reject(e);
-      }
-    });
-    request.on('error', reject);
-  });
-}
-
-function descriptorOf(ctx, ns) {
-  const settings = ctx.get('settings');
-  if (settings === void 0) return void 0;
-  return settings.describe({ redactSecrets: true }).find((candidate) => candidate.ns === ns);
-}
-
-function viewOf(descriptor, settings) {
-  return {
-    available: true,
-    writable: settings.writable,
-    hasDocument: settings.hasDocument,
-    value: descriptor.value,
-    ...descriptor.base === void 0 ? {} : { base: descriptor.base },
-    ...descriptor.user === void 0 || Object.keys(descriptor.user).length === 0 ? {} : { user: descriptor.user },
-    revision: descriptor.revision,
-  };
-}
-
-async function writeSection(ctx, ns, section, expectedRevision, res) {
-  const settings = ctx.get('settings');
-  if (settings === void 0) {
-    json(res, 503, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: no settings provider is mounted' } });
-    return;
-  }
-  try {
-    await settings.replace(ns, section, expectedRevision);
-  } catch (error) {
-    if (error?.code === 'SETTINGS_CONFLICT') {
-      json(res, 409, { error: { code: 'settings-conflict', message: `dsh-key-rotation: changed elsewhere (expected revision ${String(error.expected)}, current ${String(error.actual)}); reload and retry` } });
-      return;
-    }
-    json(res, 400, { error: { code: 'settings-rejected', message: error instanceof Error ? error.message : String(error) } });
-    return;
-  }
-  const descriptor = descriptorOf(ctx, ns);
-  if (descriptor === void 0) {
-    json(res, 500, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: namespace vanished after write' } });
-    return;
-  }
-  json(res, 200, viewOf(descriptor, { writable: settings.writable, hasDocument: settings.documentPath !== void 0 }));
-}
-
-/** Provider catalog for the GUI dropdown, minus clone routes of configured chains. */
-function providerCatalog(ctx, cloneIds) {
-  const seen = new Set();
-  const out = [];
-  for (const info of ctx.llm.listProviders()) {
-    if (seen.has(info.id) || cloneIds.has(info.id)) continue;
-    seen.add(info.id);
-    out.push({ id: info.id, name: info.name ?? info.id });
-  }
-  return out;
-}
-
-async function handleConfigBridge(ctx, request, res, getCloneIds) {
-  if (!isTrustedBridgeRequest(request)) {
-    res.writeHead(403);
-    res.end();
-    return;
-  }
-  const method = request.method ?? 'GET';
-  if (method === 'GET') {
-    const settings = ctx.get('settings');
-    const descriptor = descriptorOf(ctx, NS);
-    const body = {
-      providers: providerCatalog(ctx, getCloneIds()),
-    };
-    if (descriptor === void 0) {
-      json(res, 200, {
-        ...body,
-        available: false,
-        writable: settings?.writable ?? false,
-        hasDocument: settings?.documentPath !== void 0,
-        value: void 0,
-        revision: 0,
-      });
-      return;
-    }
-    json(res, 200, {
-      ...body,
-      ...viewOf(descriptor, {
-        writable: settings?.writable ?? false,
-        hasDocument: settings?.documentPath !== void 0,
-      }),
-    });
-    return;
-  }
-  if (method === 'PUT' || method === 'DELETE') {
-    let section;
-    let expectedRevision;
-    if (method === 'PUT') {
-      let body;
-      try {
-        body = await readJson(request);
-      } catch (error) {
-        json(res, 400, { error: { code: 'settings-rejected', message: `dsh-key-rotation: invalid request body: ${error instanceof Error ? error.message : String(error)}` } });
-        return;
-      }
-      if (typeof body !== 'object' || body === null || typeof body.section !== 'object' || body.section === null || Array.isArray(body.section)) {
-        json(res, 400, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: PUT requires {"section": {...}}' } });
-        return;
-      }
-      section = body.section;
-      expectedRevision = typeof body.expectedRevision === 'number' ? body.expectedRevision : void 0;
-    } else {
-      section = {};
-    }
-    await writeSection(ctx, NS, section, expectedRevision, res);
-    return;
-  }
-  res.writeHead(405);
-  res.end();
-}
-
-function registerConfigBridge(ctx, getCloneIds) {
-  return ctx.webServer.register({
-    kind: 'exact',
-    path: CONFIG_PATH,
-    handler: (req, res) => void handleConfigBridge(ctx, req, res, getCloneIds),
-  });
-}
-
-// ── plugin ──
-
-export function apply(ctx, config = {}) {
-  // GUI section: defaults -> cordis row config -> saved user section.
-  // (installSettingsSection inlined: no @deepseek-ai/dsh-settings import, so the
-  // profile does not need a second copy of that package.)
-  let getConfig = () => config;
-  registerConfigBridge(ctx, () => buildRuntime().cloneIds);
-  lastTestCacheRunnerCtx = ctx;
-  // Cache should not survive profile restarts (apply is called per reload).
-  // We deliberately do NOT clear on every apply — that would wipe badges when
-  // the user is just typing in the settings card. Re-init only on true reload.
-  ensureSandboxRunner(ctx);
-
-  // Self-healing idle cooldowns: every 60s, lift expired cooldowns for keys
-  // that have been idle for selfHealIdleMs (default 1h). ponytail: small
-  // interval, low cost; skipped when selfHealCooldown is disabled in config.
-  // ponytail: keep handle on the same ctx via closure so buildRuntime() reads
-  // fresh config on every tick. Naive but correct: 60s cadence is cheap.
-  // #196: canary probing before key release from cooldown.
-  // Every canaryIntervalMs, probe refs that are in cooldown and close to expiry.
-  let canaryTimer = null;
-  const startCanary = () => {
-    const cfg = getConfig();
-    if (!cfg || !cfg.canaryProbingEnabled) return;
-    if (canaryTimer) return;
-    canaryTimer = setInterval(() => {
-      try {
-        const c = getConfig();
-        if (!c || !c.canaryProbingEnabled) return;
-        const runner = ensureSandboxRunner();
-        if (!runner) return;
-        if (!canaryProber) {
-          canaryProber = new CanaryProber({ sandboxRunner: runner, intervalMs: c.canaryIntervalMs });
-        }
-        const providers = Array.isArray(c.providers) ? c.providers : [];
-        for (const p of providers) {
-          const pool = buildRuntime().providerToPool.get(p.provider);
-          if (!pool) continue;
-          for (const ref of pool.refs) {
-            const until = pool.state.failedUntil.get(ref) ?? 0;
-            const now = Date.now();
-            // Probe refs in cooldown whose expiry is within canaryIntervalMs of now
-            if (until > now && until - now < (c.canaryIntervalMs ?? 30000)) {
-              canaryProber.probe(ref, ref);
-            }
-          }
-        }
-      } catch (_) { /* ponytail: never crash the timer */ }
-    }, cfg.canaryIntervalMs ?? 30000);
-    if (typeof canaryTimer.unref === 'function') canaryTimer.unref();
-  };
-  startCanary();
-
-  const selfHealTimer = setInterval(() => {
-    const cfg = getConfig();
-    if (!cfg || cfg.selfHealCooldown === false) return;
-    try {
-      const idle = Number.isFinite(cfg.selfHealIdleMs) && cfg.selfHealIdleMs > 0 ? cfg.selfHealIdleMs : 3600000;
-      const providers = Array.isArray(cfg.providers) ? cfg.providers : [];
-      const pools = providers
-        .map((p) => buildRuntime().providerToPool.get(p.provider))
-        .filter(Boolean);
-      healIdleCooldowns(pools, idle);
-    } catch (_) { /* ponytail: never crash the timer */ }
-  }, 60000);
-  if (typeof selfHealTimer.unref === 'function') selfHealTimer.unref();
-
-  // Dashboard widget now lives in client.js (mountDashboard, see issue #152).
-  const DASH_HTML = '';
-    // ── key-pool state, persisted across config reloads ──
-  // base provider -> { failedUntil: Map<ref, epochMs>, pointer: number, lastUsed: ref }
-  const poolState = new Map();
-  // Periodic backup of pools config
-  ctx.effect(() => {
-    const { backupDir, backupIntervalMs, backupKeep } = buildRuntime();
-    if (!backupDir) return;
-    const id = setInterval(() => {
-      try {
-        const fs = require('node:fs');
-        const path = require('node:path');
-        const dir = backupDir;
-        fs.mkdirSync(dir, { recursive: true });
-        const now = new Date();
-        const dateStr = now.toISOString().slice(0,10).replace(/-/g,'');
-        const file = path.join(dir, 'pools-' + dateStr + '.json');
-        const data = JSON.stringify({ backup: now.toISOString(), providers: getConfig()?.providers ?? [] }, null, 2);
-        fs.writeFileSync(file, data, 'utf8');
-        // prune old backups
-        const keep = backupKeep || 7;
-        const files = fs.readdirSync(dir).filter((f) => f.startsWith('pools-') && f.endsWith('.json')).sort();
-        while (files.length > keep) {
-          const old = files.shift();
-          fs.unlinkSync(path.join(dir, old));
-        }
-      } catch (e) {
-        console.warn('[dsh-key-rotation] backup failed:', String(e?.message ?? e));
-      }
-    }, backupIntervalMs || 86400000);
-    return () => clearInterval(id);
-  }, 'dsh-key-rotation: backup pools');
-  // Periodic save of usage/cost stats to file
-  ctx.effect(() => {
-    const { backupDir } = buildRuntime();
-    if (!backupDir) return;
-    try {
-      const fs = require('node:fs');
-      const path = require('node:path');
-      const statsFile = path.join(backupDir, 'stats.json');
-      // Load existing stats at startup
-      try {
-        if (fs.existsSync(statsFile)) {
-          const saved = JSON.parse(fs.readFileSync(statsFile, 'utf8'));
-          for (const st of poolState.values()) {
-            if (saved.usageCounts && st.usageCounts) { for (const [k, v] of Object.entries(saved.usageCounts)) st.usageCounts.set(k, (st.usageCounts.get(k) ?? 0) + v); }
-            if (saved.costPerKey && st.costPerKey) { for (const [k, v] of Object.entries(saved.costPerKey)) st.costPerKey.set(k, (st.costPerKey.get(k) ?? 0) + v); }
-            if (saved.lastUsedAt && st.lastUsedAt) { for (const [k, v] of Object.entries(saved.lastUsedAt)) { if (!st.lastUsedAt.has(k) || v > st.lastUsedAt.get(k)) st.lastUsedAt.set(k, v); } }
-          }
-        }
-      } catch {}
-      // Periodic save
-      const id = setInterval(() => {
-        try {
-          const usageCounts = {}; const costPerKey = {}; const lastUsedAt = {};
-          for (const [base, st] of poolState) {
-            if (st.usageCounts) for (const [k, v] of st.usageCounts) usageCounts[k] = v;
-            if (st.costPerKey) for (const [k, v] of st.costPerKey) costPerKey[k] = v;
-            if (st.lastUsedAt) for (const [k, v] of st.lastUsedAt) lastUsedAt[k] = v;
-          }
-          fs.writeFileSync(statsFile, JSON.stringify({ t: Date.now(), usageCounts, costPerKey, lastUsedAt }), 'utf8');
-        } catch {}
-      }, 60000);
-      return () => clearInterval(id);
-    } catch { return () => {}; }
-  }, 'dsh-key-rotation: persist stats');
-  // Rotation schedule: shift pointer every N days
-  ctx.effect(() => {
-    const { rotationScheduleDays } = buildRuntime();
-    if (!rotationScheduleDays || rotationScheduleDays <= 0) return;
-    const intervalMs = Math.min(rotationScheduleDays * 86400000, 2147483647);
-    const id = setInterval(() => {
-      try {
-        const rt = buildRuntime();
-        let shifted = 0;
-        for (const pool of rt.poolByRef.values()) {
-          if (pool.refs.length < 2) continue;
-          const oldPtr = pool.state.pointer ?? 0;
-          pool.state.pointer = (oldPtr + 1) % pool.refs.length;
-          shifted++;
-          console.warn(`[dsh-key-rotation] ${pool.base}: scheduled rotation -> ${pool.refs[pool.state.pointer]} (day ${rotationScheduleDays})`);
-        }
-        if (shifted) console.warn(`[dsh-key-rotation] schedule: rotated ${shifted} pools`);
-      } catch (e) {
-        console.warn('[dsh-key-rotation] schedule error:', String(e?.message ?? e));
-      }
-    }, intervalMs);
-    return () => clearInterval(id);
-  }, 'dsh-key-rotation: rotation schedule');
-  // Periodic sweep of expired cooldowns — keeps health probe cheap and avoids waiting for next user request
-  ctx.effect(() => {
-    const id = setInterval(() => {
-      const now = Date.now();
-      // probe events for keys whose cooldown just expired
-      for (const st of poolState.values()) {
-        for (const [ref, until] of [...(st.failedUntil?.entries() ?? [])]) {
-          if (until <= now && !st.probedAt?.has(ref)) {
-            st.events.push({ at: until, ref, reason: 'probe', cooldownMs: 0, type: 'probe' });
-            if (st.events.length > 50) st.events.shift();
-            if (!st.probedAt) st.probedAt = new Map();
-            st.probedAt.set(ref, until);
-          }
-        }
-      }
-      const n = sweepExpired(poolState, now);
-      if (n > 0) console.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
-    }, 30000);
-    return () => clearInterval(id);
-  }, 'dsh-key-rotation: sweep expired cooldowns');
-
-  // ── runtime snapshot: config + llm-pi-ai profile mapping ──
-  function buildRuntime() {
-    // Deep-clone before resolving: the frozen snapshot from settings.register
-    // must never be written to by schemastery's dict resolver.
-    const cfg = Config(structuredClone(getConfig() ?? {})) ?? {};
-    const switchCodes = new Set(cfg.switchCodes ?? DEFAULT_SWITCH_CODES);
-    const cooldownMs = cfg.cooldownMs ?? 60000;
-    const maxCooldownMs = cfg.maxCooldownMs ?? undefined;
-    const notifyWebhook = cfg.notifyWebhook ?? '';
-    const notifyThreshold = cfg.notifyThreshold ?? 3;
-    const backupDir = cfg.backupDir ?? '';
-    const backupIntervalMs = cfg.backupIntervalMs ?? 86400000;
-    const backupKeep = cfg.backupKeep ?? 7;
-    const rotationScheduleDays = cfg.rotationScheduleDays ?? 0;
-    const rateLimitThreshold = cfg.rateLimitThreshold ?? 0.1;
-    const incidentThreshold = cfg.incidentThreshold ?? 5;
-    const concurrencyLimit = cfg.concurrencyLimit ?? 0;
-    const canaryProbingEnabled = cfg.canaryProbingEnabled ?? false;
-    const canaryIntervalMs = cfg.canaryIntervalMs ?? 30000;
-    const cascade = Array.isArray(cfg.cascade) ? cfg.cascade : [];
-    const quotaResetWindow = cfg.quotaResetWindow || null;
-
-    // ref -> pool (every key env of every configured provider)
-    const poolByRef = new Map();
-    // provider route (from llm-pi-ai profiles) -> its key pool
-    const providerToPool = new Map();
-    // per-model key pools: provider -> Map<model, pool>
-    const modelPoolByProvider = new Map();
-    // clone route ids (for the settings dropdown filter)
-    const cloneIds = new Set();
-
-    const makeState = (base) => {
-      let st = poolState.get(base);
-      if (!st) {
-        st = {
-          failedUntil: new Map(),
-          failCounts: new Map(),
-          authFailCounts: new Map(),
-          brokenUntil: new Map(),
-          costPerKey: new Map(),
-          lastUsedAt: new Map(),
-          usageCounts: new Map(),
-          byModel: new Map(),
-          usageDays: new Map(),
-          quotaWindows: new Map(),
-          pointer: 0,
-          lastUsed: undefined,
-          switches: 0,
-          lastReason: undefined,
-          lastSwitchAt: undefined,
-          lastExhaustionAt: undefined,
-          exhaustionCount: 0,
-          events: [],
-        };
-        poolState.set(base, st);
-      }
-      return st;
-    };
-    const parseExpiry = (v) => {
-      if (typeof v === 'number' && v > 0) return v;
-      if (typeof v === 'string' && v.length > 0) { const ts = Date.parse(v); return Number.isNaN(ts) ? undefined : ts; }
-      return undefined;
-    };
-    const buildPool = (base, keys, weights, poolCooldown, poolMax, expiresAt) => {
-      const refs = (keys ?? []).filter((ref) => typeof ref === 'string' && ref.length > 0);
-      if (refs.length === 0) return null;
-      const w = Array.isArray(weights) ? weights : [];
-      const weightedRefs = [];
-      for (let i = 0; i < refs.length; i++) {
-        const ww = typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1;
-        for (let k = 0; k < ww; k++) weightedRefs.push(refs[i]);
-      }
-      const parsedExpiry = {};
-      if (Array.isArray(expiresAt)) {
-        for (let i = 0; i < refs.length; i++) {
-          const exp = parseExpiry(expiresAt[i]);
-          if (exp !== undefined) parsedExpiry[refs[i]] = exp;
-        }
-      }
-      return { base, refs, weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs,
-               state: makeState(base), cooldownMs: poolCooldown, maxCooldownMs: poolMax, expiresAt: parsedExpiry };
-    };
-    for (const p of cfg.providers ?? []) {
-      const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
-      const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
-      // base provider pool (fallback)
-      const pool = buildPool(p.provider, p.keys, p.weights, poolCooldown, poolMax);
-      if (pool) {
-        for (const ref of pool.refs) poolByRef.set(ref, pool);
-        for (let i = 1; i < pool.refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
-      }
-      // per-model pools
-      const models = p.models ?? {};
-      const byModel = new Map();
-      for (const [model, mp] of Object.entries(models)) {
-        const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, poolCooldown, poolMax);
-        if (mpool) {
-          byModel.set(model, mpool);
-          for (const ref of mpool.refs) poolByRef.set(ref, mpool);
-        }
-      }
-      if (byModel.size > 0) modelPoolByProvider.set(p.provider, byModel);
-    }
-
-    let profiles = {};
-    try {
-      profiles = ctx.get('settings')?.get(PIAI_NS)?.providers ?? {};
-    } catch {
-      /* settings not mounted yet — empty mapping */
-    }
-    for (const [provider, profile] of Object.entries(profiles)) {
-      if (profile?.apiKeyEnv && poolByRef.has(profile.apiKeyEnv)) {
-        providerToPool.set(provider, poolByRef.get(profile.apiKeyEnv));
-      }
-    }
-
-    // auto-cleanup: remove poolState for providers that are now empty or removed
-    for (const key of [...poolState.keys()]) {
-      if (![...poolByRef.values()].some((p) => p.base === key)) poolState.delete(key);
-    }
-    return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, concurrencyLimit, canaryProbingEnabled, canaryIntervalMs, cascade, quotaResetWindow, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
-  }
-
-  // ── patch credentials.resolve: pool refs resolve to the next healthy key ──
-  // Round-robin over the pool, skipping keys in cooldown; the request's
-  // provider identity never changes, so pi-ai replay state stays consistent.
-  const credentials = ctx.get('credentials');
-  if (credentials && typeof credentials.resolve === 'function' && !credentials.__dshKeyRotationPatched) {
-    const original = credentials.resolve.bind(credentials);
-    // Kept for the status route: it must ask about one exact ref instead of
-    // being rotated to a different key by the patch below.
-    credentials.__dshKeyRotationOriginalResolve = original;
-    credentials.resolve = async (ref) => {
-      const { poolByRef } = buildRuntime();
-      const pool = poolByRef.get(ref);
-      if (!pool) return original(ref);
-      const now = Date.now();
-      const list = pool.weightedRefs ?? pool.refs;
-      const start = pool.state.pointer ?? 0;
-      for (let i = 0; i < list.length; i++) {
-        const index = (start + i) % list.length;
-        const candidate = list[index];
-        const until = pool.state.failedUntil.get(candidate);
-        if (until !== undefined && until > now) continue;
-        if (pool.expiresAt?.[candidate] !== undefined && now >= pool.expiresAt[candidate]) continue;
-        // perHour quota check
-        if (pool.perHour) {
-          if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
-          let win = pool.state.quotaWindows.get(candidate);
-          if (!win || now - win.start >= 3600000) win = { count: 0, start: now };
-          if (win.count >= pool.perHour) {
-            const until = win.start + 3600000;
-            if ((pool.state.failedUntil.get(candidate) ?? 0) < until) pool.state.failedUntil.set(candidate, until);
-            continue;
-          }
-        }
-        let hit = await original(candidate);
-        if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
-          pool.state.pointer = (index + 1) % list.length;
-          pool.state.lastUsed = candidate;
-          if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
-          pool.state.failedUntil.delete(candidate);
-          if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
-          if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
-          if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
-          pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
-          if (pool.perHour) {
-            if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
-            let win2 = pool.state.quotaWindows.get(candidate);
-            if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
-            win2.count++;
-            pool.state.quotaWindows.set(candidate, win2);
-          }
-          return hit;
-        }
-        // fallback: env var (transient, not persisted)
-        const envVal = envValue(candidate);
-        if (envVal !== undefined) {
-          pool.state.pointer = (index + 1) % list.length;
-          pool.state.lastUsed = candidate;
-          if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
-          pool.state.failedUntil.delete(candidate);
-          if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
-          if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
-          if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
-          pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
-          if (pool.perHour) {
-            if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
-            let win2 = pool.state.quotaWindows.get(candidate);
-            if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
-            win2.count++;
-            pool.state.quotaWindows.set(candidate, win2);
-          }
-          return { value: envVal, source: 'env' };
-        }
-      }
-      return original(ref); // everything cooled/missing — surface the base value
-    };
-    credentials.__dshKeyRotationPatched = true;
-  }
-
-  const finishError = (code, message) => ({
-    type: 'finish',
-    reason: { kind: 'error', failure: Object.freeze({ code, message }) },
-  });
-
-  // Latency recording (#6): record successful llm/stream latency per ref.
-  // ponytail: only the true success path (finish-chunk). Failures are not recorded.
-  let _rotateStartMs = Date.now();
-  function recordLatency(pool) {
-    try {
-      const cfg = getConfig();
-      if (!cfg || cfg.latencyEnabled === false) return;
-      const ref = pool && pool.state && pool.state.lastUsed;
-      if (!ref) return;
-      const elapsed = Date.now() - _rotateStartMs;
-      if (!Number.isFinite(elapsed) || elapsed < 0) return;
-      latencyHistogram.record(ref, elapsed);
-    } catch (_) { /* ponytail: never crash */ }
-  }
-
-  // Retry one request on the next pool key when the current key fails with a
-  // switchable error before any content chunk. The provider never changes —
-  // the resolve patch hands out the next key on each dispatch.
-  function rotate(options, pool) {
-    return (async function* () {
-      const { switchCodes, cooldownMs, maxCooldownMs } = buildRuntime();
-      let lastFailure = null;
-      _rotateStartMs = Date.now();
-
-      const runtime0 = buildRuntime();
-      if (runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
-        // #193: prefer least-loaded key within limit
-        const available = (pool.weightedRefs ?? pool.refs).filter((r) => {
-          const fu = pool.state.failedUntil.get(r) ?? 0;
-          if (fu > Date.now()) return false;
-          const exp = pool.expiresAt ? pool.expiresAt[r] : undefined;
-          if (exp !== undefined && Date.now() >= exp) return false;
-          return true;
-        });
-        const preferred = concurrencyTracker.pickLeastLoaded(available);
-        if (preferred && (pool.weightedRefs ?? pool.refs)[0] !== preferred) {
-          // Move preferred to front of the attempt list
-          const list = (pool.weightedRefs ?? pool.refs).slice();
-          const i = list.indexOf(preferred);
-          if (i > 0) { list.splice(i, 1); list.unshift(preferred); }
-          pool.weightedRefs = list;
-        }
-      }
-      for (let attempt = 0; attempt < (pool.weightedRefs ?? pool.refs).length; attempt++) {
-        let yielded = false;
-        let switching = false;
-        let inner;
-        try {
-          // mark the internal dispatch so the interceptor does not re-rotate
-          inner = ctx.llm.stream({ ...options, [MARKER]: true });
-        } catch (e) {
-          if (pool.state.lastUsed) { const _retry = parseRetryAfter(String(e?.message ?? '')); const _base = pool.cooldownMs ?? cooldownMs; const _max = pool.maxCooldownMs ?? maxCooldownMs; const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base; const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max); pushEvent(pool, pool.state.lastUsed, e?.code ?? 'TRANSPORT', _b); const _code = String(e?.code ?? ''); if (_code === 'AUTH' || /auth/i.test(String(e?.message ?? ''))) { const _c = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1; pool.state.authFailCounts.set(pool.state.lastUsed, _c); if (_c >= 3) { pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); } } else { pool.state.authFailCounts.delete(pool.state.lastUsed); } }
-          lastFailure = finishError(e?.code ?? 'TRANSPORT',
-            `dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
-          console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(pool.state.lastUsed ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
-          continue;
-        }
-
-        const _pickedRef = pool.state.lastUsed;
-        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.acquire(_pickedRef);
-        try {
-          for await (const chunk of inner) {
-            // Only actual content deltas lock the stream (no more rotation).
-            // Structural/metadata chunks (block-start/end, usage) do not.
-            if (chunk && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta')) {
-              yielded = true;
-              yield chunk;
-              continue;
-            }
-            if (chunk && chunk.type === 'finish') {
-              const kind = chunk.reason?.kind;
-              const failure = chunk.reason?.failure;
-              const code = failure?.code;
-              const message = failure?.message ?? '';
-              const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-              const switchable = !yielded && kind === 'error' &&
-                (effectiveSwitchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message));
-              if (switchable) {
-                if (pool.state.lastUsed) {
-                  const _retry = parseRetryAfter(message);
-                  const _base = pool.cooldownMs ?? cooldownMs;
-                  const _max = pool.maxCooldownMs ?? maxCooldownMs;
-                  const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base;
-                  const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max);
-                  pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b);
-                  // authFailCounts/brokenUntil: lazy-init if state was created by an older plugin version
-                  if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
-                  if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
-                  const _code2 = String(code ?? '');
-                  if (_code2 === 'AUTH' || /auth/i.test(message)) {
-                    const _c2 = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1;
-                    pool.state.authFailCounts.set(pool.state.lastUsed, _c2);
-                    if (_c2 >= 3) {
-                      pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30);
-                      pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30);
-                    }
-                  } else {
-                    pool.state.authFailCounts.delete(pool.state.lastUsed);
-                  }
-                }
-                pool.state.switches = (pool.state.switches ?? 0) + 1;
-                pool.state.lastReason = String(code ?? 'UNKNOWN');
-                pool.state.lastSwitchAt = Date.now();
-                lastFailure = chunk;
-                console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(pool.state.lastUsed ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) — next key`);
-                switching = true;
-                break;
-              }
-              // cost tracking if provider returns usage.cost
-              if (chunk.usage?.cost != null && pool.state.lastUsed) {
-                const c = Number(chunk.usage.cost);
-                if (!isNaN(c)) {
-                  if (!pool.state.costPerKey) pool.state.costPerKey = new Map();
-                  pool.state.costPerKey.set(pool.state.lastUsed, (pool.state.costPerKey.get(pool.state.lastUsed) ?? 0) + c);
-                }
-              }
-              // Usage by day (#119)
-              if (pool.state.lastUsed) {
-                if (!pool.state.usageDays) pool.state.usageDays = new Map();
-                const day = new Date().toISOString().slice(0, 10);
-                const dayMap = pool.state.usageDays.get(pool.state.lastUsed) || new Map();
-                dayMap.set(day, (dayMap.get(day) ?? 0) + 1);
-                pool.state.usageDays.set(pool.state.lastUsed, dayMap);
-              }
-              // Per-model request detail (#121)
-              if (pool.state.lastUsed && options.model) {
-                if (!pool.state.byModel) pool.state.byModel = new Map();
-                let byRef = pool.state.byModel.get(pool.state.lastUsed);
-                if (!byRef) { byRef = new Map(); pool.state.byModel.set(pool.state.lastUsed, byRef); }
-                byRef.set(options.model, (byRef.get(options.model) ?? 0) + 1);
-              }
-              // Proactive rate-limit (#115): if response headers say this key is near
-              // its quota, cool it down so the NEXT request starts on a different key.
-              // We do NOT re-run this (already successful) request — that would double-send.
-              const rate = extractRateLimit(chunk?.metadata?.headers ?? chunk?.headers);
-              if (rate && pool.state.lastUsed) {
-                const { rateLimitThreshold } = buildRuntime();
-                if (isRateLimited(rate, rateLimitThreshold ?? 0.1)) {
-                  const cool = rate.reset && rate.reset > Date.now() ? (rate.reset - Date.now()) : pool.cooldownMs;
-                  recordFailure(pool, pool.state.lastUsed, Date.now(), cool, pool.maxCooldownMs);
-                  pushEvent(pool, pool.state.lastUsed, 'RATE_LIMIT', cool);
-                  console.warn(`[dsh-key-rotation] ${options.provider}: key ${pool.state.lastUsed} near quota (remaining ${String(rate.remaining)}/${String(rate.limit)}) — next request will rotate`);
-                }
-              }
-              // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
-              if (rate && pool.state.lastUsed && Number.isFinite(rate.remaining)) {
-                quotaStore.set(pool.state.lastUsed, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: Date.now() });
-              }
-              yield chunk;
-              recordLatency(pool);
-              return;
-            }
-            yield chunk;
-          }
-        } catch (e) {
-          if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
-          yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
-          return;
-        }
-
-        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
-        if (switching) continue; // try the next key
-        return; // clean end — served
-      }
-
-      // pool exhausted — all keys cooling or missing
-      pool.state.lastExhaustionAt = Date.now();
-      pool.state.exhaustionCount = (pool.state.exhaustionCount ?? 0) + 1;
-      console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — all ${pool.refs.length} keys cooling`);
-      // notify via extracted helper (see notifyExhaustion above)
-      notifyExhaustion(buildRuntime(), pool, { provider: options.provider });
-
-      // #194: cross-provider cascade failover
-      const runtime = buildRuntime();
-      if (Array.isArray(runtime.cascade) && runtime.cascade.length > 0) {
-        const pools = runtime.providerToPool;
-        const fb = pickCascadeFallback(options.provider, runtime, pools);
-        if (fb && fb.pool && fb.pool !== pool) {
-          console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — cascading to ${fb.provider}`);
-          pool.state.lastReason = 'CASCADE';
-          pool.state.lastSwitchAt = Date.now();
-          // Re-dispatch on the fallback pool (depth-1 via marker check)
-          const innerCascade = rotate({ ...options, provider: fb.provider }, fb.pool);
-          for await (const chunk of innerCascade) {
-            yield chunk;
-          }
-          return;
-        }
-      }
-
-      yield lastFailure ?? finishError('TRANSPORT', 'dsh-key-rotation: all keys failed');
-    })();
-  }
-
-  // ── status route: what the settings card cannot know on its own ──
-  //
-  // Reports, per configured provider, which key is in use, which are cooling
-  // down and until when, whether an env name resolves to a credential at all
-  // (a typo is otherwise silent), and how often rotation has fired.
-  //
-  // Key VALUES never leave the host — only the boolean fact that one exists.
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: STATUS_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'GET') {
-        json(res, 405, { error: { code: 'method', message: 'GET only' } });
-        return;
-      }
-      if (!isTrustedBridgeRequest(req)) {
-        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: status is local-only' } });
-        return;
-      }
-      const { poolByRef } = buildRuntime();
-      const base = ctx.get('credentials');
-      const now = Date.now();
-      const seen = new Set();
-      const providers = [];
-      for (const pool of poolByRef.values()) {
-        if (seen.has(pool.base)) continue;
-        seen.add(pool.base);
-        const keys = [];
-        for (const ref of pool.refs) {
-          let present = false;
-          let tail = '';
-          let source = null;
-          let writable = true;
-          try {
-            // The resolve patch is installed on this same service, so ask for
-            // the exact ref: a pool ref would otherwise round-robin to another
-            // key and report a missing name as present.
-            let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
-            present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
-            if (present) tail = keyTail(hit.value);
-            // fallback: env var bootstrapping (issue #7)
-            if (!present) {
-              const ev = envValue(ref);
-              if (ev !== undefined) { present = true; tail = keyTail(ev); source = 'env'; writable = false; }
-            }
-          } catch {
-            present = false;
-          }
-          try {
-            const described = await base?.describe?.(ref);
-            source = described?.source ?? null;
-            writable = described?.writable !== false;
-          } catch {
-            /* describe is optional — the card falls back to editable */
-          }
-          const until = pool.state.failedUntil.get(ref);
-          keys.push({
-            ref,
-            present,
-            tail,
-            source,
-            writable,
-            active: pool.state.lastUsed === ref,
-            cooldownMsLeft: until !== undefined && until > now ? until - now : 0,
-            usage: pool.state.usageCounts?.get(ref) ?? 0,
-            byModel: pool.state.byModel?.get(ref) ? Object.fromEntries(pool.state.byModel.get(ref)) : {},
-            usageDays: pool.state.usageDays?.get(ref) ? Object.fromEntries(pool.state.usageDays.get(ref)) : {},
-            cost: pool.state.costPerKey?.get(ref) ?? 0,
-            lastUsedAt: pool.state.lastUsedAt?.get(ref) ?? null,
-            expiresAt: pool.expiresAt?.[ref] ?? null,
-            expired: pool.expiresAt?.[ref] !== undefined && now >= pool.expiresAt[ref],
-            broken: pool.state.brokenUntil?.has(ref) ?? false,
-          });
-        }
-        providers.push({
-          provider: pool.base,
-          keys,
-          switches: pool.state.switches ?? 0,
-          lastReason: pool.state.lastReason ?? null,
-          lastSwitchAt: pool.state.lastSwitchAt ?? null,
-          lastExhaustionAt: pool.state.lastExhaustionAt ?? null,
-          exhaustionCount: pool.state.exhaustionCount ?? 0,
-          totalUsage: [...(pool.state.usageCounts?.values() ?? [])].reduce((a, b) => a + b, 0),
-          events: (pool.state.events ?? []).slice(-50),
-          healthScore: computeHealthScore(pool.state),
-        });
-      }
-      json(res, 200, { providers });
-    },
-  }), 'dsh-key-rotation: status route');
-
-  // ── key route: store a key value without leaving the rotation card ──
-  //
-  // Adding a key used to mean two screens: create the credential elsewhere,
-  // then type its env name here. The value is write-only from the browser —
-  // it is never sent back, only its last few characters are (see the status
-  // route) — and the route is loopback- and same-origin-gated like the config
-  // bridge next to it.
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: KEY_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'PUT' && req.method !== 'DELETE') {
-        json(res, 405, { error: { code: 'method', message: 'PUT or DELETE only' } });
-        return;
-      }
-      if (!isTrustedBridgeRequest(req)) {
-        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: keys are local-only' } });
-        return;
-      }
-      const credentialsService = ctx.get('credentials');
-      if (!credentialsService || typeof credentialsService.set !== 'function') {
-        json(res, 503, { error: { code: 'no-credentials', message: 'dsh-key-rotation: no credentials service is mounted' } });
-        return;
-      }
-      let body;
-      try {
-        body = await readJson(req);
-      } catch (error) {
-        json(res, 400, { error: { code: 'bad-request', message: String(error?.message ?? error) } });
-        return;
-      }
-      const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
-      if (!isValidRef(ref)) {
-        json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } });
-        return;
-      }
-      try {
-        if (req.method === 'DELETE') {
-          await credentialsService.unset(ref);
-          json(res, 200, { ok: true, ref });
-          return;
-        }
-        const value = typeof body?.value === 'string' ? body.value.trim() : '';
-        if (value.length === 0) {
-          json(res, 400, { error: { code: 'empty-value', message: 'dsh-key-rotation: an empty key cannot be stored' } });
-          return;
-        }
-        await credentialsService.set(ref, value);
-        json(res, 200, { ok: true, ref, tail: keyTail(value) });
-      } catch (error) {
-        // A ref supplied by the launching environment is read-only, and the
-        // service says so in plain words — pass that through to the card.
-        json(res, 409, { error: { code: 'write-rejected', message: String(error?.message ?? error) } });
-      }
-    },
-  }), 'dsh-key-rotation: key route');
-
-  // ── reset route: clear cooldown for a provider (or a single ref) ──
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: RESET_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'POST') {
-        json(res, 405, { error: { code: 'method', message: 'POST only' } });
-        return;
-      }
-      if (!isTrustedBridgeRequest(req)) {
-        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: reset is local-only' } });
-        return;
-      }
-      let body;
-      try { body = await readJson(req); } catch (e) {
-        json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } });
-        return;
-      }
-      const provider = typeof body?.provider === 'string' ? body.provider.trim() : '';
-      const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
-      if (provider) {
-        const st = poolState.get(provider);
-        if (!st) { json(res, 404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }); return; }
-        const cleared = st.failedUntil.size;
-        st.failedUntil.clear();
-        st.failCounts?.clear();
-        st.authFailCounts?.clear();
-        st.brokenUntil?.clear();
-        st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
-        json(res, 200, { ok: true, provider, cleared });
-        return;
-      }
-      if (ref) {
-        let found = false;
-        for (const st of poolState.values()) {
-          if (st.failedUntil.has(ref) || st.failCounts?.has(ref)) {
-            st.failedUntil.delete(ref);
-            st.failCounts?.delete(ref);
-            st.authFailCounts?.delete(ref);
-            st.brokenUntil?.delete(ref);
-            if (st.lastUsed === ref) st.lastUsed = undefined;
-            found = true; break;
-          }
-        }
-        // idempotent: even if ref was not cooling, report ok if it looks like a valid ref name
-        if (!found && !isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
-        json(res, 200, { ok: true, ref });
-        return;
-      }
-      json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: POST requires {"provider": "..."} or {"ref": "..."}' } });
-    },
-  }), 'dsh-key-rotation: reset route');
-
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: IMPORT_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: import is local-only' } }); return; }
-      let body; try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
-      const url = typeof body?.url === 'string' ? body.url.trim() : '';
-      if (!url || !url.startsWith('https://')) { json(res, 400, { error: { code: 'bad-url', message: 'dsh-key-rotation: only HTTPS URLs are allowed' } }); return; }
-      try {
-        const resp = await fetch(url);
-        if (!resp.ok) { json(res, 400, { error: { code: 'fetch-failed', message: 'dsh-key-rotation: fetch returned ' + resp.status } }); return; }
-        const data = await resp.json();
-        if (!Array.isArray(data)) { json(res, 400, { error: { code: 'bad-format', message: 'dsh-key-rotation: expected JSON array of providers' } }); return; }
-        const settings = ctx.get('settings');
-        if (!settings) { json(res, 503, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: no settings provider' } }); return; }
-        const desc = settings.describe({ redactSecrets: true }).find((c) => c.ns === NS);
-        const cur = desc?.value?.providers ?? [];
-        const merged = new Map();
-        for (const p of cur) if (p && p.provider) merged.set(p.provider, p);
-        for (const p of data) if (p && p.provider && typeof p.provider === 'string') merged.set(p.provider, p);
-        const mergedArr = [...merged.values()];
-        await settings.replace(NS, { ...(desc?.value ?? {}), providers: mergedArr }, desc?.revision);
-        json(res, 200, { ok: true, providersImported: data.length, total: mergedArr.length });
-      } catch (e) { json(res, 400, { error: { code: 'import-failed', message: String(e?.message ?? e) } }); }
-    },
-  }), 'dsh-key-rotation: import route');
-
-  // Health for external panels (Beszel/Uptime)
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: HEALTH_PATH,
-    handler: async (req, res) => {
-      if (!isTrustedBridgeRequest(req) && req.socket?.remoteAddress !== '127.0.0.1' && req.socket?.remoteAddress !== '::1') { } // allow same-origin already checked
-      if (!isTrustedBridgeRequest(req)) {
-        // also allow plain loopback without Origin
-        if (!isLoopbackAddress(req.socket?.remoteAddress)) { res.writeHead(403); res.end(); return; }
-        if (req.headers['sec-fetch-site'] === 'cross-site') { res.writeHead(403); res.end(); return; }
-      }
-      if (req.method !== 'GET') { json(res, 405, { error: { code: 'method', message: 'GET only' } }); return; }
-      const now = Date.now();
-      const pools = {};
-      let exhaustedAny = false;
-      const { poolByRef: pr } = buildRuntime();
-      const seenH = new Set();
-      for (const pool of pr.values()) {
-        if (seenH.has(pool.base)) continue;
-        seenH.add(pool.base);
-        let healthy = 0;
-        for (const ref of pool.refs) {
-          const until = pool.state.failedUntil.get(ref);
-          if (until !== undefined && until > now) continue;
-          const exp = pool.expiresAt?.[ref];
-          if (exp !== undefined && now >= exp) continue;
-          healthy++;
-        }
-        const total = pool.refs.length;
-        const exhausted = healthy === 0 && total > 0;
-        if (exhausted) exhaustedAny = true;
-        pools[pool.base] = { healthy, total, exhausted, healthScore: computeHealthScore(pool.state) };
-      }
-      json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: quotaStore.snapshot() });
-    },
-  }), 'dsh-key-rotation: health');
-
-  // ── test route: dry-run a single key without rotation ──
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: TEST_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: test is local-only' } }); return; }
-      let body; try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
-      const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
-      if (!isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
-      // Optional value for pre-save validation (issue #118)
-      const testValue = typeof body?.value === 'string' && body.value.length > 0 ? body.value : undefined;
-      const probe = body?.probe === 'models' || body?.probe === 'chat' ? body.probe : undefined;
-      const base = ctx.get('credentials');
-      try {
-        let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
-        let present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
-        const effectiveValue = testValue || hit?.value;
-        const valid = present ? Boolean(effectiveValue && typeof effectiveValue === 'string' && effectiveValue.length > 0) : Boolean(testValue);
-        const tail = valid ? keyTail(effectiveValue) : '';
-        let source = null;
-        try { const d = await base?.describe?.(ref); source = d?.source ?? null; } catch {}
-        if (!present && !testValue) { json(res, 200, { ok: false, ref, code: 'no-credential', message: 'no such credential' }); return; }
-        if (!present && testValue) { source = 'pre-save'; }
-        else if (!present) {
-          const ev = envValue(ref);
-          if (ev !== undefined) { present = true; json(res, 200, { ok: true, ref, tail: keyTail(ev), source: 'env' }); return; }
-        }
-        // sandbox probe (models is free; chat is hook-only, see sandbox.js)
-        if (probe) {
-          const keyForProbe = effectiveValue;
-          const runner = ensureSandboxRunner(ctx);
-          const result = probe === 'chat' ? await runner.probeChat(ref, keyForProbe) : await runner.probeModels(ref, keyForProbe);
-          const cached = { ...result, at: Date.now() };
-          lastTestCache.set(ref, cached);
-          json(res, 200, { ok: cached.ok, ref, tail, source, probe, code: cached.code, latencyMs: cached.latencyMs, modelsCount: cached.modelsCount });
-          return;
-        }
-        json(res, 200, { ok: true, ref, tail, source });
-      } catch (e) {
-        json(res, 200, { ok: false, ref, code: 'error', message: String(e?.message ?? e) });
-      }
-    },
-  }), 'dsh-key-rotation: test route');
-
-  // Intercept the llm/stream waterfall: rotate any request whose provider maps
-  // to a configured key pool; pass everything else (and internal dispatches)
-  // straight through.
-  // Read-only cache snapshot for clients (badge polling).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: SANDBOX_CACHE_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: cache is local-only' } }); return; }
-      json(res, 200, lastTestCache.snapshot());
-    },
-  }), 'dsh-key-rotation: sandbox cache');
-
-  // Auto-incident reset (#8).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: INCIDENT_RESET_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: incident-reset is local-only' } }); return; }
-      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-      readJson(req).then((body) => {
-        const provider = typeof body?.provider === 'string' ? body.provider : '';
-        if (provider) incidentReporter.resetCooldown(provider);
-        else incidentReporter.resetCooldown();
-        json(res, 200, { ok: true, reset: provider || 'all' });
-      }).catch((e) => json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }));
-    },
-  }), 'dsh-key-rotation: incident-reset');
-
-  // #198: 1-click Health Matrix — parallel probe of all configured keys.
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: TEST_MATRIX_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: matrix is local-only' } }); return; }
-      const cfg = getConfig();
-      const runner = ensureSandboxRunner();
-      if (!runner) { json(res, 500, { error: { code: 'no-runner', message: 'sandbox runner unavailable' } }); return; }
-      const providers = Array.isArray(cfg?.providers) ? cfg.providers : [];
-      const jobs = [];
-      for (const p of providers) {
-        for (const ref of (p.keys ?? [])) {
-          if (typeof ref !== 'string' || !ref) continue;
-          jobs.push((async () => {
-            try {
-              const probeResult = await runner.probeModels(ref, ref);
-              return { provider: p.provider, ref, ok: probeResult.ok, code: probeResult.code, latencyMs: probeResult.latencyMs, modelsCount: probeResult.modelsCount ?? 0 };
-            } catch (e) {
-              return { provider: p.provider, ref, ok: false, code: 'error', latencyMs: 0, modelsCount: 0 };
-            }
-          })());
-        }
-      }
-      const results = await Promise.all(jobs);
-      json(res, 200, { at: Date.now(), total: results.length, ok: results.filter(r => r.ok).length, results });
-    },
-  }), 'dsh-key-rotation: test-matrix');
-
-  // Webhook test endpoint (#10): dry-run that validates webhookSender setup.
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: WEBHOOK_TEST_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: webhook-test is local-only' } }); return; }
-      json(res, 200, { ok: true, snapshot: webhookSender.snapshot() });
-    },
-  }), 'dsh-key-rotation: webhook-test');
-
-  // Shadow A/B sampling snapshot (#9).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: SHADOW_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: shadow is local-only' } }); return; }
-      json(res, 200, shadowRouter.snapshot());
-    },
-  }), 'dsh-key-rotation: shadow');
-
-  // Region tags + failover chain (#4).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: REGIONS_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: regions is local-only' } }); return; }
-      const body = regionMap.snapshot();
-      // Add pickFallback hints per provider for inspection.
-      const out = {};
-      for (const p of Object.keys(body)) out[p] = { region: body[p], fallback: regionMap.pickFallback(p) };
-      json(res, 200, out);
-    },
-  }), 'dsh-key-rotation: regions');
-
-  // Per-agent rate budget snapshot (#3).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: AGENT_BUDGET_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: agent-budget is local-only' } }); return; }
-      json(res, 200, { enabled: agentBudget.isEnabled(), agents: agentBudget.snapshot() });
-    },
-  }), 'dsh-key-rotation: agent-budget');
-
-  ctx.on('llm/stream', (options, next) => {
-    if (options[MARKER]) return next();
-    const { providerToPool, modelPoolByProvider } = buildRuntime();
-    const byModel = modelPoolByProvider.get(options.provider);
-    const pool = (byModel && byModel.get(options.model)) || providerToPool.get(options.provider);
-    if (!pool) return next();
-    console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${(pool.weightedRefs ?? pool.refs).length} slots (${pool.refs.length} keys)`);
-    return rotate(options, pool);
-  });
-
-  // Safety net for non-stream requests (agent/request-error waterfall).
-  // llm/stream covers streaming calls; sync calls (embeddings, batch) go
-  // through agent/request and surface errors here. If the error is
-  // switchable, mark the key and ask the agent loop to retry.
-  ctx.on('agent/request-error', async (payload, next) => {
-    const provider = payload?.provider ?? payload?.failure?.provider ?? '';
-    if (!provider) return next();
-    const { providerToPool, modelPoolByProvider, switchCodes } = buildRuntime();
-    const model = payload?.model || payload?.failure?.model || '';
-    const byModel = modelPoolByProvider.get(provider);
-    const pool = (byModel && byModel.get(model)) || providerToPool.get(provider);
-    if (!pool) return next();
-    const code = String(payload?.failure?.code ?? payload?.code ?? '');
-    const message = String(payload?.failure?.message ?? payload?.message ?? '');
-    const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-    const switchable = effectiveSwitchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message);
-    if (!switchable) return next();
-    const ref = pool.state.lastUsed;
-    if (ref) {
-      const backoff = recordFailure(pool, ref, Date.now(), pool.cooldownMs ?? 60000);
-      pushEvent(pool, ref, code || 'UNKNOWN', backoff);
-      pool.state.switches = (pool.state.switches ?? 0) + 1;
-      pool.state.lastReason = code || 'UNKNOWN';
-      pool.state.lastSwitchAt = Date.now();
-      console.warn(`[dsh-key-rotation] ${provider}: key ${String(ref)} failed via agent/request-error (${String(code)} ${String(message).slice(0, 80)}) — retry`);
-    }
-    return { kind: 'retry' };
-  });
-
-  ctx.inject(['settings'], (sctx) => {
-    const scope = sctx.settings.register(NS, Config, { base: config });
-    getConfig = () => scope.get() ?? config;
-    sctx.effect(() => () => {
-      getConfig = () => config;
-    });
-  });
-}
-
-// Notify on exhaustion: webhook + (optional) GitHub incident.
-// Extracted at module scope for testability. No I/O outside the injected hooks.
-// ponytail: thresholds and URLs are runtime-resolved per call, so changing Config is reflected immediately.
-export function notifyExhaustion(runtime, pool, options, hooks = { webhookSender, ensureIncidentReporter }) {
-  if (!runtime || !pool) return;
-  const count = pool.state ? (pool.state.exhaustionCount ?? 0) : 0;
-  if (count <= 0) return;
-  try {
-    if (runtime.notifyWebhook && count >= (runtime.notifyThreshold ?? 0)) {
-      hooks.webhookSender.send(runtime.notifyWebhook, {
-        provider: options.provider,
-        exhaustionCount: count,
-        at: pool.state.lastExhaustionAt,
-        keys: pool.refs,
-      });
-    }
-    if (runtime.incidentThreshold && count >= runtime.incidentThreshold) {
-      const reporter = hooks.ensureIncidentReporter();
-      if (reporter) reporter.open(options.provider, pool.state.lastExhaustionAt);
-    }
-  } catch (_) { /* ponytail: never crash rotate() */ }
-}
-
+// ─────────────────────────────────────────────────────────────────────────────
+//  dsh-key-rotation — per-provider API key rotation for DeepSeek Harness.
+//
+//  Transparent key rotation, Hermes-style: every configured provider has a KEY
+//  POOL (env refs). The plugin patches `ctx.credentials.resolve` so a pool ref
+//  resolves to the next available key (round-robin, skipping keys in cooldown),
+//  and intercepts `llm/stream` to retry a request on the next key when the
+//  current one fails with a switchable error (QUOTA, RATE_LIMIT, ...) before
+//  any content chunk.
+//
+//  The PROVIDER IDENTITY NEVER CHANGES: requests always go out with the
+//  provider the user selected, only the resolved API key differs. This keeps
+//  pi-ai's replay state consistent across multi-call turns and multi-turn
+//  sessions (the earlier clone-provider approach broke it with
+//  INVALID_REPLAY_STATE).
+//
+//  Config is a KEY POOL PER PROVIDER: you list a real provider id and the env
+//  names of its API keys (e.g. <PROVIDER>_API_KEY, <PROVIDER>_API_KEY_2, ...).
+//  When a key's limit is exhausted, the request retries on the next key in the
+//  list; exhausted keys stay in cooldown for cooldownMs. Clone provider routes
+//  (named `<base>-2`, `<base>-3`, ...) are no longer used for rotation but
+//  remain registered, so selecting them also rotates (their apiKeyEnv ref
+//  belongs to the same pool).
+//
+//  The Settings section ("Key Rotation") edits the provider key pools as a
+//  simple list: pick a provider from the dropdown of every provider registered
+//  with ctx.llm (clone routes are hidden from the dropdown), then add/remove key
+//  env names. Plus cooldown and switch codes.
+//
+//  Config (all optional, sane defaults):
+//    switchCodes: string[]   failure codes eligible to switch
+//    cooldownMs: number      key cooldown after a switchable failure
+//    providers:  array       [{ provider, keys: [envName, ...] }]
+// ─────────────────────────────────────────────────────────────────────────────
+import Schema from '@deepseek-ai/schemastery';
+import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter, computeHealthScore, extractRateLimit, isRateLimited, selectPool } from './pool.js';
+
+export const name = 'dsh-key-rotation';
+export const inject = ['llm', 'webServer', 'settings', 'credentials'];
+export { keyTail, isLoopbackAddress, isTrustedBridgeRequest, DEFAULT_SWITCH_CODES };
+
+/** Settings namespace owning the GUI-editable section (settingsNamespace-valid). */
+const NS = 'dsh-key-rotation';
+/** Config bridge route (GET / PUT / DELETE), loopback-fenced like llm-fallback. */
+const CONFIG_PATH = '/dsh-key-rotation/config';
+const STATUS_PATH = '/dsh-key-rotation/status';
+const KEY_PATH = '/dsh-key-rotation/key';
+const RESET_PATH = '/dsh-key-rotation/reset';
+const IMPORT_PATH = '/dsh-key-rotation/import';
+const HEALTH_PATH = '/dsh-key-rotation/health';
+const USAGE_PATH = '/dsh-key-rotation/usage';
+const TEST_PATH = '/dsh-key-rotation/test';
+const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
+const AGENT_BUDGET_PATH = '/dsh-key-rotation/agent-budget';
+const REGIONS_PATH = '/dsh-key-rotation/regions';
+const INCIDENT_RESET_PATH = '/dsh-key-rotation/incident-reset';
+const SHADOW_PATH = '/dsh-key-rotation/shadow';
+const WEBHOOK_TEST_PATH = '/dsh-key-rotation/webhook-test';
+const TEST_MATRIX_PATH = '/dsh-key-rotation/test-matrix';
+import { LastTestCache, SandboxRunner } from './sandbox.js';
+import { healIdleCooldowns } from './heal.js';
+import { LatencyHistogram } from './histogram.js';
+import { pickCascadeFallback } from './cascade.js';
+import { ConcurrencyTracker } from './concurrency.js';
+import { nextQuotaReset } from './quota-window.js';
+import { CanaryProber } from './canary.js';
+import { QuotaStore } from './quota.js';
+import { AgentBudget } from './agent-budget.js';
+import { RegionMap } from './region.js';
+import { IncidentReporter } from './incident.js';
+import { ShadowRouter } from './shadow.js';
+import { WebhookSender } from './webhook.js';
+import { bucketAllow, bucketRetryMs, bucketSweep, bucketInfo } from './bucket.js';
+import { expiringSoon, shouldNotifyDaily, costForDay, budgetVerdict, costForWeek } from './maintenance.js';
+import { usageRows, usageCsv } from './usage-report.js';
+import { findSecrets, looksLikeApiSecret } from './keycheck.js';
+
+/** The llm-pi-ai namespace whose provider profiles map providers to pools. */
+const PIAI_NS = 'llm-pi-ai';
+/** Marker on internally re-dispatched requests so the interceptor does not loop. */
+const MARKER = '__dshKeyRotation';
+/** #199: set true via webhook action; checked in the llm/stream interceptor. */
+let rotationDisabled = false;
+// #207/#208 dedupe maps: one notification per key/window per day.
+const expiryNotifiedAt = new Map();
+const budgetNotifiedAt = new Map();
+const DAY_MS = 86400000;
+const MAX_EVENTS = 50;
+function pushEvent(pool, ref, reason, cooldownMs, type) {
+  const ev = { at: Date.now(), ref, reason: String(reason ?? 'UNKNOWN'), cooldownMs, type: type ?? 'fail' };
+  pool.state.events.push(ev);
+  if (pool.state.events.length > MAX_EVENTS) pool.state.events.shift();
+}
+
+
+// Fallback classification by failure message. pi-ai surfaces many real quota /
+// rate-limit / transport failures as thrown exceptions (e.g. the OpenAI SDK
+// throws on HTTP 429 before the stream starts), and dsh-llm then normalizes
+// them to finish chunks with code "UNKNOWN". The message still carries the
+// provider's own text ("429: ...", "Weekly usage limit reached", ...), so we
+// treat pre-content failures whose message matches these patterns as
+// switchable even when the code is not in `switchCodes`.
+
+// Sandbox-test infrastructure (sandbox.js): in-memory cache + runner.
+let lastTestCacheRunnerCtx = null;
+const lastTestCache = new LastTestCache();
+const latencyHistogram = new LatencyHistogram();
+const quotaStore = new QuotaStore();
+const agentBudget = new AgentBudget();
+const regionMap = new RegionMap();
+// IncidentReporter: lazily built when Config provides incidentGitHubToken + incidentGitHubBaseUrl.
+// ponytail: never bake the token into source; repo is hardcoded (this plugin's home repo) but token is per-deploy.
+let incidentReporter = null;
+function ensureIncidentReporter() {
+  if (incidentReporter) return incidentReporter;
+  const cfg = getConfig();
+  const token = cfg ? cfg.incidentGitHubToken : '';
+  const baseUrl = cfg ? cfg.incidentGitHubBaseUrl : '';
+  if (!token || !baseUrl) return null;
+  incidentReporter = new IncidentReporter({ token, baseUrl, repo: 'goodandready/dsh-key-rotation', fetchImpl: globalThis.fetch });
+  return incidentReporter;
+}
+const shadowRouter = new ShadowRouter({ primary: '', secondary: '', percent: 0 });let sandboxRunner = null;
+const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
+const concurrencyTracker = new ConcurrencyTracker();
+let canaryProber = null;
+function ensureSandboxRunner(ctx) {
+  if (sandboxRunner) return sandboxRunner;
+  // provider id -> baseUrl (stripped of trailing /) for fetch /models probe
+  function resolveBaseUrl(provider) {
+    try {
+      const ns = ctx.get(PIAI_NS);
+      const list = ns && (ns.providers || (ns.config && ns.config.providers) || []);
+      if (!Array.isArray(list)) return null;
+      // ponytail: match by id OR name OR alias; pick first hit
+      const hit = list.find((p) => p && (p.id === provider || p.name === provider || (Array.isArray(p.aliases) && p.aliases.includes(provider))));
+      const base = hit && (hit.baseUrl || hit.endpoint || hit.url);
+      return base ? String(base) : null;
+    } catch (e) {
+      return null;
+    }
+  }
+  sandboxRunner = new SandboxRunner({ fetchImpl: globalThis.fetch, resolveBaseUrl });
+  return sandboxRunner;
+}
+async function probeRef(ref, key) {
+  // ref may be like "PROVIDER/KEY_NAME" — for sandbox we only care about the credential ref
+  // (the resolveBaseUrl uses the full provider id; ref can carry any string)
+  const runner = ensureSandboxRunner(lastTestCacheRunnerCtx);
+  const result = await runner.probeModels(ref, key);
+  lastTestCache.set(ref, { ...result, at: Date.now() });
+  return result;
+}
+
+// // Bootstrap key pools. The user configures them in the Settings GUI or via
+// the dsh profile bundle config; the plugin itself ships no provider defaults
+// so it does not bind to any specific installation. Empty array means: until
+// the user adds a pool, no rotation happens, and every provider falls back to
+// its single configured credential exactly as before this plugin was installed.
+const DEFAULT_PROVIDERS = [];
+
+export const Config = Schema.object({
+  switchCodes: Schema.array(Schema.string()).default([...DEFAULT_SWITCH_CODES]),
+  cooldownMs: Schema.number().default(60000),
+  maxCooldownMs: Schema.number(),
+  notifyWebhook: Schema.string().default(''),
+  notifyThreshold: Schema.number().default(3),
+  backupDir: Schema.string().default(''),
+  backupIntervalMs: Schema.number().default(86400000),
+  backupKeep: Schema.number().default(7),
+  rotationScheduleDays: Schema.number().default(0),
+  selfHealCooldown: Schema.boolean().default(true),
+  selfHealIdleMs: Schema.number().default(3600000),
+  latencyEnabled: Schema.boolean().default(true),
+  latencyWindow: Schema.number().default(200),
+  incidentGitHubToken: Schema.string().default(''),
+  incidentGitHubBaseUrl: Schema.string().default(''),
+  incidentThreshold: Schema.number().default(5),
+  concurrencyLimit: Schema.number().default(0),
+  canaryProbingEnabled: Schema.boolean().default(false),
+  canaryIntervalMs: Schema.number().default(30000),
+  cascade: Schema.array(Schema.object({
+    provider: Schema.string().required(),
+    model: Schema.string(),
+  })).default([]),
+  quotaResetWindow: Schema.object({
+    type: Schema.string().default('midnight_utc'),
+    hour: Schema.number().default(0),
+  }),
+  rateLimitThreshold: Schema.number().default(0.1),
+  rpmLimit: Schema.number().default(0),
+  webhookActionToken: Schema.string().default(''),
+  expiryWarnDays: Schema.number().default(7),
+  providers: Schema.array(Schema.object({
+    provider: Schema.string().required(),
+    keys: Schema.array(Schema.string()).default([]),
+    weights: Schema.array(Schema.number()).default([]),
+    expiresAt: Schema.array(Schema.union([Schema.number(), Schema.string()])).default([]),
+    tags: Schema.array(Schema.string()).default([]),
+    costBudgetDaily: Schema.number(),
+    costBudgetWeekly: Schema.number(),
+    pauseOnBudget: Schema.boolean().default(false),
+    models: Schema.dict(Schema.object({
+      keys: Schema.array(Schema.string()).default([]),
+      weights: Schema.array(Schema.number()).default([]),
+    })).default({}),
+    cooldownMs: Schema.number(),
+    maxCooldownMs: Schema.number(),
+  })).default([...DEFAULT_PROVIDERS]),
+});
+
+// ── config bridge (GET/PUT/DELETE on CONFIG_PATH), mirroring llm-fallback ──
+
+
+function json(res, status, obj) {
+  res.writeHead(status, { 'content-type': 'application/json' });
+  res.end(JSON.stringify(obj));
+}
+
+function readJson(request) {
+  return new Promise((resolve, reject) => {
+    let raw = '';
+    request.on('data', (c) => { raw += c; });
+    request.on('end', () => {
+      try {
+        resolve(JSON.parse(raw || '{}'));
+      } catch (e) {
+        reject(e);
+      }
+    });
+    request.on('error', reject);
+  });
+}
+
+function descriptorOf(ctx, ns) {
+  const settings = ctx.get('settings');
+  if (settings === void 0) return void 0;
+  return settings.describe({ redactSecrets: true }).find((candidate) => candidate.ns === ns);
+}
+
+function viewOf(descriptor, settings) {
+  return {
+    available: true,
+    writable: settings.writable,
+    hasDocument: settings.hasDocument,
+    value: descriptor.value,
+    ...descriptor.base === void 0 ? {} : { base: descriptor.base },
+    ...descriptor.user === void 0 || Object.keys(descriptor.user).length === 0 ? {} : { user: descriptor.user },
+    revision: descriptor.revision,
+  };
+}
+
+async function writeSection(ctx, ns, section, expectedRevision, res) {
+  const settings = ctx.get('settings');
+  if (settings === void 0) {
+    json(res, 503, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: no settings provider is mounted' } });
+    return;
+  }
+  try {
+    await settings.replace(ns, section, expectedRevision);
+  } catch (error) {
+    if (error?.code === 'SETTINGS_CONFLICT') {
+      json(res, 409, { error: { code: 'settings-conflict', message: `dsh-key-rotation: changed elsewhere (expected revision ${String(error.expected)}, current ${String(error.actual)}); reload and retry` } });
+      return;
+    }
+    json(res, 400, { error: { code: 'settings-rejected', message: error instanceof Error ? error.message : String(error) } });
+    return;
+  }
+  const descriptor = descriptorOf(ctx, ns);
+  if (descriptor === void 0) {
+    json(res, 500, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: namespace vanished after write' } });
+    return;
+  }
+  json(res, 200, viewOf(descriptor, { writable: settings.writable, hasDocument: settings.documentPath !== void 0 }));
+}
+
+/** Provider catalog for the GUI dropdown, minus clone routes of configured chains. */
+function providerCatalog(ctx, cloneIds) {
+  const seen = new Set();
+  const out = [];
+  for (const info of ctx.llm.listProviders()) {
+    if (seen.has(info.id) || cloneIds.has(info.id)) continue;
+    seen.add(info.id);
+    out.push({ id: info.id, name: info.name ?? info.id });
+  }
+  return out;
+}
+
+async function handleConfigBridge(ctx, request, res, getCloneIds) {
+  if (!isTrustedBridgeRequest(request)) {
+    res.writeHead(403);
+    res.end();
+    return;
+  }
+  const method = request.method ?? 'GET';
+  if (method === 'GET') {
+    const settings = ctx.get('settings');
+    const descriptor = descriptorOf(ctx, NS);
+    const body = {
+      providers: providerCatalog(ctx, getCloneIds()),
+    };
+    if (descriptor === void 0) {
+      json(res, 200, {
+        ...body,
+        available: false,
+        writable: settings?.writable ?? false,
+        hasDocument: settings?.documentPath !== void 0,
+        value: void 0,
+        revision: 0,
+      });
+      return;
+    }
+    json(res, 200, {
+      ...body,
+      ...viewOf(descriptor, {
+        writable: settings?.writable ?? false,
+        hasDocument: settings?.documentPath !== void 0,
+      }),
+    });
+    return;
+  }
+  if (method === 'PUT' || method === 'DELETE') {
+    let section;
+    let expectedRevision;
+    if (method === 'PUT') {
+      let body;
+      try {
+        body = await readJson(request);
+      } catch (error) {
+        json(res, 400, { error: { code: 'settings-rejected', message: `dsh-key-rotation: invalid request body: ${error instanceof Error ? error.message : String(error)}` } });
+        return;
+      }
+      if (typeof body !== 'object' || body === null || typeof body.section !== 'object' || body.section === null || Array.isArray(body.section)) {
+        json(res, 400, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: PUT requires {"section": {...}}' } });
+        return;
+      }
+      section = body.section;
+      expectedRevision = typeof body.expectedRevision === 'number' ? body.expectedRevision : void 0;
+      // #200 leak detector: a live secret pasted into the config section is
+      // almost always a mistake (real key values belong in PUT /key). The two
+      // fields that legitimately hold tokens are masked before scanning.
+      try {
+        const masked = structuredClone(section);
+        if (masked.incidentGitHubToken) masked.incidentGitHubToken = '***';
+        if (masked.webhookActionToken) masked.webhookActionToken = '***';
+        // notifyWebhook legitimately carries bot tokens inside URLs
+        // (api.telegram.org/bot<token>/...) - scan it for nothing.
+        if (masked.notifyWebhook) masked.notifyWebhook = '***';
+        const findings = findSecrets(JSON.stringify(masked));
+        if (findings.length > 0) {
+          json(res, 400, {
+            error: {
+              code: 'secret-in-config',
+              message: `dsh-key-rotation: value looks like a live credential (${findings[0].type}); store key values via the key field, not the config section`,
+              findings,
+            },
+          });
+          return;
+        }
+      } catch {
+        /* scanning must never block a valid save */
+      }
+    } else {
+      section = {};
+    }
+    await writeSection(ctx, NS, section, expectedRevision, res);
+    return;
+  }
+  res.writeHead(405);
+  res.end();
+}
+
+function registerConfigBridge(ctx, getCloneIds) {
+  return ctx.webServer.register({
+    kind: 'exact',
+    path: CONFIG_PATH,
+    handler: (req, res) => void handleConfigBridge(ctx, req, res, getCloneIds),
+  });
+}
+
+// ── plugin ──
+
+export function apply(ctx, config = {}) {
+  // GUI section: defaults -> cordis row config -> saved user section.
+  // (installSettingsSection inlined: no @deepseek-ai/dsh-settings import, so the
+  // profile does not need a second copy of that package.)
+  let getConfig = () => config;
+  registerConfigBridge(ctx, () => buildRuntime().cloneIds);
+  lastTestCacheRunnerCtx = ctx;
+  // Cache should not survive profile restarts (apply is called per reload).
+  // We deliberately do NOT clear on every apply — that would wipe badges when
+  // the user is just typing in the settings card. Re-init only on true reload.
+  ensureSandboxRunner(ctx);
+
+  // Self-healing idle cooldowns: every 60s, lift expired cooldowns for keys
+  // that have been idle for selfHealIdleMs (default 1h). ponytail: small
+  // interval, low cost; skipped when selfHealCooldown is disabled in config.
+  // ponytail: keep handle on the same ctx via closure so buildRuntime() reads
+  // fresh config on every tick. Naive but correct: 60s cadence is cheap.
+  // #196: canary probing before key release from cooldown.
+  // Every canaryIntervalMs, probe refs that are in cooldown and close to expiry.
+  let canaryTimer = null;
+  const startCanary = () => {
+    const cfg = getConfig();
+    if (!cfg || !cfg.canaryProbingEnabled) return;
+    if (canaryTimer) return;
+    canaryTimer = setInterval(() => {
+      try {
+        const c = getConfig();
+        if (!c || !c.canaryProbingEnabled) return;
+        const runner = ensureSandboxRunner();
+        if (!runner) return;
+        if (!canaryProber) {
+          canaryProber = new CanaryProber({ sandboxRunner: runner, intervalMs: c.canaryIntervalMs });
+        }
+        const providers = Array.isArray(c.providers) ? c.providers : [];
+        for (const p of providers) {
+          const pool = buildRuntime().providerToPool.get(p.provider);
+          if (!pool) continue;
+          for (const ref of pool.refs) {
+            const until = pool.state.failedUntil.get(ref) ?? 0;
+            const now = Date.now();
+            // Probe refs in cooldown whose expiry is within canaryIntervalMs of now
+            if (until > now && until - now < (c.canaryIntervalMs ?? 30000)) {
+              canaryProber.probe(ref, ref);
+            }
+          }
+        }
+      } catch (_) { /* ponytail: never crash the timer */ }
+    }, cfg.canaryIntervalMs ?? 30000);
+    if (typeof canaryTimer.unref === 'function') canaryTimer.unref();
+  };
+  startCanary();
+
+  const selfHealTimer = setInterval(() => {
+    const cfg = getConfig();
+    if (!cfg || cfg.selfHealCooldown === false) return;
+    try {
+      const idle = Number.isFinite(cfg.selfHealIdleMs) && cfg.selfHealIdleMs > 0 ? cfg.selfHealIdleMs : 3600000;
+      const providers = Array.isArray(cfg.providers) ? cfg.providers : [];
+      const pools = providers
+        .map((p) => buildRuntime().providerToPool.get(p.provider))
+        .filter(Boolean);
+      healIdleCooldowns(pools, idle);
+    } catch (_) { /* ponytail: never crash the timer */ }
+  }, 60000);
+  if (typeof selfHealTimer.unref === 'function') selfHealTimer.unref();
+
+  // Dashboard widget now lives in client.js (mountDashboard, see issue #152).
+  const DASH_HTML = '';
+    // ── key-pool state, persisted across config reloads ──
+  // base provider -> { failedUntil: Map<ref, epochMs>, pointer: number, lastUsed: ref }
+  const poolState = new Map();
+  // Periodic backup of pools config
+  ctx.effect(() => {
+    const { backupDir, backupIntervalMs, backupKeep } = buildRuntime();
+    if (!backupDir) return;
+    const id = setInterval(() => {
+      try {
+        const fs = require('node:fs');
+        const path = require('node:path');
+        const dir = backupDir;
+        fs.mkdirSync(dir, { recursive: true });
+        const now = new Date();
+        const dateStr = now.toISOString().slice(0,10).replace(/-/g,'');
+        const file = path.join(dir, 'pools-' + dateStr + '.json');
+        const data = JSON.stringify({ backup: now.toISOString(), providers: getConfig()?.providers ?? [] }, null, 2);
+        fs.writeFileSync(file, data, 'utf8');
+        // prune old backups
+        const keep = backupKeep || 7;
+        const files = fs.readdirSync(dir).filter((f) => f.startsWith('pools-') && f.endsWith('.json')).sort();
+        while (files.length > keep) {
+          const old = files.shift();
+          fs.unlinkSync(path.join(dir, old));
+        }
+      } catch (e) {
+        console.warn('[dsh-key-rotation] backup failed:', String(e?.message ?? e));
+      }
+    }, backupIntervalMs || 86400000);
+    return () => clearInterval(id);
+  }, 'dsh-key-rotation: backup pools');
+  // Periodic save of usage/cost stats to file
+  ctx.effect(() => {
+    const { backupDir } = buildRuntime();
+    if (!backupDir) return;
+    try {
+      const fs = require('node:fs');
+      const path = require('node:path');
+      const statsFile = path.join(backupDir, 'stats.json');
+      // Load existing stats at startup
+      try {
+        if (fs.existsSync(statsFile)) {
+          const saved = JSON.parse(fs.readFileSync(statsFile, 'utf8'));
+          for (const st of poolState.values()) {
+            if (saved.usageCounts && st.usageCounts) { for (const [k, v] of Object.entries(saved.usageCounts)) st.usageCounts.set(k, (st.usageCounts.get(k) ?? 0) + v); }
+            if (saved.costPerKey && st.costPerKey) { for (const [k, v] of Object.entries(saved.costPerKey)) st.costPerKey.set(k, (st.costPerKey.get(k) ?? 0) + v); }
+            if (saved.lastUsedAt && st.lastUsedAt) { for (const [k, v] of Object.entries(saved.lastUsedAt)) { if (!st.lastUsedAt.has(k) || v > st.lastUsedAt.get(k)) st.lastUsedAt.set(k, v); } }
+          }
+        }
+      } catch {}
+      // Periodic save
+      const id = setInterval(() => {
+        try {
+          const usageCounts = {}; const costPerKey = {}; const lastUsedAt = {};
+          for (const [base, st] of poolState) {
+            if (st.usageCounts) for (const [k, v] of st.usageCounts) usageCounts[k] = v;
+            if (st.costPerKey) for (const [k, v] of st.costPerKey) costPerKey[k] = v;
+            if (st.lastUsedAt) for (const [k, v] of st.lastUsedAt) lastUsedAt[k] = v;
+          }
+          fs.writeFileSync(statsFile, JSON.stringify({ t: Date.now(), usageCounts, costPerKey, lastUsedAt }), 'utf8');
+        } catch {}
+      }, 60000);
+      return () => clearInterval(id);
+    } catch { return () => {}; }
+  }, 'dsh-key-rotation: persist stats');
+  // Rotation schedule: shift pointer every N days
+  ctx.effect(() => {
+    const { rotationScheduleDays } = buildRuntime();
+    if (!rotationScheduleDays || rotationScheduleDays <= 0) return;
+    const intervalMs = Math.min(rotationScheduleDays * 86400000, 2147483647);
+    const id = setInterval(() => {
+      try {
+        const rt = buildRuntime();
+        let shifted = 0;
+        for (const pool of rt.poolByRef.values()) {
+          if (pool.refs.length < 2) continue;
+          const oldPtr = pool.state.pointer ?? 0;
+          pool.state.pointer = (oldPtr + 1) % pool.refs.length;
+          shifted++;
+          console.warn(`[dsh-key-rotation] ${pool.base}: scheduled rotation -> ${pool.refs[pool.state.pointer]} (day ${rotationScheduleDays})`);
+        }
+        if (shifted) console.warn(`[dsh-key-rotation] schedule: rotated ${shifted} pools`);
+      } catch (e) {
+        console.warn('[dsh-key-rotation] schedule error:', String(e?.message ?? e));
+      }
+    }, intervalMs);
+    return () => clearInterval(id);
+  }, 'dsh-key-rotation: rotation schedule');
+  // Periodic sweep of expired cooldowns — keeps health probe cheap and avoids waiting for next user request
+  ctx.effect(() => {
+    const id = setInterval(() => {
+      const now = Date.now();
+      // probe events for keys whose cooldown just expired
+      for (const st of poolState.values()) {
+        for (const [ref, until] of [...(st.failedUntil?.entries() ?? [])]) {
+          if (until <= now && !st.probedAt?.has(ref)) {
+            st.events.push({ at: until, ref, reason: 'probe', cooldownMs: 0, type: 'probe' });
+            if (st.events.length > 50) st.events.shift();
+            if (!st.probedAt) st.probedAt = new Map();
+            st.probedAt.set(ref, until);
+          }
+        }
+      }
+      const n = sweepExpired(poolState, now);
+      if (n > 0) console.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
+      // #207 expiry pre-warning + #208 cost budget - piggybacked on this timer,
+      // deduped to one notification per key/window per day (shouldNotifyDaily).
+      try {
+        const runtime = buildRuntime();
+        const seen = new Set();
+        for (const pool of runtime.poolByRef.values()) {
+          if (seen.has(pool.base)) continue;
+          seen.add(pool.base);
+          // #207: keys expiring within expiryWarnDays -> one webhook per key/day
+          for (const { ref, expiresInDays } of expiringSoon(pool, runtime.expiryWarnDays, now)) {
+            if (!shouldNotifyDaily(expiryNotifiedAt, pool.base + ':' + ref, now)) continue;
+            console.warn(`[dsh-key-rotation] ${pool.base}: key ${ref} expires in ~${expiresInDays}d`);
+            if (runtime.notifyWebhook) {
+              webhookSender.send(runtime.notifyWebhook, {
+                title: `Key expiring soon: ${pool.base}`,
+                text: `${ref} expires in ~${expiresInDays} day(s)`,
+                provider: pool.base,
+                kind: 'expiry',
+                keys: [ref],
+              });
+            }
+          }
+          // #208: daily/weekly budget -> warn webhook, optional 1-day pause at 100%
+          const budget = runtime.providerBudgets.get(pool.base);
+          if (!budget) continue;
+          const daily = costForDay(pool.state.costDays);
+          const weekly = costForWeek(pool.state.costDays, now);
+          const verdict = budgetVerdict(daily, budget.costBudgetDaily);
+          const wVerdict = budgetVerdict(weekly, budget.costBudgetWeekly);
+          const hit = verdict.warn || wVerdict.warn;
+          if (hit && shouldNotifyDaily(budgetNotifiedAt, pool.base + ':budget', now)) {
+            console.warn(`[dsh-key-rotation] ${pool.base}: cost budget - day $${daily.toFixed(2)}/$${budget.costBudgetDaily} week $${weekly.toFixed(2)}/$${budget.costBudgetWeekly}`);
+            if (runtime.notifyWebhook) {
+              webhookSender.send(runtime.notifyWebhook, {
+                title: `Cost budget: ${pool.base}`,
+                text: `day $${daily.toFixed(2)} of $${budget.costBudgetDaily} · week $${weekly.toFixed(2)} of $${budget.costBudgetWeekly}` + (verdict.exceeded || wVerdict.exceeded ? ' · EXCEEDED' : ''),
+                provider: pool.base,
+                kind: 'budget',
+                spend: { daily, weekly },
+              });
+            }
+          }
+          if ((verdict.exceeded || wVerdict.exceeded) && budget.pauseOnBudget) {
+            const until = now + DAY_MS;
+            for (const ref of pool.refs) {
+              if ((pool.state.failedUntil.get(ref) ?? 0) < until) pool.state.failedUntil.set(ref, until);
+            }
+          }
+        }
+      } catch (_) { /* maintenance must never crash the sweep */ }
+    }, 30000);
+    return () => clearInterval(id);
+  }, 'dsh-key-rotation: sweep expired cooldowns');
+
+  // ── runtime snapshot: config + llm-pi-ai profile mapping ──
+  function buildRuntime() {
+    // Deep-clone before resolving: the frozen snapshot from settings.register
+    // must never be written to by schemastery's dict resolver.
+    const cfg = Config(structuredClone(getConfig() ?? {})) ?? {};
+    const switchCodes = new Set(cfg.switchCodes ?? DEFAULT_SWITCH_CODES);
+    const cooldownMs = cfg.cooldownMs ?? 60000;
+    const maxCooldownMs = cfg.maxCooldownMs ?? undefined;
+    const notifyWebhook = cfg.notifyWebhook ?? '';
+    const notifyThreshold = cfg.notifyThreshold ?? 3;
+    const backupDir = cfg.backupDir ?? '';
+    const backupIntervalMs = cfg.backupIntervalMs ?? 86400000;
+    const backupKeep = cfg.backupKeep ?? 7;
+    const rotationScheduleDays = cfg.rotationScheduleDays ?? 0;
+    const rateLimitThreshold = cfg.rateLimitThreshold ?? 0.1;
+    const rpmLimit = cfg.rpmLimit ?? 0;
+    const webhookActionToken = cfg.webhookActionToken ?? '';
+    const incidentThreshold = cfg.incidentThreshold ?? 5;
+    const concurrencyLimit = cfg.concurrencyLimit ?? 0;
+    const canaryProbingEnabled = cfg.canaryProbingEnabled ?? false;
+    const canaryIntervalMs = cfg.canaryIntervalMs ?? 30000;
+    const cascade = Array.isArray(cfg.cascade) ? cfg.cascade : [];
+    const quotaResetWindow = cfg.quotaResetWindow || null;
+
+    // ref -> pool (every key env of every configured provider)
+    const poolByRef = new Map();
+    // provider route (from llm-pi-ai profiles) -> its key pool
+    const providerToPool = new Map();
+    // per-model key pools: provider -> Map<model, pool>
+    const modelPoolByProvider = new Map();
+    // clone route ids (for the settings dropdown filter)
+    const cloneIds = new Set();
+
+    const makeState = (base) => {
+      let st = poolState.get(base);
+      if (!st) {
+        st = {
+          failedUntil: new Map(),
+          failCounts: new Map(),
+          authFailCounts: new Map(),
+          brokenUntil: new Map(),
+          costPerKey: new Map(),
+          lastUsedAt: new Map(),
+          usageCounts: new Map(),
+          byModel: new Map(),
+          usageDays: new Map(),
+          quotaWindows: new Map(),
+          pointer: 0,
+          lastUsed: undefined,
+          switches: 0,
+          lastReason: undefined,
+          lastSwitchAt: undefined,
+          lastExhaustionAt: undefined,
+          exhaustionCount: 0,
+          events: [],
+        };
+        poolState.set(base, st);
+      }
+      return st;
+    };
+    const parseExpiry = (v) => {
+      if (typeof v === 'number' && v > 0) return v;
+      if (typeof v === 'string' && v.length > 0) { const ts = Date.parse(v); return Number.isNaN(ts) ? undefined : ts; }
+      return undefined;
+    };
+    const buildPool = (base, keys, weights, poolCooldown, poolMax, expiresAt) => {
+      const refs = (keys ?? []).filter((ref) => typeof ref === 'string' && ref.length > 0);
+      if (refs.length === 0) return null;
+      const w = Array.isArray(weights) ? weights : [];
+      const weightedRefs = [];
+      for (let i = 0; i < refs.length; i++) {
+        const ww = typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1;
+        for (let k = 0; k < ww; k++) weightedRefs.push(refs[i]);
+      }
+      const parsedExpiry = {};
+      if (Array.isArray(expiresAt)) {
+        for (let i = 0; i < refs.length; i++) {
+          const exp = parseExpiry(expiresAt[i]);
+          if (exp !== undefined) parsedExpiry[refs[i]] = exp;
+        }
+      }
+      return { base, refs, weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs,
+               state: makeState(base), cooldownMs: poolCooldown, maxCooldownMs: poolMax, expiresAt: parsedExpiry, rpmLimit };
+    };
+    for (const p of cfg.providers ?? []) {
+      const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
+      const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
+      // base provider pool (fallback)
+      const pool = buildPool(p.provider, p.keys, p.weights, poolCooldown, poolMax);
+      if (pool) {
+        for (const ref of pool.refs) poolByRef.set(ref, pool);
+        for (let i = 1; i < pool.refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
+      }
+      // per-model pools
+      const models = p.models ?? {};
+      const byModel = new Map();
+      for (const [model, mp] of Object.entries(models)) {
+        const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, poolCooldown, poolMax);
+        if (mpool) {
+          byModel.set(model, mpool);
+          for (const ref of mpool.refs) poolByRef.set(ref, mpool);
+        }
+      }
+      if (byModel.size > 0) modelPoolByProvider.set(p.provider, byModel);
+    }
+
+    let profiles = {};
+    try {
+      profiles = ctx.get('settings')?.get(PIAI_NS)?.providers ?? {};
+    } catch {
+      /* settings not mounted yet — empty mapping */
+    }
+    for (const [provider, profile] of Object.entries(profiles)) {
+      if (profile?.apiKeyEnv && poolByRef.has(profile.apiKeyEnv)) {
+        providerToPool.set(provider, poolByRef.get(profile.apiKeyEnv));
+      }
+    }
+
+    // auto-cleanup: remove poolState for providers that are now empty or removed
+    for (const key of [...poolState.keys()]) {
+      if (![...poolByRef.values()].some((p) => p.base === key)) poolState.delete(key);
+    }
+    // #192: drop RPM windows for refs that no longer belong to any pool
+    for (const st of poolState.values()) {
+      if (st.rpmWindows) bucketSweep(st.rpmWindows, new Set(poolByRef.keys()));
+    }
+    // #195: provider -> tags (metadata, surfaced in status)
+    const providerTags = new Map();
+    // #208: provider -> { costBudgetDaily, costBudgetWeekly, pauseOnBudget }
+    const providerBudgets = new Map();
+    for (const p of cfg.providers ?? []) {
+      if (Array.isArray(p.tags) && p.tags.length > 0) providerTags.set(p.provider, p.tags);
+      const daily = typeof p.costBudgetDaily === 'number' ? p.costBudgetDaily : 0;
+      const weekly = typeof p.costBudgetWeekly === 'number' ? p.costBudgetWeekly : 0;
+      if (daily > 0 || weekly > 0) providerBudgets.set(p.provider, { costBudgetDaily: daily, costBudgetWeekly: weekly, pauseOnBudget: p.pauseOnBudget ?? false });
+    }
+    return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, concurrencyLimit, canaryProbingEnabled, canaryIntervalMs, cascade, quotaResetWindow, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
+  }
+
+  // ── patch credentials.resolve: pool refs resolve to the next healthy key ──
+  // Round-robin over the pool, skipping keys in cooldown; the request's
+  // provider identity never changes, so pi-ai replay state stays consistent.
+  const credentials = ctx.get('credentials');
+  if (credentials && typeof credentials.resolve === 'function' && !credentials.__dshKeyRotationPatched) {
+    const original = credentials.resolve.bind(credentials);
+    // Kept for the status route: it must ask about one exact ref instead of
+    // being rotated to a different key by the patch below.
+    credentials.__dshKeyRotationOriginalResolve = original;
+    credentials.resolve = async (ref) => {
+      const { poolByRef } = buildRuntime();
+      const pool = poolByRef.get(ref);
+      if (!pool) return original(ref);
+      const now = Date.now();
+      const list = pool.weightedRefs ?? pool.refs;
+      const start = pool.state.pointer ?? 0;
+      for (let i = 0; i < list.length; i++) {
+        const index = (start + i) % list.length;
+        const candidate = list[index];
+        const until = pool.state.failedUntil.get(candidate);
+        if (until !== undefined && until > now) continue;
+        if (pool.expiresAt?.[candidate] !== undefined && now >= pool.expiresAt[candidate]) continue;
+        // #192 RPM token bucket: skip a key that already hit its requests/min cap
+        const rpmLimit = pool.rpmLimit ?? 0;
+        if (rpmLimit > 0) {
+          if (!pool.state.rpmWindows) pool.state.rpmWindows = new Map();
+          if (!bucketAllow(pool.state.rpmWindows, candidate, rpmLimit, now)) {
+            const waitMs = bucketRetryMs(pool.state.rpmWindows, candidate, rpmLimit, now);
+            if ((pool.state.failedUntil.get(candidate) ?? 0) < now + waitMs) {
+              pool.state.failedUntil.set(candidate, now + waitMs);
+            }
+            continue;
+          }
+        }
+        if (pool.perHour) {
+          if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+          let win = pool.state.quotaWindows.get(candidate);
+          if (!win || now - win.start >= 3600000) win = { count: 0, start: now };
+          if (win.count >= pool.perHour) {
+            const until = win.start + 3600000;
+            if ((pool.state.failedUntil.get(candidate) ?? 0) < until) pool.state.failedUntil.set(candidate, until);
+            continue;
+          }
+        }
+        let hit = await original(candidate);
+        if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
+          pool.state.pointer = (index + 1) % list.length;
+          pool.state.lastUsed = candidate;
+          if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
+          pool.state.failedUntil.delete(candidate);
+          if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
+          if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
+          if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
+          pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
+          if (pool.perHour) {
+            if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+            let win2 = pool.state.quotaWindows.get(candidate);
+            if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
+            win2.count++;
+            pool.state.quotaWindows.set(candidate, win2);
+          }
+          return hit;
+        }
+        // fallback: env var (transient, not persisted)
+        const envVal = envValue(candidate);
+        if (envVal !== undefined) {
+          pool.state.pointer = (index + 1) % list.length;
+          pool.state.lastUsed = candidate;
+          if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
+          pool.state.failedUntil.delete(candidate);
+          if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
+          if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
+          if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
+          pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
+          if (pool.perHour) {
+            if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+            let win2 = pool.state.quotaWindows.get(candidate);
+            if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
+            win2.count++;
+            pool.state.quotaWindows.set(candidate, win2);
+          }
+          return { value: envVal, source: 'env' };
+        }
+      }
+      return original(ref); // everything cooled/missing — surface the base value
+    };
+    credentials.__dshKeyRotationPatched = true;
+  }
+
+  const finishError = (code, message) => ({
+    type: 'finish',
+    reason: { kind: 'error', failure: Object.freeze({ code, message }) },
+  });
+
+  // Latency recording (#6): record successful llm/stream latency per ref.
+  // ponytail: only the true success path (finish-chunk). Failures are not recorded.
+  let _rotateStartMs = Date.now();
+  function recordLatency(pool) {
+    try {
+      const cfg = getConfig();
+      if (!cfg || cfg.latencyEnabled === false) return;
+      const ref = pool && pool.state && pool.state.lastUsed;
+      if (!ref) return;
+      const elapsed = Date.now() - _rotateStartMs;
+      if (!Number.isFinite(elapsed) || elapsed < 0) return;
+      latencyHistogram.record(ref, elapsed);
+    } catch (_) { /* ponytail: never crash */ }
+  }
+
+  // Retry one request on the next pool key when the current key fails with a
+  // switchable error before any content chunk. The provider never changes —
+  // the resolve patch hands out the next key on each dispatch.
+  function rotate(options, pool) {
+    return (async function* () {
+      const { switchCodes, cooldownMs, maxCooldownMs } = buildRuntime();
+      let lastFailure = null;
+      _rotateStartMs = Date.now();
+
+      const runtime0 = buildRuntime();
+      if (runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
+        // #193: prefer least-loaded key within limit
+        const available = (pool.weightedRefs ?? pool.refs).filter((r) => {
+          const fu = pool.state.failedUntil.get(r) ?? 0;
+          if (fu > Date.now()) return false;
+          const exp = pool.expiresAt ? pool.expiresAt[r] : undefined;
+          if (exp !== undefined && Date.now() >= exp) return false;
+          return true;
+        });
+        const preferred = concurrencyTracker.pickLeastLoaded(available);
+        if (preferred && (pool.weightedRefs ?? pool.refs)[0] !== preferred) {
+          // Move preferred to front of the attempt list
+          const list = (pool.weightedRefs ?? pool.refs).slice();
+          const i = list.indexOf(preferred);
+          if (i > 0) { list.splice(i, 1); list.unshift(preferred); }
+          pool.weightedRefs = list;
+        }
+      }
+      for (let attempt = 0; attempt < (pool.weightedRefs ?? pool.refs).length; attempt++) {
+        let yielded = false;
+        let switching = false;
+        let inner;
+        try {
+          // mark the internal dispatch so the interceptor does not re-rotate
+          inner = ctx.llm.stream({ ...options, [MARKER]: true });
+        } catch (e) {
+          if (pool.state.lastUsed) { const _retry = parseRetryAfter(String(e?.message ?? '')); const _base = pool.cooldownMs ?? cooldownMs; const _max = pool.maxCooldownMs ?? maxCooldownMs; const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base; const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max); pushEvent(pool, pool.state.lastUsed, e?.code ?? 'TRANSPORT', _b); const _code = String(e?.code ?? ''); if (_code === 'AUTH' || /auth/i.test(String(e?.message ?? ''))) { const _c = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1; pool.state.authFailCounts.set(pool.state.lastUsed, _c); if (_c >= 3) { pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); } } else { pool.state.authFailCounts.delete(pool.state.lastUsed); } }
+          lastFailure = finishError(e?.code ?? 'TRANSPORT',
+            `dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
+          console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(pool.state.lastUsed ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
+          continue;
+        }
+
+        const _pickedRef = pool.state.lastUsed;
+        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.acquire(_pickedRef);
+        try {
+          for await (const chunk of inner) {
+            // Only actual content deltas lock the stream (no more rotation).
+            // Structural/metadata chunks (block-start/end, usage) do not.
+            if (chunk && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta')) {
+              yielded = true;
+              yield chunk;
+              continue;
+            }
+            if (chunk && chunk.type === 'finish') {
+              const kind = chunk.reason?.kind;
+              const failure = chunk.reason?.failure;
+              const code = failure?.code;
+              const message = failure?.message ?? '';
+              const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
+              const switchable = !yielded && kind === 'error' &&
+                (effectiveSwitchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message));
+              if (switchable) {
+                if (pool.state.lastUsed) {
+                  const _retry = parseRetryAfter(message);
+                  const _base = pool.cooldownMs ?? cooldownMs;
+                  const _max = pool.maxCooldownMs ?? maxCooldownMs;
+                  const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base;
+                  const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max);
+                  pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b);
+                  // authFailCounts/brokenUntil: lazy-init if state was created by an older plugin version
+                  if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
+                  if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
+                  const _code2 = String(code ?? '');
+                  if (_code2 === 'AUTH' || /auth/i.test(message)) {
+                    const _c2 = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1;
+                    pool.state.authFailCounts.set(pool.state.lastUsed, _c2);
+                    if (_c2 >= 3) {
+                      pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30);
+                      pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30);
+                    }
+                  } else {
+                    pool.state.authFailCounts.delete(pool.state.lastUsed);
+                  }
+                }
+                pool.state.switches = (pool.state.switches ?? 0) + 1;
+                pool.state.lastReason = String(code ?? 'UNKNOWN');
+                pool.state.lastSwitchAt = Date.now();
+                lastFailure = chunk;
+                console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(pool.state.lastUsed ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) — next key`);
+                switching = true;
+                break;
+              }
+              // cost tracking if provider returns usage.cost
+              if (chunk.usage?.cost != null && pool.state.lastUsed) {
+                const c = Number(chunk.usage.cost);
+                if (!isNaN(c)) {
+                  if (!pool.state.costPerKey) pool.state.costPerKey = new Map();
+                  pool.state.costPerKey.set(pool.state.lastUsed, (pool.state.costPerKey.get(pool.state.lastUsed) ?? 0) + c);
+                  // #208: cost per day per key (mirrors usageDays) for budget checks
+                  if (!pool.state.costDays) pool.state.costDays = new Map();
+                  const cday = new Date().toISOString().slice(0, 10);
+                  const cMap = pool.state.costDays.get(pool.state.lastUsed) || new Map();
+                  cMap.set(cday, (cMap.get(cday) ?? 0) + c);
+                  pool.state.costDays.set(pool.state.lastUsed, cMap);
+                }
+              }
+              // Usage by day (#119)
+              if (pool.state.lastUsed) {
+                if (!pool.state.usageDays) pool.state.usageDays = new Map();
+                const day = new Date().toISOString().slice(0, 10);
+                const dayMap = pool.state.usageDays.get(pool.state.lastUsed) || new Map();
+                dayMap.set(day, (dayMap.get(day) ?? 0) + 1);
+                pool.state.usageDays.set(pool.state.lastUsed, dayMap);
+              }
+              // Per-model request detail (#121)
+              if (pool.state.lastUsed && options.model) {
+                if (!pool.state.byModel) pool.state.byModel = new Map();
+                let byRef = pool.state.byModel.get(pool.state.lastUsed);
+                if (!byRef) { byRef = new Map(); pool.state.byModel.set(pool.state.lastUsed, byRef); }
+                byRef.set(options.model, (byRef.get(options.model) ?? 0) + 1);
+              }
+              // Proactive rate-limit (#115): if response headers say this key is near
+              // its quota, cool it down so the NEXT request starts on a different key.
+              // We do NOT re-run this (already successful) request — that would double-send.
+              const rate = extractRateLimit(chunk?.metadata?.headers ?? chunk?.headers);
+              if (rate && pool.state.lastUsed) {
+                const { rateLimitThreshold } = buildRuntime();
+                if (isRateLimited(rate, rateLimitThreshold ?? 0.1)) {
+                  const cool = rate.reset && rate.reset > Date.now() ? (rate.reset - Date.now()) : pool.cooldownMs;
+                  recordFailure(pool, pool.state.lastUsed, Date.now(), cool, pool.maxCooldownMs);
+                  pushEvent(pool, pool.state.lastUsed, 'RATE_LIMIT', cool);
+                  console.warn(`[dsh-key-rotation] ${options.provider}: key ${pool.state.lastUsed} near quota (remaining ${String(rate.remaining)}/${String(rate.limit)}) — next request will rotate`);
+                }
+              }
+              // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
+              if (rate && pool.state.lastUsed && Number.isFinite(rate.remaining)) {
+                quotaStore.set(pool.state.lastUsed, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: Date.now() });
+              }
+              yield chunk;
+              recordLatency(pool);
+              return;
+            }
+            yield chunk;
+          }
+        } catch (e) {
+          if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
+          yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+          return;
+        }
+
+        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
+        if (switching) continue; // try the next key
+        return; // clean end — served
+      }
+
+      // pool exhausted — all keys cooling or missing
+      pool.state.lastExhaustionAt = Date.now();
+      pool.state.exhaustionCount = (pool.state.exhaustionCount ?? 0) + 1;
+      console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — all ${pool.refs.length} keys cooling`);
+      // notify via extracted helper (see notifyExhaustion above)
+      notifyExhaustion(buildRuntime(), pool, { provider: options.provider });
+
+      // #194: cross-provider cascade failover
+      const runtime = buildRuntime();
+      if (Array.isArray(runtime.cascade) && runtime.cascade.length > 0) {
+        const pools = runtime.providerToPool;
+        const fb = pickCascadeFallback(options.provider, runtime, pools);
+        if (fb && fb.pool && fb.pool !== pool) {
+          console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — cascading to ${fb.provider}`);
+          pool.state.lastReason = 'CASCADE';
+          pool.state.lastSwitchAt = Date.now();
+          // Re-dispatch on the fallback pool (depth-1 via marker check)
+          const innerCascade = rotate({ ...options, provider: fb.provider }, fb.pool);
+          for await (const chunk of innerCascade) {
+            yield chunk;
+          }
+          return;
+        }
+      }
+
+      yield lastFailure ?? finishError('TRANSPORT', 'dsh-key-rotation: all keys failed');
+    })();
+  }
+
+  // ── status route: what the settings card cannot know on its own ──
+  //
+  // Reports, per configured provider, which key is in use, which are cooling
+  // down and until when, whether an env name resolves to a credential at all
+  // (a typo is otherwise silent), and how often rotation has fired.
+  //
+  // Key VALUES never leave the host — only the boolean fact that one exists.
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: STATUS_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'GET') {
+        json(res, 405, { error: { code: 'method', message: 'GET only' } });
+        return;
+      }
+      if (!isTrustedBridgeRequest(req)) {
+        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: status is local-only' } });
+        return;
+      }
+      const { poolByRef, providerTags, providerBudgets } = buildRuntime();
+      const base = ctx.get('credentials');
+      const now = Date.now();
+      const seen = new Set();
+      const providers = [];
+      for (const pool of poolByRef.values()) {
+        if (seen.has(pool.base)) continue;
+        seen.add(pool.base);
+        try {
+        const keys = [];
+        for (const ref of pool.refs) {
+          let present = false;
+          let tail = '';
+          let source = null;
+          let writable = true;
+          try {
+            // The resolve patch is installed on this same service, so ask for
+            // the exact ref: a pool ref would otherwise round-robin to another
+            // key and report a missing name as present.
+            let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
+            present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
+            if (present) tail = keyTail(hit.value);
+            // fallback: env var bootstrapping (issue #7)
+            if (!present) {
+              const ev = envValue(ref);
+              if (ev !== undefined) { present = true; tail = keyTail(ev); source = 'env'; writable = false; }
+            }
+          } catch {
+            present = false;
+          }
+          try {
+            const described = await base?.describe?.(ref);
+            source = described?.source ?? null;
+            writable = described?.writable !== false;
+          } catch {
+            /* describe is optional — the card falls back to editable */
+          }
+          const until = pool.state.failedUntil.get(ref);
+          keys.push({
+            ref,
+            present,
+            tail,
+            source,
+            writable,
+            active: pool.state.lastUsed === ref,
+            cooldownMsLeft: until !== undefined && until > now ? until - now : 0,
+            // #210: RPM capacity snapshot (null when rpmLimit is off)
+            rpm: bucketInfo(pool.state.rpmWindows, ref, pool.rpmLimit, now),
+            usage: pool.state.usageCounts?.get(ref) ?? 0,
+            byModel: pool.state.byModel?.get(ref) ? Object.fromEntries(pool.state.byModel.get(ref)) : {},
+            usageDays: pool.state.usageDays?.get(ref) ? Object.fromEntries(pool.state.usageDays.get(ref)) : {},
+            cost: pool.state.costPerKey?.get(ref) ?? 0,
+            lastUsedAt: pool.state.lastUsedAt?.get(ref) ?? null,
+            expiresAt: pool.expiresAt?.[ref] ?? null,
+            expired: pool.expiresAt?.[ref] !== undefined && now >= pool.expiresAt[ref],
+            broken: pool.state.brokenUntil?.has(ref) ?? false,
+          });
+        }
+        providers.push({
+          provider: pool.base,
+          keys,
+          tags: providerTags.get(pool.base) ?? [],
+          switches: pool.state.switches ?? 0,
+          lastReason: pool.state.lastReason ?? null,
+          lastSwitchAt: pool.state.lastSwitchAt ?? null,
+          lastExhaustionAt: pool.state.lastExhaustionAt ?? null,
+          exhaustionCount: pool.state.exhaustionCount ?? 0,
+          totalUsage: [...(pool.state.usageCounts?.values() ?? [])].reduce((a, b) => a + b, 0),
+          events: (pool.state.events ?? []).slice(-50),
+          healthScore: computeHealthScore(pool.state),
+          // #208: today/week spend + configured budget for the card
+          todayCost: costForDay(pool.state.costDays),
+          weeklyCost: costForWeek(pool.state.costDays, now),
+          budgetDaily: providerBudgets.get(pool.base)?.costBudgetDaily ?? 0,
+          budgetWeekly: providerBudgets.get(pool.base)?.costBudgetWeekly ?? 0,
+          pauseOnBudget: providerBudgets.get(pool.base)?.pauseOnBudget ?? false,
+        });
+        } catch (e) {
+          console.warn(`[dsh-key-rotation] status: pool ${pool.base} failed: ${String(e?.message ?? e)} ${e?.stack ?? ''}`);
+          providers.push({ provider: pool.base, keys: [], statusError: String(e?.message ?? e) });
+        }
+      }
+      json(res, 200, { providers });
+    },
+  }), 'dsh-key-rotation: status route');
+
+  // #209: usage report - per-key requests/cost over the last N days.
+  // ?format=csv returns text/csv; ?days=N window (1..90, default 7).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: USAGE_PATH,
+    handler: (req, res) => {
+      if (req.method !== 'GET') { json(res, 405, { error: { code: 'method', message: 'GET only' } }); return; }
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: usage is local-only' } }); return; }
+      const url = new URL(req.url ?? USAGE_PATH, 'http://localhost');
+      const days = Math.min(90, Math.max(1, Number(url.searchParams.get('days')) || 7));
+      const csv = url.searchParams.get('format') === 'csv';
+      const provider = url.searchParams.get('provider') ?? '';
+      const runtime = buildRuntime();
+      const now = Date.now();
+      const seen = new Set();
+      const report = [];
+      for (const pool of runtime.poolByRef.values()) {
+        if (seen.has(pool.base)) continue;
+        seen.add(pool.base);
+        if (provider && pool.base !== provider) continue;
+        report.push({ provider: pool.base, rows: usageRows(pool, days, now) });
+      }
+      if (csv) {
+        res.writeHead(200, { 'content-type': 'text/csv; charset=utf-8', 'content-disposition': 'attachment; filename="dsh-key-rotation-usage.csv"' });
+        const parts = [];
+        for (const p of report) {
+          if (parts.length > 0) parts.push('');
+          parts.push('# ' + p.provider);
+          parts.push(usageCsv(p.rows));
+        }
+        res.end(parts.join('\n') + '\n');
+        return;
+      }
+      json(res, 200, { at: now, days, providers: report });
+    },
+  }), 'dsh-key-rotation: usage route');
+
+  // ── key route: store a key value without leaving the rotation card ──
+  //
+  // Adding a key used to mean two screens: create the credential elsewhere,
+  // then type its env name here. The value is write-only from the browser —
+  // it is never sent back, only its last few characters are (see the status
+  // route) — and the route is loopback- and same-origin-gated like the config
+  // bridge next to it.
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: KEY_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'PUT' && req.method !== 'DELETE') {
+        json(res, 405, { error: { code: 'method', message: 'PUT or DELETE only' } });
+        return;
+      }
+      if (!isTrustedBridgeRequest(req)) {
+        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: keys are local-only' } });
+        return;
+      }
+      const credentialsService = ctx.get('credentials');
+      if (!credentialsService || typeof credentialsService.set !== 'function') {
+        json(res, 503, { error: { code: 'no-credentials', message: 'dsh-key-rotation: no credentials service is mounted' } });
+        return;
+      }
+      let body;
+      try {
+        body = await readJson(req);
+      } catch (error) {
+        json(res, 400, { error: { code: 'bad-request', message: String(error?.message ?? error) } });
+        return;
+      }
+      const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
+      if (!isValidRef(ref)) {
+        json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } });
+        return;
+      }
+      try {
+        if (req.method === 'DELETE') {
+          await credentialsService.unset(ref);
+          json(res, 200, { ok: true, ref });
+          return;
+        }
+        const value = typeof body?.value === 'string' ? body.value.trim() : '';
+        if (value.length === 0) {
+          json(res, 400, { error: { code: 'empty-value', message: 'dsh-key-rotation: an empty key cannot be stored' } });
+          return;
+        }
+        await credentialsService.set(ref, value);
+        // #200: leak-detector hint - stored value should look like a credential
+        const secretShape = looksLikeApiSecret(value);
+        json(res, 200, { ok: true, ref, tail: keyTail(value), looksLikeSecret: secretShape });
+      } catch (error) {
+        // A ref supplied by the launching environment is read-only, and the
+        // service says so in plain words — pass that through to the card.
+        json(res, 409, { error: { code: 'write-rejected', message: String(error?.message ?? error) } });
+      }
+    },
+  }), 'dsh-key-rotation: key route');
+
+  // ── reset route: clear cooldown for a provider (or a single ref) ──
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: RESET_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'POST') {
+        json(res, 405, { error: { code: 'method', message: 'POST only' } });
+        return;
+      }
+      if (!isTrustedBridgeRequest(req)) {
+        json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: reset is local-only' } });
+        return;
+      }
+      let body;
+      try { body = await readJson(req); } catch (e) {
+        json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } });
+        return;
+      }
+      const provider = typeof body?.provider === 'string' ? body.provider.trim() : '';
+      const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
+      if (provider) {
+        const st = poolState.get(provider);
+        if (!st) { json(res, 404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }); return; }
+        const cleared = st.failedUntil.size;
+        st.failedUntil.clear();
+        st.failCounts?.clear();
+        st.authFailCounts?.clear();
+        st.brokenUntil?.clear();
+        st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
+        json(res, 200, { ok: true, provider, cleared });
+        return;
+      }
+      if (ref) {
+        let found = false;
+        for (const st of poolState.values()) {
+          if (st.failedUntil.has(ref) || st.failCounts?.has(ref)) {
+            st.failedUntil.delete(ref);
+            st.failCounts?.delete(ref);
+            st.authFailCounts?.delete(ref);
+            st.brokenUntil?.delete(ref);
+            if (st.lastUsed === ref) st.lastUsed = undefined;
+            found = true; break;
+          }
+        }
+        // idempotent: even if ref was not cooling, report ok if it looks like a valid ref name
+        if (!found && !isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
+        json(res, 200, { ok: true, ref });
+        return;
+      }
+      json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: POST requires {"provider": "..."} or {"ref": "..."}' } });
+    },
+  }), 'dsh-key-rotation: reset route');
+
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: IMPORT_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: import is local-only' } }); return; }
+      let body; try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
+      const url = typeof body?.url === 'string' ? body.url.trim() : '';
+      if (!url || !url.startsWith('https://')) { json(res, 400, { error: { code: 'bad-url', message: 'dsh-key-rotation: only HTTPS URLs are allowed' } }); return; }
+      try {
+        const resp = await fetch(url);
+        if (!resp.ok) { json(res, 400, { error: { code: 'fetch-failed', message: 'dsh-key-rotation: fetch returned ' + resp.status } }); return; }
+        const data = await resp.json();
+        if (!Array.isArray(data)) { json(res, 400, { error: { code: 'bad-format', message: 'dsh-key-rotation: expected JSON array of providers' } }); return; }
+        const settings = ctx.get('settings');
+        if (!settings) { json(res, 503, { error: { code: 'settings-rejected', message: 'dsh-key-rotation: no settings provider' } }); return; }
+        const desc = settings.describe({ redactSecrets: true }).find((c) => c.ns === NS);
+        const cur = desc?.value?.providers ?? [];
+        const merged = new Map();
+        for (const p of cur) if (p && p.provider) merged.set(p.provider, p);
+        for (const p of data) if (p && p.provider && typeof p.provider === 'string') merged.set(p.provider, p);
+        const mergedArr = [...merged.values()];
+        await settings.replace(NS, { ...(desc?.value ?? {}), providers: mergedArr }, desc?.revision);
+        json(res, 200, { ok: true, providersImported: data.length, total: mergedArr.length });
+      } catch (e) { json(res, 400, { error: { code: 'import-failed', message: String(e?.message ?? e) } }); }
+    },
+  }), 'dsh-key-rotation: import route');
+
+  // Health for external panels (Beszel/Uptime)
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: HEALTH_PATH,
+    handler: async (req, res) => {
+      if (!isTrustedBridgeRequest(req) && req.socket?.remoteAddress !== '127.0.0.1' && req.socket?.remoteAddress !== '::1') { } // allow same-origin already checked
+      if (!isTrustedBridgeRequest(req)) {
+        // also allow plain loopback without Origin
+        if (!isLoopbackAddress(req.socket?.remoteAddress)) { res.writeHead(403); res.end(); return; }
+        if (req.headers['sec-fetch-site'] === 'cross-site') { res.writeHead(403); res.end(); return; }
+      }
+      if (req.method !== 'GET') { json(res, 405, { error: { code: 'method', message: 'GET only' } }); return; }
+      const now = Date.now();
+      const pools = {};
+      let exhaustedAny = false;
+      const { poolByRef: pr, providerTags } = buildRuntime();
+      const seenH = new Set();
+      for (const pool of pr.values()) {
+        if (seenH.has(pool.base)) continue;
+        seenH.add(pool.base);
+        let healthy = 0;
+        for (const ref of pool.refs) {
+          const until = pool.state.failedUntil.get(ref);
+          if (until !== undefined && until > now) continue;
+          const exp = pool.expiresAt?.[ref];
+          if (exp !== undefined && now >= exp) continue;
+          healthy++;
+        }
+        const total = pool.refs.length;
+        const exhausted = healthy === 0 && total > 0;
+        if (exhausted) exhaustedAny = true;
+        pools[pool.base] = { healthy, total, exhausted, healthScore: computeHealthScore(pool.state) };
+      }
+      json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: quotaStore.snapshot() });
+    },
+  }), 'dsh-key-rotation: health');
+
+  // ── test route: dry-run a single key without rotation ──
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: TEST_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: test is local-only' } }); return; }
+      let body; try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
+      const ref = typeof body?.ref === 'string' ? body.ref.trim() : '';
+      if (!isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
+      // Optional value for pre-save validation (issue #118)
+      const testValue = typeof body?.value === 'string' && body.value.length > 0 ? body.value : undefined;
+      const probe = body?.probe === 'models' || body?.probe === 'chat' ? body.probe : undefined;
+      const base = ctx.get('credentials');
+      try {
+        let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
+        let present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
+        const effectiveValue = testValue || hit?.value;
+        const valid = present ? Boolean(effectiveValue && typeof effectiveValue === 'string' && effectiveValue.length > 0) : Boolean(testValue);
+        const tail = valid ? keyTail(effectiveValue) : '';
+        let source = null;
+        try { const d = await base?.describe?.(ref); source = d?.source ?? null; } catch {}
+        if (!present && !testValue) { json(res, 200, { ok: false, ref, code: 'no-credential', message: 'no such credential' }); return; }
+        if (!present && testValue) { source = 'pre-save'; }
+        else if (!present) {
+          const ev = envValue(ref);
+          if (ev !== undefined) { present = true; json(res, 200, { ok: true, ref, tail: keyTail(ev), source: 'env' }); return; }
+        }
+        // sandbox probe (models is free; chat is hook-only, see sandbox.js)
+        if (probe) {
+          const keyForProbe = effectiveValue;
+          const runner = ensureSandboxRunner(ctx);
+          const result = probe === 'chat' ? await runner.probeChat(ref, keyForProbe) : await runner.probeModels(ref, keyForProbe);
+          const cached = { ...result, at: Date.now() };
+          lastTestCache.set(ref, cached);
+          json(res, 200, { ok: cached.ok, ref, tail, source, probe, code: cached.code, latencyMs: cached.latencyMs, modelsCount: cached.modelsCount });
+          return;
+        }
+        json(res, 200, { ok: true, ref, tail, source });
+      } catch (e) {
+        json(res, 200, { ok: false, ref, code: 'error', message: String(e?.message ?? e) });
+      }
+    },
+  }), 'dsh-key-rotation: test route');
+
+  // Intercept the llm/stream waterfall: rotate any request whose provider maps
+  // to a configured key pool; pass everything else (and internal dispatches)
+  // straight through.
+  // Read-only cache snapshot for clients (badge polling).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: SANDBOX_CACHE_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: cache is local-only' } }); return; }
+      json(res, 200, lastTestCache.snapshot());
+    },
+  }), 'dsh-key-rotation: sandbox cache');
+
+  // Auto-incident reset (#8).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: INCIDENT_RESET_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: incident-reset is local-only' } }); return; }
+      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
+      readJson(req).then((body) => {
+        const provider = typeof body?.provider === 'string' ? body.provider : '';
+        if (provider) incidentReporter.resetCooldown(provider);
+        else incidentReporter.resetCooldown();
+        json(res, 200, { ok: true, reset: provider || 'all' });
+      }).catch((e) => json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }));
+    },
+  }), 'dsh-key-rotation: incident-reset');
+
+  // #198: 1-click Health Matrix — parallel probe of all configured keys.
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: TEST_MATRIX_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: matrix is local-only' } }); return; }
+      const cfg = getConfig();
+      const runner = ensureSandboxRunner();
+      if (!runner) { json(res, 500, { error: { code: 'no-runner', message: 'sandbox runner unavailable' } }); return; }
+      const providers = Array.isArray(cfg?.providers) ? cfg.providers : [];
+      const jobs = [];
+      for (const p of providers) {
+        for (const ref of (p.keys ?? [])) {
+          if (typeof ref !== 'string' || !ref) continue;
+          jobs.push((async () => {
+            try {
+              const probeResult = await runner.probeModels(ref, ref);
+              return { provider: p.provider, ref, ok: probeResult.ok, code: probeResult.code, latencyMs: probeResult.latencyMs, modelsCount: probeResult.modelsCount ?? 0 };
+            } catch (e) {
+              return { provider: p.provider, ref, ok: false, code: 'error', latencyMs: 0, modelsCount: 0 };
+            }
+          })());
+        }
+      }
+      const results = await Promise.all(jobs);
+      json(res, 200, { at: Date.now(), total: results.length, ok: results.filter(r => r.ok).length, results });
+    },
+  }), 'dsh-key-rotation: test-matrix');
+
+  // Webhook test endpoint (#10): dry-run that validates webhookSender setup.
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: WEBHOOK_TEST_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: webhook-test is local-only' } }); return; }
+      json(res, 200, { ok: true, snapshot: webhookSender.snapshot() });
+    },
+  }), 'dsh-key-rotation: webhook-test');
+
+  // #199 webhook-action: interactive webhook buttons call back here.
+  // Auth: bearer token from Config (external services like Telegram/Discord
+  // cannot be same-origin, so a shared secret is the gate).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: '/dsh-key-rotation/webhook-action',
+    handler: async (req, res) => {
+      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
+      const runtime = buildRuntime();
+      const expected = runtime.webhookActionToken;
+      if (!expected) { json(res, 503, { error: { code: 'no-token', message: 'dsh-key-rotation: webhookActionToken is not configured' } }); return; }
+      const auth = String(req.headers.authorization ?? '');
+      if (auth !== `Bearer ${expected}`) { json(res, 401, { error: { code: 'unauthorized', message: 'dsh-key-rotation: bad webhook action token' } }); return; }
+      let body;
+      try { body = await readJson(req); } catch (e) { json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }); return; }
+      // Accept callback payloads from formatInteractive (Telegram/Discord/Slack) or plain {action}
+      let action = typeof body?.action === 'string' ? body.action : '';
+      if (!action && typeof body?.data === 'string') {
+        try { action = String(JSON.parse(body.data)?.id ?? ''); } catch { action = ''; }
+      }
+      if (!action && typeof body?.callback_data === 'string') {
+        try { action = String(JSON.parse(body.callback_data)?.id ?? ''); } catch { action = ''; }
+      }
+      if (!action) { json(res, 400, { error: { code: 'bad-action', message: 'dsh-key-rotation: no action in payload' } }); return; }
+      const provider = action.startsWith('pause-') || action.startsWith('reset-') ? action.replace(/^(pause|reset)-/, '') : '';
+      try {
+        if (action === 'disable-rotation') {
+          rotationDisabled = true;
+          console.warn('[dsh-key-rotation] rotation DISABLED via webhook action');
+          json(res, 200, { ok: true, action });
+          return;
+        }
+        if (action === 'enable-rotation') {
+          rotationDisabled = false;
+          json(res, 200, { ok: true, action });
+          return;
+        }
+        if (action.startsWith('pause-') || action.startsWith('reset-')) {
+          const st = poolState.get(provider);
+          if (!st) { json(res, 404, { error: { code: 'not-found', message: `dsh-key-rotation: no pool for '${provider}'` } }); return; }
+          if (action.startsWith('pause-')) {
+            const until = Date.now() + 3600000; // 1h pause
+            for (const ref of (st.failedUntil ? [...st.failedUntil.keys()] : [])) st.failedUntil.set(ref, Math.max(st.failedUntil.get(ref) ?? 0, until));
+            // also pause every key currently healthy
+            for (const p of buildRuntime().poolByRef.values()) {
+              if (p.base !== provider) continue;
+              for (const ref of p.refs) st.failedUntil.set(ref, Math.max(st.failedUntil.get(ref) ?? 0, until));
+            }
+            console.warn(`[dsh-key-rotation] pool ${provider} PAUSED 1h via webhook action`);
+            json(res, 200, { ok: true, action, provider, until: Date.now() + 3600000 });
+            return;
+          }
+          const cleared = st.failedUntil.size;
+          st.failedUntil.clear(); st.failCounts?.clear(); st.brokenUntil?.clear();
+          console.warn(`[dsh-key-rotation] pool ${provider} RESET via webhook action`);
+          json(res, 200, { ok: true, action, provider, cleared });
+          return;
+        }
+        json(res, 400, { error: { code: 'unknown-action', message: `dsh-key-rotation: unknown action '${action}'` } });
+      } catch (e) {
+        json(res, 500, { error: { code: 'action-failed', message: String(e?.message ?? e) } });
+      }
+    },
+  }), 'dsh-key-rotation: webhook-action');
+
+  // Shadow A/B sampling snapshot (#9).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: SHADOW_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: shadow is local-only' } }); return; }
+      json(res, 200, shadowRouter.snapshot());
+    },
+  }), 'dsh-key-rotation: shadow');
+
+  // Region tags + failover chain (#4).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: REGIONS_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: regions is local-only' } }); return; }
+      const body = regionMap.snapshot();
+      // Add pickFallback hints per provider for inspection.
+      const out = {};
+      for (const p of Object.keys(body)) out[p] = { region: body[p], fallback: regionMap.pickFallback(p) };
+      json(res, 200, out);
+    },
+  }), 'dsh-key-rotation: regions');
+
+  // Per-agent rate budget snapshot (#3).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: AGENT_BUDGET_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: agent-budget is local-only' } }); return; }
+      json(res, 200, { enabled: agentBudget.isEnabled(), agents: agentBudget.snapshot() });
+    },
+  }), 'dsh-key-rotation: agent-budget');
+
+  ctx.on('llm/stream', (options, next) => {
+    if (options[MARKER]) return next();
+    if (rotationDisabled) return next(); // #199: disabled via webhook action
+    const { providerToPool, modelPoolByProvider } = buildRuntime();
+    // #195: exact model pool -> longest model-family prefix -> provider pool
+    const pool = selectPool(modelPoolByProvider, providerToPool, options.provider, options.model);
+    if (!pool) return next();
+    console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${(pool.weightedRefs ?? pool.refs).length} slots (${pool.refs.length} keys)`);
+    return rotate(options, pool);
+  });
+
+  // Safety net for non-stream requests (agent/request-error waterfall).
+  // llm/stream covers streaming calls; sync calls (embeddings, batch) go
+  // through agent/request and surface errors here. If the error is
+  // switchable, mark the key and ask the agent loop to retry.
+  ctx.on('agent/request-error', async (payload, next) => {
+    const provider = payload?.provider ?? payload?.failure?.provider ?? '';
+    if (!provider) return next();
+    const { providerToPool, modelPoolByProvider, switchCodes } = buildRuntime();
+    const model = payload?.model || payload?.failure?.model || '';
+    // #195: same tier-aware selection as llm/stream
+    const pool = selectPool(modelPoolByProvider, providerToPool, provider, model);
+    if (!pool) return next();
+    const code = String(payload?.failure?.code ?? payload?.code ?? '');
+    const message = String(payload?.failure?.message ?? payload?.message ?? '');
+    const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
+    const switchable = effectiveSwitchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message);
+    if (!switchable) return next();
+    const ref = pool.state.lastUsed;
+    if (ref) {
+      const backoff = recordFailure(pool, ref, Date.now(), pool.cooldownMs ?? 60000);
+      pushEvent(pool, ref, code || 'UNKNOWN', backoff);
+      pool.state.switches = (pool.state.switches ?? 0) + 1;
+      pool.state.lastReason = code || 'UNKNOWN';
+      pool.state.lastSwitchAt = Date.now();
+      console.warn(`[dsh-key-rotation] ${provider}: key ${String(ref)} failed via agent/request-error (${String(code)} ${String(message).slice(0, 80)}) — retry`);
+    }
+    return { kind: 'retry' };
+  });
+
+  ctx.inject(['settings'], (sctx) => {
+    const scope = sctx.settings.register(NS, Config, { base: config });
+    getConfig = () => scope.get() ?? config;
+    sctx.effect(() => () => {
+      getConfig = () => config;
+    });
+  });
+}
+
+// Notify on exhaustion: webhook + (optional) GitHub incident.
+// Extracted at module scope for testability. No I/O outside the injected hooks.
+// ponytail: thresholds and URLs are runtime-resolved per call, so changing Config is reflected immediately.
+export function notifyExhaustion(runtime, pool, options, hooks = { webhookSender, ensureIncidentReporter }) {
+  if (!runtime || !pool) return;
+  const count = pool.state ? (pool.state.exhaustionCount ?? 0) : 0;
+  if (count <= 0) return;
+  try {
+    if (runtime.notifyWebhook && count >= (runtime.notifyThreshold ?? 0)) {
+      // #199: interactive payload when an action token is configured - the
+      // platform formatter (webhook.js) turns `actions` into buttons whose
+      // callback carries the token back to /dsh-key-rotation/webhook-action.
+      const token = runtime.webhookActionToken ?? '';
+      const payload = {
+        title: `Key pool exhausted: ${options.provider}`,
+        text: `${count} exhaustion(s); keys: ${(pool.refs ?? []).join(', ')}`,
+        provider: options.provider,
+        exhaustionCount: count,
+        at: pool.state.lastExhaustionAt,
+        keys: pool.refs,
+        actionToken: token || undefined,
+        actions: token ? [
+          { id: `reset-${options.provider}`, label: 'Reset cooldown' },
+          { id: `pause-${options.provider}`, label: 'Pause 1h' },
+        ] : undefined,
+      };
+      hooks.webhookSender.send(runtime.notifyWebhook, payload);
+    }
+    if (runtime.incidentThreshold && count >= runtime.incidentThreshold) {
+      const reporter = hooks.ensureIncidentReporter();
+      if (reporter) reporter.open(options.provider, pool.state.lastExhaustionAt);
+    }
+  } catch (_) { /* ponytail: never crash rotate() */ }
+}
+