@goodandready/dsh-key-rotation 0.7.35 → 0.7.37

package/lib/index.js

@@ -32,12 +32,13 @@
 //    cooldownMs: number      key cooldown after a switchable failure
 //    providers:  array       [{ provider, keys: [envName, ...] }]
 // ─────────────────────────────────────────────────────────────────────────────
+import { AsyncLocalStorage } from 'node:async_hooks';
 import Schema from '@deepseek-ai/schemastery';
-import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter, computeHealthScore, extractRateLimit, isRateLimited, selectPool } from './pool.js';
+import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter, computeHealthScore, extractRateLimit, isRateLimited, selectPool, isSwitchableError, formatExhaustionMessage, expiringSoon, shouldNotifyDaily, costForDay, costForWeek, budgetVerdict } from './pool.js';
 
 export const name = 'dsh-key-rotation';
 export const inject = ['llm', 'webServer', 'settings', 'credentials'];
-export { keyTail, isLoopbackAddress, isTrustedBridgeRequest, DEFAULT_SWITCH_CODES };
+export { keyTail, isLoopbackAddress, isTrustedBridgeRequest, DEFAULT_SWITCH_CODES, isSwitchableError, formatExhaustionMessage };
 
 /** Settings namespace owning the GUI-editable section (settingsNamespace-valid). */
 const NS = 'dsh-key-rotation';
@@ -52,28 +53,18 @@ const HEALTH_PATH = '/dsh-key-rotation/health';
 const USAGE_PATH = '/dsh-key-rotation/usage';
 const TEST_PATH = '/dsh-key-rotation/test';
 const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
-const AGENT_BUDGET_PATH = '/dsh-key-rotation/agent-budget';
-const REGIONS_PATH = '/dsh-key-rotation/regions';
-const INCIDENT_RESET_PATH = '/dsh-key-rotation/incident-reset';
-const SHADOW_PATH = '/dsh-key-rotation/shadow';
-const WEBHOOK_TEST_PATH = '/dsh-key-rotation/webhook-test';
-const TEST_MATRIX_PATH = '/dsh-key-rotation/test-matrix';
 import { LastTestCache, SandboxRunner } from './sandbox.js';
 import { healIdleCooldowns } from './heal.js';
 import { LatencyHistogram } from './histogram.js';
 import { pickCascadeFallback } from './cascade.js';
 import { ConcurrencyTracker } from './concurrency.js';
 import { nextQuotaReset } from './quota-window.js';
-import { CanaryProber } from './canary.js';
 import { QuotaStore } from './quota.js';
-import { AgentBudget } from './agent-budget.js';
-import { RegionMap } from './region.js';
-import { IncidentReporter } from './incident.js';
-import { ShadowRouter } from './shadow.js';
 import { WebhookSender } from './webhook.js';
 import { bucketAllow, bucketRetryMs, bucketSweep, bucketInfo } from './bucket.js';
-import { expiringSoon, shouldNotifyDaily, costForDay, budgetVerdict, costForWeek } from './maintenance.js';
-import { usageRows, usageCsv } from './usage-report.js';
+import { usageRows, usageCsv, compactUsage } from './usage-report.js';
+
+const dispatchStorage = new AsyncLocalStorage();
 import { findSecrets, looksLikeApiSecret } from './keycheck.js';
 
 /** The llm-pi-ai namespace whose provider profiles map providers to pools. */
@@ -130,28 +121,12 @@ let lastTestCacheRunnerCtx = null;
 const lastTestCache = new LastTestCache();
 const latencyHistogram = new LatencyHistogram();
 const quotaStore = new QuotaStore();
-const agentBudget = new AgentBudget();
-const regionMap = new RegionMap();
 // Global config accessor safe against early initialization
 let getConfig = () => null;
 let getRuntime = () => null;
-
-// IncidentReporter: lazily built when Config provides incidentGitHubToken + incidentGitHubBaseUrl.
-// ponytail: never bake the token into source; repo is hardcoded (this plugin's home repo) but token is per-deploy.
-let incidentReporter = null;
-function ensureIncidentReporter() {
-  if (incidentReporter) return incidentReporter;
-  const cfg = getConfig();
-  const token = cfg ? cfg.incidentGitHubToken : '';
-  const baseUrl = cfg ? cfg.incidentGitHubBaseUrl : '';
-  if (!token || !baseUrl) return null;
-  incidentReporter = new IncidentReporter({ token, baseUrl, repo: 'goodandready/dsh-key-rotation', fetchImpl: globalThis.fetch });
-  return incidentReporter;
-}
-const shadowRouter = new ShadowRouter({ primary: '', secondary: '', percent: 0 });let sandboxRunner = null;
+let sandboxRunner = null;
 const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
 const concurrencyTracker = new ConcurrencyTracker();
-let canaryProber = null;
 function ensureSandboxRunner(ctx) {
   if (sandboxRunner) return sandboxRunner;
   // provider id or key ref -> baseUrl (stripped of trailing /) for fetch /models probe
@@ -211,20 +186,11 @@ export const Config = Schema.object({
   maxCooldownMs: Schema.number(),
   notifyWebhook: Schema.string().default(''),
   notifyThreshold: Schema.number().default(3),
-  backupDir: Schema.string().default(''),
-  backupIntervalMs: Schema.number().default(86400000),
-  backupKeep: Schema.number().default(7),
-  rotationScheduleDays: Schema.number().default(0),
   selfHealCooldown: Schema.boolean().default(true),
   selfHealIdleMs: Schema.number().default(3600000),
   latencyEnabled: Schema.boolean().default(true),
   latencyWindow: Schema.number().default(200),
-  incidentGitHubToken: Schema.string().role('secret').default(''),
-  incidentGitHubBaseUrl: Schema.string().default(''),
-  incidentThreshold: Schema.number().default(5),
   concurrencyLimit: Schema.number().default(0),
-  canaryProbingEnabled: Schema.boolean().default(false),
-  canaryIntervalMs: Schema.number().default(30000),
   cascade: Schema.array(Schema.object({
     provider: Schema.string().required(),
     model: Schema.string(),
@@ -391,7 +357,6 @@ async function handleConfigBridge(ctx, request, res, getCloneIds) {
       // fields that legitimately hold tokens are masked before scanning.
       try {
         const masked = structuredClone(section);
-        if (masked.incidentGitHubToken) masked.incidentGitHubToken = '***';
         if (masked.webhookActionToken) masked.webhookActionToken = '***';
         // notifyWebhook legitimately carries bot tokens inside URLs
         // (api.telegram.org/bot<token>/...) - scan it for nothing.
@@ -447,39 +412,6 @@ export function apply(ctx, config = {}) {
   // interval, low cost; skipped when selfHealCooldown is disabled in config.
   // ponytail: keep handle on the same ctx via closure so buildRuntime() reads
   // fresh config on every tick. Naive but correct: 60s cadence is cheap.
-  // #196: canary probing before key release from cooldown.
-  // Every canaryIntervalMs, probe refs that are in cooldown and close to expiry.
-  // Canary prober lifecycle effect
-  ctx.effect(() => {
-    const cfg = getConfig();
-    if (!cfg || !cfg.canaryProbingEnabled) return () => {};
-    const timer = setInterval(() => {
-      try {
-        const c = getConfig();
-        if (!c || !c.canaryProbingEnabled) return;
-        const runner = ensureSandboxRunner(ctx);
-        if (!runner) return;
-        if (!canaryProber) {
-          canaryProber = new CanaryProber({ sandboxRunner: runner, intervalMs: c.canaryIntervalMs });
-        }
-        const providers = Array.isArray(c.providers) ? c.providers : [];
-        for (const p of providers) {
-          const pool = buildRuntime().providerToPool.get(p.provider);
-          if (!pool) continue;
-          for (const ref of pool.refs) {
-            const until = pool.state.failedUntil.get(ref) ?? 0;
-            const now = Date.now();
-            if (until > now && until - now < (c.canaryIntervalMs ?? 30000)) {
-              canaryProber.probe(ref, ref);
-            }
-          }
-        }
-      } catch (_) { /* ponytail: never crash the timer */ }
-    }, cfg.canaryIntervalMs ?? 30000);
-    if (typeof timer.unref === 'function') timer.unref();
-    return () => clearInterval(timer);
-  }, 'dsh-key-rotation: canary prober');
-
   // Self-healing idle cooldowns lifecycle effect
   ctx.effect(() => {
     const cfg = getConfig();
@@ -500,96 +432,9 @@ export function apply(ctx, config = {}) {
     return () => clearInterval(timer);
   }, 'dsh-key-rotation: self-healing idle');
 
-  // Dashboard widget now lives in client.js (mountDashboard, see issue #152).
-  const DASH_HTML = '';
-    // ── key-pool state, persisted across config reloads ──
+  // ── key-pool state, persisted across config reloads ──
   // base provider -> { failedUntil: Map<ref, epochMs>, pointer: number, lastUsed: ref }
   const poolState = new Map();
-  // Periodic backup of pools config
-  ctx.effect(() => {
-    const { backupDir, backupIntervalMs, backupKeep } = buildRuntime();
-    if (!backupDir) return;
-    const id = setInterval(() => {
-      try {
-        const fs = require('node:fs');
-        const path = require('node:path');
-        const dir = backupDir;
-        fs.mkdirSync(dir, { recursive: true });
-        const now = new Date();
-        const dateStr = now.toISOString().slice(0,10).replace(/-/g,'');
-        const file = path.join(dir, 'pools-' + dateStr + '.json');
-        const data = JSON.stringify({ backup: now.toISOString(), providers: getConfig()?.providers ?? [] }, null, 2);
-        fs.writeFileSync(file, data, 'utf8');
-        // prune old backups
-        const keep = backupKeep || 7;
-        const files = fs.readdirSync(dir).filter((f) => f.startsWith('pools-') && f.endsWith('.json')).sort();
-        while (files.length > keep) {
-          const old = files.shift();
-          fs.unlinkSync(path.join(dir, old));
-        }
-      } catch (e) {
-        console.warn('[dsh-key-rotation] backup failed:', String(e?.message ?? e));
-      }
-    }, backupIntervalMs || 86400000);
-    return () => clearInterval(id);
-  }, 'dsh-key-rotation: backup pools');
-  // Periodic save of usage/cost stats to file
-  ctx.effect(() => {
-    const { backupDir } = buildRuntime();
-    if (!backupDir) return;
-    try {
-      const fs = require('node:fs');
-      const path = require('node:path');
-      const statsFile = path.join(backupDir, 'stats.json');
-      // Load existing stats at startup
-      try {
-        if (fs.existsSync(statsFile)) {
-          const saved = JSON.parse(fs.readFileSync(statsFile, 'utf8'));
-          for (const st of poolState.values()) {
-            if (saved.usageCounts && st.usageCounts) { for (const [k, v] of Object.entries(saved.usageCounts)) st.usageCounts.set(k, (st.usageCounts.get(k) ?? 0) + v); }
-            if (saved.costPerKey && st.costPerKey) { for (const [k, v] of Object.entries(saved.costPerKey)) st.costPerKey.set(k, (st.costPerKey.get(k) ?? 0) + v); }
-            if (saved.lastUsedAt && st.lastUsedAt) { for (const [k, v] of Object.entries(saved.lastUsedAt)) { if (!st.lastUsedAt.has(k) || v > st.lastUsedAt.get(k)) st.lastUsedAt.set(k, v); } }
-          }
-        }
-      } catch {}
-      // Periodic save
-      const id = setInterval(() => {
-        try {
-          const usageCounts = {}; const costPerKey = {}; const lastUsedAt = {};
-          for (const [base, st] of poolState) {
-            if (st.usageCounts) for (const [k, v] of st.usageCounts) usageCounts[k] = v;
-            if (st.costPerKey) for (const [k, v] of st.costPerKey) costPerKey[k] = v;
-            if (st.lastUsedAt) for (const [k, v] of st.lastUsedAt) lastUsedAt[k] = v;
-          }
-          fs.writeFileSync(statsFile, JSON.stringify({ t: Date.now(), usageCounts, costPerKey, lastUsedAt }), 'utf8');
-        } catch {}
-      }, 60000);
-      return () => clearInterval(id);
-    } catch { return () => {}; }
-  }, 'dsh-key-rotation: persist stats');
-  // Rotation schedule: shift pointer every N days
-  ctx.effect(() => {
-    const { rotationScheduleDays } = buildRuntime();
-    if (!rotationScheduleDays || rotationScheduleDays <= 0) return;
-    const intervalMs = Math.min(rotationScheduleDays * 86400000, 2147483647);
-    const id = setInterval(() => {
-      try {
-        const rt = buildRuntime();
-        let shifted = 0;
-        for (const pool of rt.poolByRef.values()) {
-          if (pool.refs.length < 2) continue;
-          const oldPtr = pool.state.pointer ?? 0;
-          pool.state.pointer = (oldPtr + 1) % pool.refs.length;
-          shifted++;
-          console.warn(`[dsh-key-rotation] ${pool.base}: scheduled rotation -> ${pool.refs[pool.state.pointer]} (day ${rotationScheduleDays})`);
-        }
-        if (shifted) console.warn(`[dsh-key-rotation] schedule: rotated ${shifted} pools`);
-      } catch (e) {
-        console.warn('[dsh-key-rotation] schedule error:', String(e?.message ?? e));
-      }
-    }, intervalMs);
-    return () => clearInterval(id);
-  }, 'dsh-key-rotation: rotation schedule');
   // Periodic sweep of expired cooldowns — keeps health probe cheap and avoids waiting for next user request
   ctx.effect(() => {
     const id = setInterval(() => {
@@ -607,6 +452,9 @@ export function apply(ctx, config = {}) {
       }
       const n = sweepExpired(poolState, now);
       if (n > 0) console.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
+      for (const pool of buildRuntime().poolByRef.values()) {
+        compactUsage(pool, 30, now);
+      }
       // #207 expiry pre-warning + #208 cost budget - piggybacked on this timer,
       // deduped to one notification per key/window per day (shouldNotifyDaily).
       try {
@@ -719,27 +567,34 @@ export function apply(ctx, config = {}) {
   }, 'dsh-key-rotation: sweep expired cooldowns');
 
   // ── runtime snapshot: config + llm-pi-ai profile mapping ──
+  let cachedRuntime = null;
+  let lastConfigRef = null;
+  let lastProfilesRef = null;
+
   getRuntime = buildRuntime;
   function buildRuntime() {
-    // Deep-clone before resolving: the frozen snapshot from settings.register
-    // must never be written to by schemastery's dict resolver.
-    const cfg = Config(structuredClone(getConfig() ?? {})) ?? {};
+    const rawConfig = getConfig() ?? {};
+    let currentProfiles = null;
+    try {
+      currentProfiles = ctx.get('settings')?.get(PIAI_NS)?.providers ?? null;
+    } catch {
+      /* settings not mounted yet — empty mapping */
+    }
+
+    if (cachedRuntime && lastConfigRef === rawConfig && lastProfilesRef === currentProfiles) {
+      return cachedRuntime;
+    }
+
+    const cfg = Config(rawConfig) ?? {};
     const switchCodes = new Set(cfg.switchCodes ?? DEFAULT_SWITCH_CODES);
     const cooldownMs = cfg.cooldownMs ?? 60000;
     const maxCooldownMs = cfg.maxCooldownMs ?? undefined;
     const notifyWebhook = cfg.notifyWebhook ?? '';
     const notifyThreshold = cfg.notifyThreshold ?? 3;
-    const backupDir = cfg.backupDir ?? '';
-    const backupIntervalMs = cfg.backupIntervalMs ?? 86400000;
-    const backupKeep = cfg.backupKeep ?? 7;
-    const rotationScheduleDays = cfg.rotationScheduleDays ?? 0;
     const rateLimitThreshold = cfg.rateLimitThreshold ?? 0.1;
     const rpmLimit = cfg.rpmLimit ?? 0;
     const webhookActionToken = cfg.webhookActionToken ?? '';
-    const incidentThreshold = cfg.incidentThreshold ?? 5;
     const concurrencyLimit = cfg.concurrencyLimit ?? 0;
-    const canaryProbingEnabled = cfg.canaryProbingEnabled ?? false;
-    const canaryIntervalMs = cfg.canaryIntervalMs ?? 30000;
     const cascade = Array.isArray(cfg.cascade) ? cfg.cascade : [];
     const quotaResetWindow = cfg.quotaResetWindow || null;
 
@@ -856,7 +711,10 @@ export function apply(ctx, config = {}) {
       const weekly = typeof p.costBudgetWeekly === 'number' ? p.costBudgetWeekly : 0;
       if (daily > 0 || weekly > 0) providerBudgets.set(p.provider, { costBudgetDaily: daily, costBudgetWeekly: weekly, pauseOnBudget: p.pauseOnBudget ?? false });
     }
-    return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, concurrencyLimit, canaryProbingEnabled, canaryIntervalMs, cascade, quotaResetWindow, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
+    cachedRuntime = { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, concurrencyLimit, cascade, quotaResetWindow, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
+    lastConfigRef = rawConfig;
+    lastProfilesRef = currentProfiles;
+    return cachedRuntime;
   }
 
   // ── patch credentials.resolve: pool refs resolve to the next healthy key ──
@@ -904,10 +762,13 @@ export function apply(ctx, config = {}) {
             continue;
           }
         }
+        // Advance pointer immediately so concurrent requests round-robin across distinct healthy keys
+        pool.state.pointer = (index + 1) % list.length;
         let hit = await original(candidate);
         if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
-          pool.state.pointer = (index + 1) % list.length;
           pool.state.lastUsed = candidate;
+          const store = dispatchStorage.getStore();
+          if (store && store.pool === pool) store.pickedRef = candidate;
           if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
           pool.state.failedUntil.delete(candidate);
           if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
@@ -926,8 +787,9 @@ export function apply(ctx, config = {}) {
         // fallback: env var (transient, not persisted)
         const envVal = envValue(candidate);
         if (envVal !== undefined) {
-          pool.state.pointer = (index + 1) % list.length;
           pool.state.lastUsed = candidate;
+          const store = dispatchStorage.getStore();
+          if (store && store.pool === pool) store.pickedRef = candidate;
           if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
           pool.state.failedUntil.delete(candidate);
           if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
@@ -965,13 +827,14 @@ export function apply(ctx, config = {}) {
   // Latency recording (#6): record successful llm/stream latency per ref.
   // ponytail: only the true success path (finish-chunk). Failures are not recorded.
   let _rotateStartMs = Date.now();
-  function recordLatency(pool) {
+  function recordLatency(pool, reqStore) {
     try {
       const cfg = getConfig();
       if (!cfg || cfg.latencyEnabled === false) return;
-      const ref = pool && pool.state && pool.state.lastUsed;
+      const ref = reqStore?.pickedRef ?? pool?.state?.lastUsed;
       if (!ref) return;
-      const elapsed = Date.now() - _rotateStartMs;
+      const startMs = reqStore?.startMs ?? _rotateStartMs;
+      const elapsed = Date.now() - startMs;
       if (!Number.isFinite(elapsed) || elapsed < 0) return;
       latencyHistogram.record(ref, elapsed);
     } catch (_) { /* ponytail: never crash */ }
@@ -984,12 +847,14 @@ export function apply(ctx, config = {}) {
     return (async function* () {
       const { switchCodes, cooldownMs, maxCooldownMs } = buildRuntime();
       let lastFailure = null;
-      _rotateStartMs = Date.now();
+      const reqStore = { pool, pickedRef: undefined, startMs: Date.now() };
+      _rotateStartMs = reqStore.startMs;
 
       const runtime0 = buildRuntime();
+      let attemptList = (pool.weightedRefs ?? pool.refs).slice();
       if (runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
         // #193: prefer least-loaded key within limit
-        const available = (pool.weightedRefs ?? pool.refs).filter((r) => {
+        const available = attemptList.filter((r) => {
           const fu = pool.state.failedUntil.get(r) ?? 0;
           if (fu > Date.now()) return false;
           const exp = pool.expiresAt ? pool.expiresAt[r] : undefined;
@@ -997,30 +862,54 @@ export function apply(ctx, config = {}) {
           return true;
         });
         const preferred = concurrencyTracker.pickLeastLoaded(available);
-        if (preferred && (pool.weightedRefs ?? pool.refs)[0] !== preferred) {
-          // Move preferred to front of the attempt list
-          const list = (pool.weightedRefs ?? pool.refs).slice();
+        if (preferred && attemptList[0] !== preferred) {
+          const list = attemptList.slice();
           const i = list.indexOf(preferred);
           if (i > 0) { list.splice(i, 1); list.unshift(preferred); }
-          pool.weightedRefs = list;
+          attemptList = list;
         }
       }
-      for (let attempt = 0; attempt < (pool.weightedRefs ?? pool.refs).length; attempt++) {
+
+      const penalizeRef = (targetRef, errCode, errMsg) => {
+        if (!targetRef) return;
+        const _retry = parseRetryAfter(errMsg);
+        const _base = pool.cooldownMs ?? cooldownMs;
+        const _max = pool.maxCooldownMs ?? maxCooldownMs;
+        const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base;
+        const _b = recordFailure(pool, targetRef, Date.now(), _effBase, _max);
+        pushEvent(pool, targetRef, errCode ?? 'UNKNOWN', _b);
+        if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
+        if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
+        const _cStr = String(errCode ?? '');
+        if (_cStr === 'AUTH' || /auth/i.test(errMsg)) {
+          const _c2 = (pool.state.authFailCounts.get(targetRef) ?? 0) + 1;
+          pool.state.authFailCounts.set(targetRef, _c2);
+          if (_c2 >= 3) {
+            pool.state.brokenUntil.set(targetRef, Date.now() + 86400000 * 30);
+            pool.state.failedUntil.set(targetRef, Date.now() + 86400000 * 30);
+          }
+        } else {
+          pool.state.authFailCounts.delete(targetRef);
+        }
+      };
+
+      for (let attempt = 0; attempt < attemptList.length; attempt++) {
         let yielded = false;
         let switching = false;
         let inner;
         try {
           // mark the internal dispatch so the interceptor does not re-rotate
-          inner = ctx.llm.stream({ ...options, [MARKER]: true });
+          inner = dispatchStorage.run(reqStore, () => ctx.llm.stream({ ...options, [MARKER]: true }));
         } catch (e) {
-          if (pool.state.lastUsed) { const _retry = parseRetryAfter(String(e?.message ?? '')); const _base = pool.cooldownMs ?? cooldownMs; const _max = pool.maxCooldownMs ?? maxCooldownMs; const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base; const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max); pushEvent(pool, pool.state.lastUsed, e?.code ?? 'TRANSPORT', _b); const _code = String(e?.code ?? ''); if (_code === 'AUTH' || /auth/i.test(String(e?.message ?? ''))) { const _c = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1; pool.state.authFailCounts.set(pool.state.lastUsed, _c); if (_c >= 3) { pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); } } else { pool.state.authFailCounts.delete(pool.state.lastUsed); } }
+          const curRef = reqStore.pickedRef ?? pool.state.lastUsed;
+          penalizeRef(curRef, e?.code ?? 'TRANSPORT', String(e?.message ?? ''));
           lastFailure = finishError(e?.code ?? 'TRANSPORT',
             `dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
-          console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(pool.state.lastUsed ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
+          console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(curRef ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
           continue;
         }
 
-        const _pickedRef = pool.state.lastUsed;
+        const _pickedRef = reqStore.pickedRef ?? pool.state.lastUsed;
         if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.acquire(_pickedRef);
         try {
           for await (const chunk of inner) {
@@ -1037,41 +926,20 @@ export function apply(ctx, config = {}) {
               const code = failure?.code;
               const message = failure?.message ?? '';
               const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-              const switchable = !yielded && kind === 'error' &&
-                (effectiveSwitchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message));
+              const switchable = !yielded && kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes);
+              const activeRef = reqStore.pickedRef ?? pool.state.lastUsed;
               if (switchable) {
-                if (pool.state.lastUsed) {
-                  const _retry = parseRetryAfter(message);
-                  const _base = pool.cooldownMs ?? cooldownMs;
-                  const _max = pool.maxCooldownMs ?? maxCooldownMs;
-                  const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base;
-                  const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max);
-                  pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b);
-                  // authFailCounts/brokenUntil: lazy-init if state was created by an older plugin version
-                  if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
-                  if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
-                  const _code2 = String(code ?? '');
-                  if (_code2 === 'AUTH' || /auth/i.test(message)) {
-                    const _c2 = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1;
-                    pool.state.authFailCounts.set(pool.state.lastUsed, _c2);
-                    if (_c2 >= 3) {
-                      pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30);
-                      pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30);
-                    }
-                  } else {
-                    pool.state.authFailCounts.delete(pool.state.lastUsed);
-                  }
-                }
+                penalizeRef(activeRef, code ?? 'UNKNOWN', message);
                 pool.state.switches = (pool.state.switches ?? 0) + 1;
                 pool.state.lastReason = String(code ?? 'UNKNOWN');
                 pool.state.lastSwitchAt = Date.now();
                 lastFailure = chunk;
-                console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(pool.state.lastUsed ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) - next key`);
+                console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) - next key`);
                 // #216: per-switch webhook (opt-in switchNotify), deduped per provider
-                if (buildRuntime().switchNotify && pool.state.lastUsed) {
+                if (buildRuntime().switchNotify && activeRef) {
                   notifySwitch(buildRuntime(), pool, {
                     provider: options.provider,
-                    from: pool.state.lastUsed,
+                    from: activeRef,
                     code: String(code ?? 'UNKNOWN'),
                     at: pool.state.lastSwitchAt,
                   });
@@ -1080,59 +948,78 @@ export function apply(ctx, config = {}) {
                 break;
               }
               // cost tracking if provider returns usage.cost
-              if (chunk.usage?.cost != null && pool.state.lastUsed) {
+              if (chunk.usage?.cost != null && activeRef) {
                 const c = Number(chunk.usage.cost);
                 if (!isNaN(c)) {
                   if (!pool.state.costPerKey) pool.state.costPerKey = new Map();
-                  pool.state.costPerKey.set(pool.state.lastUsed, (pool.state.costPerKey.get(pool.state.lastUsed) ?? 0) + c);
+                  pool.state.costPerKey.set(activeRef, (pool.state.costPerKey.get(activeRef) ?? 0) + c);
                   // #208: cost per day per key (mirrors usageDays) for budget checks
                   if (!pool.state.costDays) pool.state.costDays = new Map();
                   const cday = new Date().toISOString().slice(0, 10);
-                  const cMap = pool.state.costDays.get(pool.state.lastUsed) || new Map();
+                  const cMap = pool.state.costDays.get(activeRef) || new Map();
                   cMap.set(cday, (cMap.get(cday) ?? 0) + c);
-                  pool.state.costDays.set(pool.state.lastUsed, cMap);
+                  pool.state.costDays.set(activeRef, cMap);
                 }
               }
               // Usage by day (#119)
-              if (pool.state.lastUsed) {
+              if (activeRef) {
                 if (!pool.state.usageDays) pool.state.usageDays = new Map();
                 const day = new Date().toISOString().slice(0, 10);
-                const dayMap = pool.state.usageDays.get(pool.state.lastUsed) || new Map();
+                const dayMap = pool.state.usageDays.get(activeRef) || new Map();
                 dayMap.set(day, (dayMap.get(day) ?? 0) + 1);
-                pool.state.usageDays.set(pool.state.lastUsed, dayMap);
+                pool.state.usageDays.set(activeRef, dayMap);
               }
               // Per-model request detail (#121)
-              if (pool.state.lastUsed && options.model) {
+              if (activeRef && options.model) {
                 if (!pool.state.byModel) pool.state.byModel = new Map();
-                let byRef = pool.state.byModel.get(pool.state.lastUsed);
-                if (!byRef) { byRef = new Map(); pool.state.byModel.set(pool.state.lastUsed, byRef); }
+                let byRef = pool.state.byModel.get(activeRef);
+                if (!byRef) { byRef = new Map(); pool.state.byModel.set(activeRef, byRef); }
                 byRef.set(options.model, (byRef.get(options.model) ?? 0) + 1);
               }
               // Proactive rate-limit (#115): if response headers say this key is near
               // its quota, cool it down so the NEXT request starts on a different key.
               // We do NOT re-run this (already successful) request — that would double-send.
               const rate = extractRateLimit(chunk?.metadata?.headers ?? chunk?.headers);
-              if (rate && pool.state.lastUsed) {
+              if (rate && activeRef) {
                 const { rateLimitThreshold } = buildRuntime();
                 if (isRateLimited(rate, rateLimitThreshold ?? 0.1)) {
                   const cool = rate.reset && rate.reset > Date.now() ? (rate.reset - Date.now()) : pool.cooldownMs;
-                  recordFailure(pool, pool.state.lastUsed, Date.now(), cool, pool.maxCooldownMs);
-                  pushEvent(pool, pool.state.lastUsed, 'RATE_LIMIT', cool);
-                  console.warn(`[dsh-key-rotation] ${options.provider}: key ${pool.state.lastUsed} near quota (remaining ${String(rate.remaining)}/${String(rate.limit)}) — next request will rotate`);
+                  recordFailure(pool, activeRef, Date.now(), cool, pool.maxCooldownMs);
+                  pushEvent(pool, activeRef, 'RATE_LIMIT', cool);
+                  console.warn(`[dsh-key-rotation] ${options.provider}: key ${activeRef} near quota (remaining ${String(rate.remaining)}/${String(rate.limit)}) — next request will rotate`);
                 }
               }
               // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
-              if (rate && pool.state.lastUsed && Number.isFinite(rate.remaining)) {
-                quotaStore.set(pool.state.lastUsed, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: Date.now() });
+              if (rate && activeRef && Number.isFinite(rate.remaining)) {
+                quotaStore.set(activeRef, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: Date.now() });
               }
               yield chunk;
-              recordLatency(pool);
+              recordLatency(pool, reqStore);
               return;
             }
             yield chunk;
           }
         } catch (e) {
           if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
+          const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
+          const activeRef = _pickedRef ?? reqStore.pickedRef ?? pool.state.lastUsed;
+          if (!yielded && isSwitchableError(e, effectiveSwitchCodes)) {
+            penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+            pool.state.switches = (pool.state.switches ?? 0) + 1;
+            pool.state.lastReason = String(e?.code ?? 'TRANSPORT');
+            pool.state.lastSwitchAt = Date.now();
+            lastFailure = finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+            console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} stream threw ${String(e?.code ?? e?.message ?? e)} - failover to next key`);
+            if (buildRuntime().switchNotify && activeRef) {
+              notifySwitch(buildRuntime(), pool, {
+                provider: options.provider,
+                from: activeRef,
+                code: String(e?.code ?? 'TRANSPORT'),
+                at: pool.state.lastSwitchAt,
+              });
+            }
+            continue; // Failover to next key!
+          }
           yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
           return;
         }
@@ -1167,7 +1054,8 @@ export function apply(ctx, config = {}) {
         }
       }
 
-      yield lastFailure ?? finishError('TRANSPORT', 'dsh-key-rotation: all keys failed');
+      const exhaustionMsg = formatExhaustionMessage(options.provider, pool);
+      yield lastFailure ?? finishError('QUOTA', exhaustionMsg);
     })();
   }
 
@@ -1337,7 +1225,6 @@ export function apply(ctx, config = {}) {
         const exportable = { ...value };
         // token-shaped fields stay empty in the file; refs are names, not secrets
         exportable.webhookActionToken = '';
-        if (exportable.incidentGitHubToken) exportable.incidentGitHubToken = '';
         json(res, 200, { at: Date.now(), version: 1, snapshot: exportable });
         return;
       }
@@ -1349,7 +1236,6 @@ export function apply(ctx, config = {}) {
       // #200 leak guard applies to imported content too
       try {
         const masked = structuredClone(snap);
-        if (masked.incidentGitHubToken) masked.incidentGitHubToken = '***';
         if (masked.webhookActionToken) masked.webhookActionToken = '***';
         if (masked.notifyWebhook) masked.notifyWebhook = '***';
         const findings = findSecrets(JSON.stringify(masked));
@@ -1363,7 +1249,6 @@ export function apply(ctx, config = {}) {
       // empty token fields in the file keep the current values (never wipe a secret)
       const merged = { ...cur, ...snap };
       if (!snap.webhookActionToken) merged.webhookActionToken = cur.webhookActionToken ?? '';
-      if (!snap.incidentGitHubToken) merged.incidentGitHubToken = cur.incidentGitHubToken ?? '';
       try {
         await settings.replace(NS, merged, desc.revision);
         const after = descriptorOf(ctx, NS);
@@ -1586,6 +1471,16 @@ export function apply(ctx, config = {}) {
           const result = probe === 'chat' ? await runner.probeChat(ref, keyForProbe) : await runner.probeModels(ref, keyForProbe);
           const cached = { ...result, at: Date.now() };
           lastTestCache.set(ref, cached);
+          if (cached.ok) {
+            for (const st of poolState.values()) {
+              if (st.failedUntil?.has(ref) || st.failCounts?.has(ref) || st.brokenUntil?.has(ref)) {
+                st.failedUntil?.delete(ref);
+                st.failCounts?.delete(ref);
+                st.authFailCounts?.delete(ref);
+                st.brokenUntil?.delete(ref);
+              }
+            }
+          }
           json(res, 200, { ok: cached.ok, ref, tail, source, probe, code: cached.code, latencyMs: cached.latencyMs, modelsCount: cached.modelsCount });
           return;
         }
@@ -1609,61 +1504,7 @@ export function apply(ctx, config = {}) {
     },
   }), 'dsh-key-rotation: sandbox cache');
 
-  // Auto-incident reset (#8).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: INCIDENT_RESET_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: incident-reset is local-only' } }); return; }
-      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-      readJson(req).then((body) => {
-        const provider = typeof body?.provider === 'string' ? body.provider : '';
-        if (provider) incidentReporter.resetCooldown(provider);
-        else incidentReporter.resetCooldown();
-        json(res, 200, { ok: true, reset: provider || 'all' });
-      }).catch((e) => json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }));
-    },
-  }), 'dsh-key-rotation: incident-reset');
-
-  // #198: 1-click Health Matrix — parallel probe of all configured keys.
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: TEST_MATRIX_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: matrix is local-only' } }); return; }
-      const cfg = getConfig();
-      const runner = ensureSandboxRunner();
-      if (!runner) { json(res, 500, { error: { code: 'no-runner', message: 'sandbox runner unavailable' } }); return; }
-      const providers = Array.isArray(cfg?.providers) ? cfg.providers : [];
-      const jobs = [];
-      for (const p of providers) {
-        for (const ref of (p.keys ?? [])) {
-          if (typeof ref !== 'string' || !ref) continue;
-          jobs.push((async () => {
-            try {
-              const probeResult = await runner.probeModels(ref, ref);
-              return { provider: p.provider, ref, ok: probeResult.ok, code: probeResult.code, latencyMs: probeResult.latencyMs, modelsCount: probeResult.modelsCount ?? 0 };
-            } catch (e) {
-              return { provider: p.provider, ref, ok: false, code: 'error', latencyMs: 0, modelsCount: 0 };
-            }
-          })());
-        }
-      }
-      const results = await Promise.all(jobs);
-      json(res, 200, { at: Date.now(), total: results.length, ok: results.filter(r => r.ok).length, results });
-    },
-  }), 'dsh-key-rotation: test-matrix');
 
-  // Webhook test endpoint (#10): dry-run that validates webhookSender setup.
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: WEBHOOK_TEST_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: webhook-test is local-only' } }); return; }
-      json(res, 200, { ok: true, snapshot: webhookSender.snapshot() });
-    },
-  }), 'dsh-key-rotation: webhook-test');
 
   // #199 webhook-action: interactive webhook buttons call back here.
   // Auth: bearer token from Config (external services like Telegram/Discord
@@ -1752,39 +1593,7 @@ export function apply(ctx, config = {}) {
     },
   }), 'dsh-key-rotation: webhook-action');
 
-  // Shadow A/B sampling snapshot (#9).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: SHADOW_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: shadow is local-only' } }); return; }
-      json(res, 200, shadowRouter.snapshot());
-    },
-  }), 'dsh-key-rotation: shadow');
 
-  // Region tags + failover chain (#4).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: REGIONS_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: regions is local-only' } }); return; }
-      const body = regionMap.snapshot();
-      // Add pickFallback hints per provider for inspection.
-      const out = {};
-      for (const p of Object.keys(body)) out[p] = { region: body[p], fallback: regionMap.pickFallback(p) };
-      json(res, 200, out);
-    },
-  }), 'dsh-key-rotation: regions');
-
-  // Per-agent rate budget snapshot (#3).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: AGENT_BUDGET_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: agent-budget is local-only' } }); return; }
-      json(res, 200, { enabled: agentBudget.isEnabled(), agents: agentBudget.snapshot() });
-    },
-  }), 'dsh-key-rotation: agent-budget');
 
   ctx.effect(() => ctx.on('llm/stream', (options, next) => {
     if (options[MARKER]) return next();
@@ -1812,7 +1621,7 @@ export function apply(ctx, config = {}) {
     const code = String(payload?.failure?.code ?? payload?.code ?? '');
     const message = String(payload?.failure?.message ?? payload?.message ?? '');
     const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-    const switchable = effectiveSwitchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message);
+    const switchable = isSwitchableError(payload, effectiveSwitchCodes);
     if (!switchable) return next();
     const ref = pool.state.lastUsed;
     if (ref) {
@@ -1835,18 +1644,15 @@ export function apply(ctx, config = {}) {
   });
 }
 
-// Notify on exhaustion: webhook + (optional) GitHub incident.
+// Notify on exhaustion: webhook notification.
 // Extracted at module scope for testability. No I/O outside the injected hooks.
 // ponytail: thresholds and URLs are runtime-resolved per call, so changing Config is reflected immediately.
-export function notifyExhaustion(runtime, pool, options, hooks = { webhookSender, ensureIncidentReporter }) {
+export function notifyExhaustion(runtime, pool, options, hooks = { webhookSender }) {
   if (!runtime || !pool) return;
   const count = pool.state ? (pool.state.exhaustionCount ?? 0) : 0;
   if (count <= 0) return;
   try {
     if (runtime.notifyWebhook && count >= (runtime.notifyThreshold ?? 0)) {
-      // #199: interactive payload when an action token is configured - the
-      // platform formatter (webhook.js) turns `actions` into buttons whose
-      // callback carries the token back to /dsh-key-rotation/webhook-action.
       const token = runtime.webhookActionToken ?? '';
       const payload = {
         title: `Key pool exhausted: ${options.provider}`,
@@ -1863,10 +1669,6 @@ export function notifyExhaustion(runtime, pool, options, hooks = { webhookSender
       };
       hooks.webhookSender.send(runtime.notifyWebhook, payload);
     }
-    if (runtime.incidentThreshold && count >= runtime.incidentThreshold) {
-      const reporter = hooks.ensureIncidentReporter();
-      if (reporter) reporter.open(options.provider, pool.state.lastExhaustionAt);
-    }
   } catch (_) { /* ponytail: never crash rotate() */ }
 }