@goodandready/dsh-key-rotation 0.7.34 → 0.7.36

package/lib/index.js

@@ -33,11 +33,11 @@
 //    providers:  array       [{ provider, keys: [envName, ...] }]
 // ─────────────────────────────────────────────────────────────────────────────
 import Schema from '@deepseek-ai/schemastery';
-import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter, computeHealthScore, extractRateLimit, isRateLimited, selectPool } from './pool.js';
+import { keyTail, isLoopbackAddress, isTrustedBridgeRequest, SWITCHABLE_MESSAGE_PATTERN, DEFAULT_SWITCH_CODES, isValidRef, pickNext, applyCooldown, recordFailure, recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter, computeHealthScore, extractRateLimit, isRateLimited, selectPool, isSwitchableError, formatExhaustionMessage, expiringSoon, shouldNotifyDaily, costForDay, costForWeek, budgetVerdict } from './pool.js';
 
 export const name = 'dsh-key-rotation';
 export const inject = ['llm', 'webServer', 'settings', 'credentials'];
-export { keyTail, isLoopbackAddress, isTrustedBridgeRequest, DEFAULT_SWITCH_CODES };
+export { keyTail, isLoopbackAddress, isTrustedBridgeRequest, DEFAULT_SWITCH_CODES, isSwitchableError, formatExhaustionMessage };
 
 /** Settings namespace owning the GUI-editable section (settingsNamespace-valid). */
 const NS = 'dsh-key-rotation';
@@ -52,27 +52,15 @@ const HEALTH_PATH = '/dsh-key-rotation/health';
 const USAGE_PATH = '/dsh-key-rotation/usage';
 const TEST_PATH = '/dsh-key-rotation/test';
 const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
-const AGENT_BUDGET_PATH = '/dsh-key-rotation/agent-budget';
-const REGIONS_PATH = '/dsh-key-rotation/regions';
-const INCIDENT_RESET_PATH = '/dsh-key-rotation/incident-reset';
-const SHADOW_PATH = '/dsh-key-rotation/shadow';
-const WEBHOOK_TEST_PATH = '/dsh-key-rotation/webhook-test';
-const TEST_MATRIX_PATH = '/dsh-key-rotation/test-matrix';
 import { LastTestCache, SandboxRunner } from './sandbox.js';
 import { healIdleCooldowns } from './heal.js';
 import { LatencyHistogram } from './histogram.js';
 import { pickCascadeFallback } from './cascade.js';
 import { ConcurrencyTracker } from './concurrency.js';
 import { nextQuotaReset } from './quota-window.js';
-import { CanaryProber } from './canary.js';
 import { QuotaStore } from './quota.js';
-import { AgentBudget } from './agent-budget.js';
-import { RegionMap } from './region.js';
-import { IncidentReporter } from './incident.js';
-import { ShadowRouter } from './shadow.js';
 import { WebhookSender } from './webhook.js';
 import { bucketAllow, bucketRetryMs, bucketSweep, bucketInfo } from './bucket.js';
-import { expiringSoon, shouldNotifyDaily, costForDay, budgetVerdict, costForWeek } from './maintenance.js';
 import { usageRows, usageCsv } from './usage-report.js';
 import { findSecrets, looksLikeApiSecret } from './keycheck.js';
 
@@ -130,28 +118,12 @@ let lastTestCacheRunnerCtx = null;
 const lastTestCache = new LastTestCache();
 const latencyHistogram = new LatencyHistogram();
 const quotaStore = new QuotaStore();
-const agentBudget = new AgentBudget();
-const regionMap = new RegionMap();
 // Global config accessor safe against early initialization
 let getConfig = () => null;
 let getRuntime = () => null;
-
-// IncidentReporter: lazily built when Config provides incidentGitHubToken + incidentGitHubBaseUrl.
-// ponytail: never bake the token into source; repo is hardcoded (this plugin's home repo) but token is per-deploy.
-let incidentReporter = null;
-function ensureIncidentReporter() {
-  if (incidentReporter) return incidentReporter;
-  const cfg = getConfig();
-  const token = cfg ? cfg.incidentGitHubToken : '';
-  const baseUrl = cfg ? cfg.incidentGitHubBaseUrl : '';
-  if (!token || !baseUrl) return null;
-  incidentReporter = new IncidentReporter({ token, baseUrl, repo: 'goodandready/dsh-key-rotation', fetchImpl: globalThis.fetch });
-  return incidentReporter;
-}
-const shadowRouter = new ShadowRouter({ primary: '', secondary: '', percent: 0 });let sandboxRunner = null;
+let sandboxRunner = null;
 const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
 const concurrencyTracker = new ConcurrencyTracker();
-let canaryProber = null;
 function ensureSandboxRunner(ctx) {
   if (sandboxRunner) return sandboxRunner;
   // provider id or key ref -> baseUrl (stripped of trailing /) for fetch /models probe
@@ -211,20 +183,11 @@ export const Config = Schema.object({
   maxCooldownMs: Schema.number(),
   notifyWebhook: Schema.string().default(''),
   notifyThreshold: Schema.number().default(3),
-  backupDir: Schema.string().default(''),
-  backupIntervalMs: Schema.number().default(86400000),
-  backupKeep: Schema.number().default(7),
-  rotationScheduleDays: Schema.number().default(0),
   selfHealCooldown: Schema.boolean().default(true),
   selfHealIdleMs: Schema.number().default(3600000),
   latencyEnabled: Schema.boolean().default(true),
   latencyWindow: Schema.number().default(200),
-  incidentGitHubToken: Schema.string().default(''),
-  incidentGitHubBaseUrl: Schema.string().default(''),
-  incidentThreshold: Schema.number().default(5),
   concurrencyLimit: Schema.number().default(0),
-  canaryProbingEnabled: Schema.boolean().default(false),
-  canaryIntervalMs: Schema.number().default(30000),
   cascade: Schema.array(Schema.object({
     provider: Schema.string().required(),
     model: Schema.string(),
@@ -235,7 +198,7 @@ export const Config = Schema.object({
   }),
   rateLimitThreshold: Schema.number().default(0.1),
   rpmLimit: Schema.number().default(0),
-  webhookActionToken: Schema.string().default(''),
+  webhookActionToken: Schema.string().role('secret').default(''),
   expiryWarnDays: Schema.number().default(7),
   switchNotify: Schema.boolean().default(false),
   switchNotifyThrottleMs: Schema.number().default(60000),
@@ -391,7 +354,6 @@ async function handleConfigBridge(ctx, request, res, getCloneIds) {
       // fields that legitimately hold tokens are masked before scanning.
       try {
         const masked = structuredClone(section);
-        if (masked.incidentGitHubToken) masked.incidentGitHubToken = '***';
         if (masked.webhookActionToken) masked.webhookActionToken = '***';
         // notifyWebhook legitimately carries bot tokens inside URLs
         // (api.telegram.org/bot<token>/...) - scan it for nothing.
@@ -447,39 +409,6 @@ export function apply(ctx, config = {}) {
   // interval, low cost; skipped when selfHealCooldown is disabled in config.
   // ponytail: keep handle on the same ctx via closure so buildRuntime() reads
   // fresh config on every tick. Naive but correct: 60s cadence is cheap.
-  // #196: canary probing before key release from cooldown.
-  // Every canaryIntervalMs, probe refs that are in cooldown and close to expiry.
-  // Canary prober lifecycle effect
-  ctx.effect(() => {
-    const cfg = getConfig();
-    if (!cfg || !cfg.canaryProbingEnabled) return () => {};
-    const timer = setInterval(() => {
-      try {
-        const c = getConfig();
-        if (!c || !c.canaryProbingEnabled) return;
-        const runner = ensureSandboxRunner(ctx);
-        if (!runner) return;
-        if (!canaryProber) {
-          canaryProber = new CanaryProber({ sandboxRunner: runner, intervalMs: c.canaryIntervalMs });
-        }
-        const providers = Array.isArray(c.providers) ? c.providers : [];
-        for (const p of providers) {
-          const pool = buildRuntime().providerToPool.get(p.provider);
-          if (!pool) continue;
-          for (const ref of pool.refs) {
-            const until = pool.state.failedUntil.get(ref) ?? 0;
-            const now = Date.now();
-            if (until > now && until - now < (c.canaryIntervalMs ?? 30000)) {
-              canaryProber.probe(ref, ref);
-            }
-          }
-        }
-      } catch (_) { /* ponytail: never crash the timer */ }
-    }, cfg.canaryIntervalMs ?? 30000);
-    if (typeof timer.unref === 'function') timer.unref();
-    return () => clearInterval(timer);
-  }, 'dsh-key-rotation: canary prober');
-
   // Self-healing idle cooldowns lifecycle effect
   ctx.effect(() => {
     const cfg = getConfig();
@@ -500,96 +429,9 @@ export function apply(ctx, config = {}) {
     return () => clearInterval(timer);
   }, 'dsh-key-rotation: self-healing idle');
 
-  // Dashboard widget now lives in client.js (mountDashboard, see issue #152).
-  const DASH_HTML = '';
-    // ── key-pool state, persisted across config reloads ──
+  // ── key-pool state, persisted across config reloads ──
   // base provider -> { failedUntil: Map<ref, epochMs>, pointer: number, lastUsed: ref }
   const poolState = new Map();
-  // Periodic backup of pools config
-  ctx.effect(() => {
-    const { backupDir, backupIntervalMs, backupKeep } = buildRuntime();
-    if (!backupDir) return;
-    const id = setInterval(() => {
-      try {
-        const fs = require('node:fs');
-        const path = require('node:path');
-        const dir = backupDir;
-        fs.mkdirSync(dir, { recursive: true });
-        const now = new Date();
-        const dateStr = now.toISOString().slice(0,10).replace(/-/g,'');
-        const file = path.join(dir, 'pools-' + dateStr + '.json');
-        const data = JSON.stringify({ backup: now.toISOString(), providers: getConfig()?.providers ?? [] }, null, 2);
-        fs.writeFileSync(file, data, 'utf8');
-        // prune old backups
-        const keep = backupKeep || 7;
-        const files = fs.readdirSync(dir).filter((f) => f.startsWith('pools-') && f.endsWith('.json')).sort();
-        while (files.length > keep) {
-          const old = files.shift();
-          fs.unlinkSync(path.join(dir, old));
-        }
-      } catch (e) {
-        console.warn('[dsh-key-rotation] backup failed:', String(e?.message ?? e));
-      }
-    }, backupIntervalMs || 86400000);
-    return () => clearInterval(id);
-  }, 'dsh-key-rotation: backup pools');
-  // Periodic save of usage/cost stats to file
-  ctx.effect(() => {
-    const { backupDir } = buildRuntime();
-    if (!backupDir) return;
-    try {
-      const fs = require('node:fs');
-      const path = require('node:path');
-      const statsFile = path.join(backupDir, 'stats.json');
-      // Load existing stats at startup
-      try {
-        if (fs.existsSync(statsFile)) {
-          const saved = JSON.parse(fs.readFileSync(statsFile, 'utf8'));
-          for (const st of poolState.values()) {
-            if (saved.usageCounts && st.usageCounts) { for (const [k, v] of Object.entries(saved.usageCounts)) st.usageCounts.set(k, (st.usageCounts.get(k) ?? 0) + v); }
-            if (saved.costPerKey && st.costPerKey) { for (const [k, v] of Object.entries(saved.costPerKey)) st.costPerKey.set(k, (st.costPerKey.get(k) ?? 0) + v); }
-            if (saved.lastUsedAt && st.lastUsedAt) { for (const [k, v] of Object.entries(saved.lastUsedAt)) { if (!st.lastUsedAt.has(k) || v > st.lastUsedAt.get(k)) st.lastUsedAt.set(k, v); } }
-          }
-        }
-      } catch {}
-      // Periodic save
-      const id = setInterval(() => {
-        try {
-          const usageCounts = {}; const costPerKey = {}; const lastUsedAt = {};
-          for (const [base, st] of poolState) {
-            if (st.usageCounts) for (const [k, v] of st.usageCounts) usageCounts[k] = v;
-            if (st.costPerKey) for (const [k, v] of st.costPerKey) costPerKey[k] = v;
-            if (st.lastUsedAt) for (const [k, v] of st.lastUsedAt) lastUsedAt[k] = v;
-          }
-          fs.writeFileSync(statsFile, JSON.stringify({ t: Date.now(), usageCounts, costPerKey, lastUsedAt }), 'utf8');
-        } catch {}
-      }, 60000);
-      return () => clearInterval(id);
-    } catch { return () => {}; }
-  }, 'dsh-key-rotation: persist stats');
-  // Rotation schedule: shift pointer every N days
-  ctx.effect(() => {
-    const { rotationScheduleDays } = buildRuntime();
-    if (!rotationScheduleDays || rotationScheduleDays <= 0) return;
-    const intervalMs = Math.min(rotationScheduleDays * 86400000, 2147483647);
-    const id = setInterval(() => {
-      try {
-        const rt = buildRuntime();
-        let shifted = 0;
-        for (const pool of rt.poolByRef.values()) {
-          if (pool.refs.length < 2) continue;
-          const oldPtr = pool.state.pointer ?? 0;
-          pool.state.pointer = (oldPtr + 1) % pool.refs.length;
-          shifted++;
-          console.warn(`[dsh-key-rotation] ${pool.base}: scheduled rotation -> ${pool.refs[pool.state.pointer]} (day ${rotationScheduleDays})`);
-        }
-        if (shifted) console.warn(`[dsh-key-rotation] schedule: rotated ${shifted} pools`);
-      } catch (e) {
-        console.warn('[dsh-key-rotation] schedule error:', String(e?.message ?? e));
-      }
-    }, intervalMs);
-    return () => clearInterval(id);
-  }, 'dsh-key-rotation: rotation schedule');
   // Periodic sweep of expired cooldowns — keeps health probe cheap and avoids waiting for next user request
   ctx.effect(() => {
     const id = setInterval(() => {
@@ -719,27 +561,34 @@ export function apply(ctx, config = {}) {
   }, 'dsh-key-rotation: sweep expired cooldowns');
 
   // ── runtime snapshot: config + llm-pi-ai profile mapping ──
+  let cachedRuntime = null;
+  let lastConfigRef = null;
+  let lastProfilesRef = null;
+
   getRuntime = buildRuntime;
   function buildRuntime() {
-    // Deep-clone before resolving: the frozen snapshot from settings.register
-    // must never be written to by schemastery's dict resolver.
-    const cfg = Config(structuredClone(getConfig() ?? {})) ?? {};
+    const rawConfig = getConfig() ?? {};
+    let currentProfiles = null;
+    try {
+      currentProfiles = ctx.get('settings')?.get(PIAI_NS)?.providers ?? null;
+    } catch {
+      /* settings not mounted yet — empty mapping */
+    }
+
+    if (cachedRuntime && lastConfigRef === rawConfig && lastProfilesRef === currentProfiles) {
+      return cachedRuntime;
+    }
+
+    const cfg = Config(rawConfig) ?? {};
     const switchCodes = new Set(cfg.switchCodes ?? DEFAULT_SWITCH_CODES);
     const cooldownMs = cfg.cooldownMs ?? 60000;
     const maxCooldownMs = cfg.maxCooldownMs ?? undefined;
     const notifyWebhook = cfg.notifyWebhook ?? '';
     const notifyThreshold = cfg.notifyThreshold ?? 3;
-    const backupDir = cfg.backupDir ?? '';
-    const backupIntervalMs = cfg.backupIntervalMs ?? 86400000;
-    const backupKeep = cfg.backupKeep ?? 7;
-    const rotationScheduleDays = cfg.rotationScheduleDays ?? 0;
     const rateLimitThreshold = cfg.rateLimitThreshold ?? 0.1;
     const rpmLimit = cfg.rpmLimit ?? 0;
     const webhookActionToken = cfg.webhookActionToken ?? '';
-    const incidentThreshold = cfg.incidentThreshold ?? 5;
     const concurrencyLimit = cfg.concurrencyLimit ?? 0;
-    const canaryProbingEnabled = cfg.canaryProbingEnabled ?? false;
-    const canaryIntervalMs = cfg.canaryIntervalMs ?? 30000;
     const cascade = Array.isArray(cfg.cascade) ? cfg.cascade : [];
     const quotaResetWindow = cfg.quotaResetWindow || null;
 
@@ -856,19 +705,23 @@ export function apply(ctx, config = {}) {
       const weekly = typeof p.costBudgetWeekly === 'number' ? p.costBudgetWeekly : 0;
       if (daily > 0 || weekly > 0) providerBudgets.set(p.provider, { costBudgetDaily: daily, costBudgetWeekly: weekly, pauseOnBudget: p.pauseOnBudget ?? false });
     }
-    return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, concurrencyLimit, canaryProbingEnabled, canaryIntervalMs, cascade, quotaResetWindow, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
+    cachedRuntime = { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, concurrencyLimit, cascade, quotaResetWindow, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
+    lastConfigRef = rawConfig;
+    lastProfilesRef = currentProfiles;
+    return cachedRuntime;
   }
 
   // ── patch credentials.resolve: pool refs resolve to the next healthy key ──
   // Round-robin over the pool, skipping keys in cooldown; the request's
   // provider identity never changes, so pi-ai replay state stays consistent.
-  const credentials = ctx.get('credentials');
-  if (credentials && typeof credentials.resolve === 'function' && !credentials.__dshKeyRotationPatched) {
-    const original = credentials.resolve.bind(credentials);
-    // Kept for the status route: it must ask about one exact ref instead of
-    // being rotated to a different key by the patch below.
-    credentials.__dshKeyRotationOriginalResolve = original;
-    credentials.resolve = async (ref) => {
+  ctx.effect(() => {
+    const credentials = ctx.get('credentials');
+    if (credentials && typeof credentials.resolve === 'function' && !credentials.__dshKeyRotationPatched) {
+      const original = credentials.resolve.bind(credentials);
+      // Kept for the status route: it must ask about one exact ref instead of
+      // being rotated to a different key by the patch below.
+      credentials.__dshKeyRotationOriginalResolve = original;
+      credentials.resolve = async (ref) => {
       const { poolByRef } = buildRuntime();
       const pool = poolByRef.get(ref);
       if (!pool) return original(ref);
@@ -903,9 +756,10 @@ export function apply(ctx, config = {}) {
             continue;
           }
         }
+        // Advance pointer immediately so concurrent requests round-robin across distinct healthy keys
+        pool.state.pointer = (index + 1) % list.length;
         let hit = await original(candidate);
         if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
-          pool.state.pointer = (index + 1) % list.length;
           pool.state.lastUsed = candidate;
           if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
           pool.state.failedUntil.delete(candidate);
@@ -925,7 +779,6 @@ export function apply(ctx, config = {}) {
         // fallback: env var (transient, not persisted)
         const envVal = envValue(candidate);
         if (envVal !== undefined) {
-          pool.state.pointer = (index + 1) % list.length;
           pool.state.lastUsed = candidate;
           if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
           pool.state.failedUntil.delete(candidate);
@@ -944,9 +797,17 @@ export function apply(ctx, config = {}) {
         }
       }
       return original(ref); // everything cooled/missing — surface the base value
-    };
-    credentials.__dshKeyRotationPatched = true;
-  }
+      };
+      credentials.__dshKeyRotationPatched = true;
+      return () => {
+        if (credentials.__dshKeyRotationPatched) {
+          credentials.resolve = original;
+          delete credentials.__dshKeyRotationPatched;
+          delete credentials.__dshKeyRotationOriginalResolve;
+        }
+      };
+    }
+  }, 'dsh-key-rotation: patch credentials.resolve');
 
   const finishError = (code, message) => ({
     type: 'finish',
@@ -1028,8 +889,7 @@ export function apply(ctx, config = {}) {
               const code = failure?.code;
               const message = failure?.message ?? '';
               const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-              const switchable = !yielded && kind === 'error' &&
-                (effectiveSwitchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message));
+              const switchable = !yielded && kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes);
               if (switchable) {
                 if (pool.state.lastUsed) {
                   const _retry = parseRetryAfter(message);
@@ -1158,7 +1018,8 @@ export function apply(ctx, config = {}) {
         }
       }
 
-      yield lastFailure ?? finishError('TRANSPORT', 'dsh-key-rotation: all keys failed');
+      const exhaustionMsg = formatExhaustionMessage(options.provider, pool);
+      yield lastFailure ?? finishError('QUOTA', exhaustionMsg);
     })();
   }
 
@@ -1328,7 +1189,6 @@ export function apply(ctx, config = {}) {
         const exportable = { ...value };
         // token-shaped fields stay empty in the file; refs are names, not secrets
         exportable.webhookActionToken = '';
-        if (exportable.incidentGitHubToken) exportable.incidentGitHubToken = '';
         json(res, 200, { at: Date.now(), version: 1, snapshot: exportable });
         return;
       }
@@ -1340,7 +1200,6 @@ export function apply(ctx, config = {}) {
       // #200 leak guard applies to imported content too
       try {
         const masked = structuredClone(snap);
-        if (masked.incidentGitHubToken) masked.incidentGitHubToken = '***';
         if (masked.webhookActionToken) masked.webhookActionToken = '***';
         if (masked.notifyWebhook) masked.notifyWebhook = '***';
         const findings = findSecrets(JSON.stringify(masked));
@@ -1354,7 +1213,6 @@ export function apply(ctx, config = {}) {
       // empty token fields in the file keep the current values (never wipe a secret)
       const merged = { ...cur, ...snap };
       if (!snap.webhookActionToken) merged.webhookActionToken = cur.webhookActionToken ?? '';
-      if (!snap.incidentGitHubToken) merged.incidentGitHubToken = cur.incidentGitHubToken ?? '';
       try {
         await settings.replace(NS, merged, desc.revision);
         const after = descriptorOf(ctx, NS);
@@ -1600,61 +1458,7 @@ export function apply(ctx, config = {}) {
     },
   }), 'dsh-key-rotation: sandbox cache');
 
-  // Auto-incident reset (#8).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: INCIDENT_RESET_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: incident-reset is local-only' } }); return; }
-      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-      readJson(req).then((body) => {
-        const provider = typeof body?.provider === 'string' ? body.provider : '';
-        if (provider) incidentReporter.resetCooldown(provider);
-        else incidentReporter.resetCooldown();
-        json(res, 200, { ok: true, reset: provider || 'all' });
-      }).catch((e) => json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }));
-    },
-  }), 'dsh-key-rotation: incident-reset');
 
-  // #198: 1-click Health Matrix — parallel probe of all configured keys.
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: TEST_MATRIX_PATH,
-    handler: async (req, res) => {
-      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: matrix is local-only' } }); return; }
-      const cfg = getConfig();
-      const runner = ensureSandboxRunner();
-      if (!runner) { json(res, 500, { error: { code: 'no-runner', message: 'sandbox runner unavailable' } }); return; }
-      const providers = Array.isArray(cfg?.providers) ? cfg.providers : [];
-      const jobs = [];
-      for (const p of providers) {
-        for (const ref of (p.keys ?? [])) {
-          if (typeof ref !== 'string' || !ref) continue;
-          jobs.push((async () => {
-            try {
-              const probeResult = await runner.probeModels(ref, ref);
-              return { provider: p.provider, ref, ok: probeResult.ok, code: probeResult.code, latencyMs: probeResult.latencyMs, modelsCount: probeResult.modelsCount ?? 0 };
-            } catch (e) {
-              return { provider: p.provider, ref, ok: false, code: 'error', latencyMs: 0, modelsCount: 0 };
-            }
-          })());
-        }
-      }
-      const results = await Promise.all(jobs);
-      json(res, 200, { at: Date.now(), total: results.length, ok: results.filter(r => r.ok).length, results });
-    },
-  }), 'dsh-key-rotation: test-matrix');
-
-  // Webhook test endpoint (#10): dry-run that validates webhookSender setup.
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: WEBHOOK_TEST_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: webhook-test is local-only' } }); return; }
-      json(res, 200, { ok: true, snapshot: webhookSender.snapshot() });
-    },
-  }), 'dsh-key-rotation: webhook-test');
 
   // #199 webhook-action: interactive webhook buttons call back here.
   // Auth: bearer token from Config (external services like Telegram/Discord
@@ -1743,41 +1547,9 @@ export function apply(ctx, config = {}) {
     },
   }), 'dsh-key-rotation: webhook-action');
 
-  // Shadow A/B sampling snapshot (#9).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: SHADOW_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: shadow is local-only' } }); return; }
-      json(res, 200, shadowRouter.snapshot());
-    },
-  }), 'dsh-key-rotation: shadow');
 
-  // Region tags + failover chain (#4).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: REGIONS_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: regions is local-only' } }); return; }
-      const body = regionMap.snapshot();
-      // Add pickFallback hints per provider for inspection.
-      const out = {};
-      for (const p of Object.keys(body)) out[p] = { region: body[p], fallback: regionMap.pickFallback(p) };
-      json(res, 200, out);
-    },
-  }), 'dsh-key-rotation: regions');
-
-  // Per-agent rate budget snapshot (#3).
-  ctx.effect(() => ctx.webServer.register({
-    kind: 'exact',
-    path: AGENT_BUDGET_PATH,
-    handler: (req, res) => {
-      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: agent-budget is local-only' } }); return; }
-      json(res, 200, { enabled: agentBudget.isEnabled(), agents: agentBudget.snapshot() });
-    },
-  }), 'dsh-key-rotation: agent-budget');
 
-  ctx.on('llm/stream', (options, next) => {
+  ctx.effect(() => ctx.on('llm/stream', (options, next) => {
     if (options[MARKER]) return next();
     if (rotationDisabled) return next(); // #199: disabled via webhook action
     const { providerToPool, modelPoolByProvider } = buildRuntime();
@@ -1786,13 +1558,13 @@ export function apply(ctx, config = {}) {
     if (!pool) return next();
     console.warn(`[dsh-key-rotation] rotating ${options.provider}/${options.model} across ${(pool.weightedRefs ?? pool.refs).length} slots (${pool.refs.length} keys)`);
     return rotate(options, pool);
-  });
+  }), 'dsh-key-rotation: llm/stream');
 
   // Safety net for non-stream requests (agent/request-error waterfall).
   // llm/stream covers streaming calls; sync calls (embeddings, batch) go
   // through agent/request and surface errors here. If the error is
   // switchable, mark the key and ask the agent loop to retry.
-  ctx.on('agent/request-error', async (payload, next) => {
+  ctx.effect(() => ctx.on('agent/request-error', async (payload, next) => {
     const provider = payload?.provider ?? payload?.failure?.provider ?? '';
     if (!provider) return next();
     const { providerToPool, modelPoolByProvider, switchCodes } = buildRuntime();
@@ -1803,7 +1575,7 @@ export function apply(ctx, config = {}) {
     const code = String(payload?.failure?.code ?? payload?.code ?? '');
     const message = String(payload?.failure?.message ?? payload?.message ?? '');
     const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-    const switchable = effectiveSwitchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message);
+    const switchable = isSwitchableError(payload, effectiveSwitchCodes);
     if (!switchable) return next();
     const ref = pool.state.lastUsed;
     if (ref) {
@@ -1815,7 +1587,7 @@ export function apply(ctx, config = {}) {
       console.warn(`[dsh-key-rotation] ${provider}: key ${String(ref)} failed via agent/request-error (${String(code)} ${String(message).slice(0, 80)}) — retry`);
     }
     return { kind: 'retry' };
-  });
+  }), 'dsh-key-rotation: agent/request-error');
 
   ctx.inject(['settings'], (sctx) => {
     const scope = sctx.settings.register(NS, Config, { base: config });
@@ -1826,18 +1598,15 @@ export function apply(ctx, config = {}) {
   });
 }
 
-// Notify on exhaustion: webhook + (optional) GitHub incident.
+// Notify on exhaustion: webhook notification.
 // Extracted at module scope for testability. No I/O outside the injected hooks.
 // ponytail: thresholds and URLs are runtime-resolved per call, so changing Config is reflected immediately.
-export function notifyExhaustion(runtime, pool, options, hooks = { webhookSender, ensureIncidentReporter }) {
+export function notifyExhaustion(runtime, pool, options, hooks = { webhookSender }) {
   if (!runtime || !pool) return;
   const count = pool.state ? (pool.state.exhaustionCount ?? 0) : 0;
   if (count <= 0) return;
   try {
     if (runtime.notifyWebhook && count >= (runtime.notifyThreshold ?? 0)) {
-      // #199: interactive payload when an action token is configured - the
-      // platform formatter (webhook.js) turns `actions` into buttons whose
-      // callback carries the token back to /dsh-key-rotation/webhook-action.
       const token = runtime.webhookActionToken ?? '';
       const payload = {
         title: `Key pool exhausted: ${options.provider}`,
@@ -1854,10 +1623,6 @@ export function notifyExhaustion(runtime, pool, options, hooks = { webhookSender
       };
       hooks.webhookSender.send(runtime.notifyWebhook, payload);
     }
-    if (runtime.incidentThreshold && count >= runtime.incidentThreshold) {
-      const reporter = hooks.ensureIncidentReporter();
-      if (reporter) reporter.open(options.provider, pool.state.lastExhaustionAt);
-    }
   } catch (_) { /* ponytail: never crash rotate() */ }
 }