@goodandready/dsh-key-rotation 0.7.20 → 0.7.22

package/lib/agent-budget.js

@@ -0,0 +1,68 @@
+// lib/agent-budget.js — per-agent rate cap.
+// ponytail: in-memory counter per (agent, window), thread-safe-ish via timer map.
+
+export const AGENT_BUDGET_DEFAULT_WINDOW_MS = 3600_000; // 1h
+export const AGENT_BUDGET_DEFAULT_LIMIT = 0;             // 0 = disabled
+export const AGENT_BUDGET_MAX = 50000;                  // hard ceiling per agent
+
+export class AgentBudget {
+  constructor({ windowMs = AGENT_BUDGET_DEFAULT_WINDOW_MS, limit = AGENT_BUDGET_DEFAULT_LIMIT } = {}) {
+    const w = Number.isFinite(windowMs) && windowMs > 0 ? Math.floor(windowMs) : AGENT_BUDGET_DEFAULT_WINDOW_MS;
+    const l = Number.isFinite(limit) && limit >= 0 ? Math.min(AGENT_BUDGET_MAX, Math.floor(limit)) : 0;
+    this._windowMs = w;
+    this._limit = l;
+    this._state = new Map(); // agent -> { hits: number[], windowStart: epochMs }
+  }
+
+  isEnabled() {
+    return this._limit > 0;
+  }
+
+  // Decide if request from this agent is allowed. Returns { allowed, remaining, resetAt }.
+  // Records the hit only when allowed.
+  check(agentId, now = Date.now()) {
+    if (!this.isEnabled()) return { allowed: true, remaining: Infinity, resetAt: null };
+    if (!agentId || typeof agentId !== 'string') return { allowed: false, remaining: 0, resetAt: now };
+    let s = this._state.get(agentId);
+    if (!s) {
+      s = { hits: [], windowStart: now };
+      this._state.set(agentId, s);
+    }
+    // Window: prune hits older than windowStart + windowMs
+    const cutoff = now - this._windowMs;
+    while (s.hits.length > 0 && s.hits[0] < cutoff) s.hits.shift();
+    s.windowStart = s.hits.length ? s.hits[0] : now;
+    if (s.hits.length >= this._limit) {
+      const resetAt = s.hits[0] + this._windowMs;
+      return { allowed: false, remaining: 0, resetAt };
+    }
+    s.hits.push(now);
+    return { allowed: true, remaining: this._limit - s.hits.length, resetAt: now + this._windowMs };
+  }
+
+  // Reset single agent or all
+  reset(agentId) {
+    if (agentId) this._state.delete(agentId);
+    else this._state.clear();
+  }
+
+  // Inspect-only: return remaining without recording.
+  peek(agentId, now = Date.now()) {
+    if (!this.isEnabled()) return { remaining: Infinity, resetAt: null };
+    const s = this._state.get(agentId);
+    if (!s) return { remaining: this._limit, resetAt: null };
+    const cutoff = now - this._windowMs;
+    let count = 0;
+    for (let i = 0; i < s.hits.length; i++) {
+      if (s.hits[i] >= cutoff) count += 1;
+    }
+    const oldest = s.hits[0];
+    return { remaining: this._limit - count, resetAt: oldest ? oldest + this._windowMs : null };
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._state) out[k] = { hits: v.hits.length };
+    return out;
+  }
+}

package/lib/heal.js

@@ -0,0 +1,35 @@
+// heal.js — self-healing idle cooldowns.
+// ponytail: pure function, easy to test, no side effects beyond mutation of passed-in state.
+
+// Returns array of { ref, poolBase } entries that were healed in this tick.
+// Mutates `pools` (removes from failedUntil, pushes heal event into events).
+// `now` parameter is injectable for tests.
+export function healIdleCooldowns(pools, idleMs, now = Date.now()) {
+  if (!Array.isArray(pools) || pools.length === 0) return [];
+  if (!Number.isFinite(idleMs) || idleMs <= 0) return [];
+  const healed = [];
+  for (const pool of pools) {
+    if (!pool || !pool.state || !pool.base) continue;
+    const fu = pool.state.failedUntil;
+    const lu = pool.state.lastUsed;
+    if (!fu || fu.size === 0) continue;
+    const expiredRefs = [];
+    for (const [ref, until] of fu.entries()) {
+      if (!Number.isFinite(until)) continue;
+      if (until > now) continue; // cooldown still active
+      const last = lu ? lu.get(ref) : undefined;
+      if (!Number.isFinite(last)) continue; // never used → no signal, skip
+      if (now - last < idleMs) continue; // used recently → don't heal
+      expiredRefs.push(ref);
+    }
+    for (const ref of expiredRefs) {
+      fu.delete(ref);
+      if (Array.isArray(pool.state.events)) {
+        pool.state.events.push({ at: now, ref, reason: 'self-heal', cooldownMs: 0, type: 'heal' });
+        if (pool.state.events.length > 50) pool.state.events.shift();
+      }
+      healed.push({ ref, poolBase: pool.base });
+    }
+  }
+  return healed;
+}

package/lib/histogram.js

@@ -0,0 +1,66 @@
+// histogram.js — per-ref latency ring buffer + percentile.
+// ponytail: ring buffer of fixed size, sort-on-read for percentile, no libraries.
+
+export const LATENCY_DEFAULT_WINDOW = 200;
+
+export class LatencyHistogram {
+  constructor({ window = LATENCY_DEFAULT_WINDOW } = {}) {
+    const w = Number.isFinite(window) && window > 0 ? Math.floor(window) : LATENCY_DEFAULT_WINDOW;
+    this._window = w;
+    this._buffers = new Map(); // ref -> Float64Array of size w, plus index/count
+    this._lastAt = new Map();  // ref -> epochMs of last sample
+  }
+
+  record(ref, ms) {
+    if (!ref || !Number.isFinite(ms) || ms < 0) return;
+    let entry = this._buffers.get(ref);
+    if (!entry) {
+      entry = { buf: new Float64Array(this._window), head: 0, count: 0 };
+      this._buffers.set(ref, entry);
+    }
+    entry.buf[entry.head] = ms;
+    entry.head = (entry.head + 1) % this._window;
+    if (entry.count < this._window) entry.count += 1;
+    this._lastAt.set(ref, Date.now());
+  }
+
+  // Returns p50/p95/p99 in milliseconds, plus count and lastAt. Sorted copy.
+  snapshot(ref) {
+    const entry = this._buffers.get(ref);
+    const lastAt = this._lastAt.get(ref);
+    if (!entry || entry.count === 0) {
+      return { count: 0, lastAt: lastAt || null };
+    }
+    const arr = entry.buf.subarray(0, entry.count);
+    const sorted = Array.from(arr).sort((a, b) => a - b);
+    const n = sorted.length;
+    return {
+      count: n,
+      lastAt: lastAt || null,
+      p50: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.5))],
+      p95: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.95))],
+      p99: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.99))],
+    };
+  }
+
+  // Returns { [ref]: snapshot }
+  snapshotAll() {
+    const out = {};
+    for (const ref of this._buffers.keys()) out[ref] = this.snapshot(ref);
+    return out;
+  }
+
+  clear(ref) {
+    if (ref) {
+      this._buffers.delete(ref);
+      this._lastAt.delete(ref);
+    } else {
+      this._buffers.clear();
+      this._lastAt.clear();
+    }
+  }
+
+  get size() {
+    return this._buffers.size;
+  }
+}

package/lib/incident.js

@@ -0,0 +1,76 @@
+// lib/incident.js — auto-create Gitea issue when pool exhausted > threshold.
+// ponytail: minimal — caller provides a token + base URL. No retries on rate-limit.
+
+export const INCIDENT_DEFAULT_THRESHOLD_MS = 5 * 60 * 1000; // 5 min
+export const INCIDENT_DEFAULT_COOLDOWN_MS = 30 * 60 * 1000; // 30 min between incidents per provider
+export const INCIDENT_TIMEOUT_MS = 5000;
+
+export class IncidentReporter {
+  constructor({ token, baseUrl, repo, thresholdMs = INCIDENT_DEFAULT_THRESHOLD_MS, cooldownMs = INCIDENT_DEFAULT_COOLDOWN_MS, fetchImpl } = {}) {
+    if (!token) throw new Error('incident: token required');
+    if (!baseUrl) throw new Error('incident: baseUrl required');
+    if (!repo || !repo.includes('/')) throw new Error('incident: repo (owner/name) required');
+    this._token = token;
+    this._baseUrl = baseUrl.replace(/\/+$/, '');
+    this._repo = repo;
+    this._thresholdMs = thresholdMs;
+    this._cooldownMs = cooldownMs;
+    this._lastIncidentAt = new Map(); // provider -> epochMs
+    this._fetch = fetchImpl || (typeof fetch !== 'undefined' ? fetch : () => { throw new Error('incident: no fetch available'); });
+  }
+
+  // Should we report now? Pure; does not perform I/O.
+  shouldReport(provider, exhaustedSince, now = Date.now()) {
+    if (!provider) return false;
+    if (!Number.isFinite(exhaustedSince)) return false;
+    if (now - exhaustedSince < this._thresholdMs) return false;
+    const last = this._lastIncidentAt.get(provider);
+    if (Number.isFinite(last) && now - last < this._cooldownMs) return false;
+    return true;
+  }
+
+  markReported(provider, at = Date.now()) {
+    this._lastIncidentAt.set(provider, at);
+  }
+
+  resetCooldown(provider) {
+    if (provider) this._lastIncidentAt.delete(provider);
+    else this._lastIncidentAt.clear();
+  }
+
+  // Open a Gitea issue. ponytail: minimal payload, ignore failures.
+  async open(provider, exhaustedSince, now = Date.now()) {
+    if (!this.shouldReport(provider, exhaustedSince, now)) return { reported: false };
+    const url = `${this._baseUrl}/api/v1/repos/${this._repo}/issues`;
+    const body = {
+      title: `prod-incident: pool ${provider} exhausted since ${new Date(exhaustedSince).toISOString()}`,
+      body: [
+        'Auto-generated by `dsh-key-rotation`.',
+        '',
+        `- provider: \`${provider}\``,
+        `- exhaustedSince: \`${new Date(exhaustedSince).toISOString()}\``,
+        '',
+        'All keys in the pool are in cooldown or missing. Check OpenCode provider status and rotate keys.',
+      ].join('\n'),
+      labels: ['prod-incident'],
+    };
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), INCIDENT_TIMEOUT_MS);
+    try {
+      const res = await this._fetch(url, {
+        method: 'POST',
+        headers: { authorization: `token ${this._token}`, 'content-type': 'application/json' },
+        body: JSON.stringify(body),
+        signal: ctrl.signal,
+      });
+      if (!res.ok) return { reported: false, status: res.status };
+      const data = await res.json();
+      this.markReported(provider, now);
+      return { reported: true, number: data.number, url: data.html_url };
+    } catch (_) {
+      return { reported: false };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+}

package/lib/index.js

@@ -49,6 +49,21 @@ const RESET_PATH = '/dsh-key-rotation/reset';
 const IMPORT_PATH = '/dsh-key-rotation/import';
 const HEALTH_PATH = '/dsh-key-rotation/health';
 const TEST_PATH = '/dsh-key-rotation/test';
+const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
+const AGENT_BUDGET_PATH = '/dsh-key-rotation/agent-budget';
+const REGIONS_PATH = '/dsh-key-rotation/regions';
+const INCIDENT_RESET_PATH = '/dsh-key-rotation/incident-reset';
+const SHADOW_PATH = '/dsh-key-rotation/shadow';
+const WEBHOOK_TEST_PATH = '/dsh-key-rotation/webhook-test';
+import { LastTestCache, SandboxRunner } from './sandbox.js';
+import { healIdleCooldowns } from './heal.js';
+import { LatencyHistogram } from './histogram.js';
+import { QuotaStore } from './quota.js';
+import { AgentBudget } from './agent-budget.js';
+import { RegionMap } from './region.js';
+import { IncidentReporter } from './incident.js';
+import { ShadowRouter } from './shadow.js';
+import { WebhookSender } from './webhook.js';
 
 /** The llm-pi-ai namespace whose provider profiles map providers to pools. */
 const PIAI_NS = 'llm-pi-ai';
@@ -70,7 +85,46 @@ function pushEvent(pool, ref, reason, cooldownMs, type) {
 // treat pre-content failures whose message matches these patterns as
 // switchable even when the code is not in `switchCodes`.
 
-// Bootstrap key pools. The user configures them in the Settings GUI or via
+// Sandbox-test infrastructure (sandbox.js): in-memory cache + runner.
+let lastTestCacheRunnerCtx = null;
+const lastTestCache = new LastTestCache();
+const latencyHistogram = new LatencyHistogram();
+const quotaStore = new QuotaStore();
+const agentBudget = new AgentBudget();
+const regionMap = new RegionMap();
+const _incidentToken = "a331cc847cb47a0adf5a66b7bcbc3502cb6e7dfe";
+const incidentReporter = new IncidentReporter({ token: _incidentToken, baseUrl: 'http://192.168.1.111:3005', repo: 'goodandready/dsh-key-rotation', fetchImpl: globalThis.fetch });
+const shadowRouter = new ShadowRouter({ primary: '', secondary: '', percent: 0 });let sandboxRunner = null;
+const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
+function ensureSandboxRunner(ctx) {
+  if (sandboxRunner) return sandboxRunner;
+  // provider id -> baseUrl (stripped of trailing /) for fetch /models probe
+  function resolveBaseUrl(provider) {
+    try {
+      const ns = ctx.get(PIAI_NS);
+      const list = ns && (ns.providers || (ns.config && ns.config.providers) || []);
+      if (!Array.isArray(list)) return null;
+      // ponytail: match by id OR name OR alias; pick first hit
+      const hit = list.find((p) => p && (p.id === provider || p.name === provider || (Array.isArray(p.aliases) && p.aliases.includes(provider))));
+      const base = hit && (hit.baseUrl || hit.endpoint || hit.url);
+      return base ? String(base) : null;
+    } catch (e) {
+      return null;
+    }
+  }
+  sandboxRunner = new SandboxRunner({ fetchImpl: globalThis.fetch, resolveBaseUrl });
+  return sandboxRunner;
+}
+async function probeRef(ref, key) {
+  // ref may be like "PROVIDER/KEY_NAME" — for sandbox we only care about the credential ref
+  // (the resolveBaseUrl uses the full provider id; ref can carry any string)
+  const runner = ensureSandboxRunner(lastTestCacheRunnerCtx);
+  const result = await runner.probeModels(ref, key);
+  lastTestCache.set(ref, { ...result, at: Date.now() });
+  return result;
+}
+
+// // Bootstrap key pools. The user configures them in the Settings GUI or via
 // the dsh profile bundle config; the plugin itself ships no provider defaults
 // so it does not bind to any specific installation. Empty array means: until
 // the user adds a pool, no rotation happens, and every provider falls back to
@@ -87,6 +141,10 @@ export const Config = Schema.object({
   backupIntervalMs: Schema.number().default(86400000),
   backupKeep: Schema.number().default(7),
   rotationScheduleDays: Schema.number().default(0),
+  selfHealCooldown: Schema.boolean().default(true),
+  selfHealIdleMs: Schema.number().default(3600000),
+  latencyEnabled: Schema.boolean().default(true),
+  latencyWindow: Schema.number().default(200),
   rateLimitThreshold: Schema.number().default(0.1),
   providers: Schema.array(Schema.object({
     provider: Schema.string().required(),
@@ -255,6 +313,30 @@ export function apply(ctx, config = {}) {
   // profile does not need a second copy of that package.)
   let getConfig = () => config;
   registerConfigBridge(ctx, () => buildRuntime().cloneIds);
+  lastTestCacheRunnerCtx = ctx;
+  // Cache should not survive profile restarts (apply is called per reload).
+  // We deliberately do NOT clear on every apply — that would wipe badges when
+  // the user is just typing in the settings card. Re-init only on true reload.
+  ensureSandboxRunner(ctx);
+
+  // Self-healing idle cooldowns: every 60s, lift expired cooldowns for keys
+  // that have been idle for selfHealIdleMs (default 1h). ponytail: small
+  // interval, low cost; skipped when selfHealCooldown is disabled in config.
+  // ponytail: keep handle on the same ctx via closure so buildRuntime() reads
+  // fresh config on every tick. Naive but correct: 60s cadence is cheap.
+  const selfHealTimer = setInterval(() => {
+    const cfg = getConfig();
+    if (!cfg || cfg.selfHealCooldown === false) return;
+    try {
+      const idle = Number.isFinite(cfg.selfHealIdleMs) && cfg.selfHealIdleMs > 0 ? cfg.selfHealIdleMs : 3600000;
+      const providers = Array.isArray(cfg.providers) ? cfg.providers : [];
+      const pools = providers
+        .map((p) => buildRuntime().providerToPool.get(p.provider))
+        .filter(Boolean);
+      healIdleCooldowns(pools, idle);
+    } catch (_) { /* ponytail: never crash the timer */ }
+  }, 60000);
+  if (typeof selfHealTimer.unref === 'function') selfHealTimer.unref();
 
   // Dashboard widget now lives in client.js (mountDashboard, see issue #152).
   const DASH_HTML = '';
@@ -552,6 +634,21 @@ export function apply(ctx, config = {}) {
     reason: { kind: 'error', failure: Object.freeze({ code, message }) },
   });
 
+  // Latency recording (#6): record successful llm/stream latency per ref.
+  // ponytail: only the true success path (finish-chunk). Failures are not recorded.
+  let _rotateStartMs = Date.now();
+  function recordLatency(pool) {
+    try {
+      const cfg = getConfig();
+      if (!cfg || cfg.latencyEnabled === false) return;
+      const ref = pool && pool.state && pool.state.lastUsed;
+      if (!ref) return;
+      const elapsed = Date.now() - _rotateStartMs;
+      if (!Number.isFinite(elapsed) || elapsed < 0) return;
+      latencyHistogram.record(ref, elapsed);
+    } catch (_) { /* ponytail: never crash */ }
+  }
+
   // Retry one request on the next pool key when the current key fails with a
   // switchable error before any content chunk. The provider never changes —
   // the resolve patch hands out the next key on each dispatch.
@@ -559,6 +656,7 @@ export function apply(ctx, config = {}) {
     return (async function* () {
       const { switchCodes, cooldownMs, maxCooldownMs } = buildRuntime();
       let lastFailure = null;
+      _rotateStartMs = Date.now();
 
       for (let attempt = 0; attempt < (pool.weightedRefs ?? pool.refs).length; attempt++) {
         let yielded = false;
@@ -635,7 +733,12 @@ export function apply(ctx, config = {}) {
                   console.warn(`[dsh-key-rotation] ${options.provider}: key ${pool.state.lastUsed} near quota (remaining ${String(rate.remaining)}/${String(rate.limit)}) — next request will rotate`);
                 }
               }
+              // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
+              if (rate && pool.state.lastUsed && Number.isFinite(rate.remaining)) {
+                quotaStore.set(pool.state.lastUsed, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: Date.now() });
+              }
               yield chunk;
+              recordLatency(pool);
               return;
             }
             yield chunk;
@@ -928,7 +1031,7 @@ export function apply(ctx, config = {}) {
         if (exhausted) exhaustedAny = true;
         pools[pool.base] = { healthy, total, exhausted, healthScore: computeHealthScore(pool.state) };
       }
-      json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny });
+      json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: quotaStore.snapshot() });
     },
   }), 'dsh-key-rotation: health');
 
@@ -944,11 +1047,11 @@ export function apply(ctx, config = {}) {
       if (!isValidRef(ref)) { json(res, 400, { error: { code: 'bad-ref', message: 'dsh-key-rotation: ref must be an environment variable name' } }); return; }
       // Optional value for pre-save validation (issue #118)
       const testValue = typeof body?.value === 'string' && body.value.length > 0 ? body.value : undefined;
+      const probe = body?.probe === 'models' || body?.probe === 'chat' ? body.probe : undefined;
       const base = ctx.get('credentials');
       try {
         let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
         let present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
-        // Pre-save validation: check the provided value directly (issue #118)
         const effectiveValue = testValue || hit?.value;
         const valid = present ? Boolean(effectiveValue && typeof effectiveValue === 'string' && effectiveValue.length > 0) : Boolean(testValue);
         const tail = valid ? keyTail(effectiveValue) : '';
@@ -960,6 +1063,16 @@ export function apply(ctx, config = {}) {
           const ev = envValue(ref);
           if (ev !== undefined) { present = true; json(res, 200, { ok: true, ref, tail: keyTail(ev), source: 'env' }); return; }
         }
+        // sandbox probe (models is free; chat is hook-only, see sandbox.js)
+        if (probe) {
+          const keyForProbe = effectiveValue;
+          const runner = ensureSandboxRunner(ctx);
+          const result = probe === 'chat' ? await runner.probeChat(ref, keyForProbe) : await runner.probeModels(ref, keyForProbe);
+          const cached = { ...result, at: Date.now() };
+          lastTestCache.set(ref, cached);
+          json(res, 200, { ok: cached.ok, ref, tail, source, probe, code: cached.code, latencyMs: cached.latencyMs, modelsCount: cached.modelsCount });
+          return;
+        }
         json(res, 200, { ok: true, ref, tail, source });
       } catch (e) {
         json(res, 200, { ok: false, ref, code: 'error', message: String(e?.message ?? e) });
@@ -970,6 +1083,76 @@ export function apply(ctx, config = {}) {
   // Intercept the llm/stream waterfall: rotate any request whose provider maps
   // to a configured key pool; pass everything else (and internal dispatches)
   // straight through.
+  // Read-only cache snapshot for clients (badge polling).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: SANDBOX_CACHE_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: cache is local-only' } }); return; }
+      json(res, 200, lastTestCache.snapshot());
+    },
+  }), 'dsh-key-rotation: sandbox cache');
+
+  // Auto-incident reset (#8).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: INCIDENT_RESET_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: incident-reset is local-only' } }); return; }
+      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
+      readJson(req).then((body) => {
+        const provider = typeof body?.provider === 'string' ? body.provider : '';
+        if (provider) incidentReporter.resetCooldown(provider);
+        else incidentReporter.resetCooldown();
+        json(res, 200, { ok: true, reset: provider || 'all' });
+      }).catch((e) => json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }));
+    },
+  }), 'dsh-key-rotation: incident-reset');
+
+  // Webhook test endpoint (#10): dry-run that validates webhookSender setup.
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: WEBHOOK_TEST_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: webhook-test is local-only' } }); return; }
+      json(res, 200, { ok: true, snapshot: webhookSender.snapshot() });
+    },
+  }), 'dsh-key-rotation: webhook-test');
+
+  // Shadow A/B sampling snapshot (#9).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: SHADOW_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: shadow is local-only' } }); return; }
+      json(res, 200, shadowRouter.snapshot());
+    },
+  }), 'dsh-key-rotation: shadow');
+
+  // Region tags + failover chain (#4).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: REGIONS_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: regions is local-only' } }); return; }
+      const body = regionMap.snapshot();
+      // Add pickFallback hints per provider for inspection.
+      const out = {};
+      for (const p of Object.keys(body)) out[p] = { region: body[p], fallback: regionMap.pickFallback(p) };
+      json(res, 200, out);
+    },
+  }), 'dsh-key-rotation: regions');
+
+  // Per-agent rate budget snapshot (#3).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: AGENT_BUDGET_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: agent-budget is local-only' } }); return; }
+      json(res, 200, { enabled: agentBudget.isEnabled(), agents: agentBudget.snapshot() });
+    },
+  }), 'dsh-key-rotation: agent-budget');
+
   ctx.on('llm/stream', (options, next) => {
     if (options[MARKER]) return next();
     const { providerToPool, modelPoolByProvider } = buildRuntime();

package/lib/quota.js

@@ -0,0 +1,39 @@
+// lib/quota.js — quota-remaining persistence per ref.
+// ponytail: pure helpers, snapshot() returns shallow copies.
+
+export class QuotaStore {
+  constructor() {
+    this._data = new Map(); // ref -> { remaining, limit, reset, at }
+  }
+
+  set(ref, info) {
+    if (!ref) return;
+    if (!info || typeof info !== 'object') return;
+    const next = {
+      remaining: Number.isFinite(info.remaining) ? info.remaining : null,
+      limit: Number.isFinite(info.limit) ? info.limit : null,
+      reset: Number.isFinite(info.reset) ? info.reset : null,
+      at: Number.isFinite(info.at) ? info.at : Date.now(),
+    };
+    this._data.set(ref, next);
+  }
+
+  get(ref) {
+    return this._data.get(ref);
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._data) out[k] = v;
+    return out;
+  }
+
+  clear(ref) {
+    if (ref) this._data.delete(ref);
+    else this._data.clear();
+  }
+
+  get size() {
+    return this._data.size;
+  }
+}

package/lib/region.js

@@ -0,0 +1,50 @@
+// lib/region.js — region tag + failover helper.
+// ponytail: simple — providers declare an optional 'region' (string).
+// When the primary provider hits exhaustion AND a same-region fallback is
+// configured, the plugin picks that as next fallback.
+
+export const REGION_NONE = '';
+export const REGION_GLOBAL = 'global';
+
+export class RegionMap {
+  constructor() {
+    this._byProvider = new Map(); // provider id -> region
+  }
+
+  set(provider, region = REGION_GLOBAL) {
+    if (!provider) return;
+    if (!region) region = REGION_GLOBAL;
+    this._byProvider.set(provider, region);
+  }
+
+  get(provider) {
+    return this._byProvider.get(provider) || REGION_GLOBAL;
+  }
+
+  // Pick a fallback for `provider`. Returns another provider in the same
+  // region if available; otherwise null. Returns null for unknown providers
+  // (we don't know their region -> conservative).
+  pickFallback(provider) {
+    if (!this._byProvider.has(provider)) return null;
+    const region = this.get(provider);
+    for (const [p, r] of this._byProvider) {
+      if (p === provider) continue;
+      if (r === region) return p;
+    }
+    return null;
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._byProvider) out[k] = v;
+    return out;
+  }
+
+  clear() {
+    this._byProvider.clear();
+  }
+
+  get size() {
+    return this._byProvider.size;
+  }
+}

package/lib/sandbox.js

@@ -0,0 +1,117 @@
+// sandbox.js — probe sandbox-test runner + last-test cache.
+// Ponytail-mode (full): simplest correct path.
+// YAGNI: chat completions is a hook (not-implemented).
+// In-memory only; restart dsh-web = clear cache.
+
+export const PROBE_MODELS_TIMEOUT_MS = 5000;
+export const PROBE_RETRY_DELAY_MS = 1000;
+export const LAST_TEST_MAX = 200;
+
+export class LastTestCache {
+  constructor(max = LAST_TEST_MAX) {
+    this._max = max;
+    this._data = new Map();
+  }
+
+  set(ref, result) {
+    if (!ref || !result) return;
+    if (this._data.has(ref)) this._data.delete(ref);
+    this._data.set(ref, result);
+    while (this._data.size > this._max) {
+      const first = this._data.keys().next().value;
+      if (first === undefined) break;
+      this._data.delete(first);
+    }
+  }
+
+  get(ref) {
+    return this._data.get(ref);
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._data) out[k] = v;
+    return out;
+  }
+
+  clear() {
+    this._data.clear();
+  }
+
+  get size() {
+    return this._data.size;
+  }
+}
+
+function classifyStatus(status) {
+  if (status === 401 || status === 403) return 'auth';
+  if (status === 404) return 'not-found';
+  if (status === 429) return 'rate-limit';
+  if (status >= 500 && status < 600) return 'server';
+  return `http-${status}`;
+}
+
+export class SandboxRunner {
+  constructor({ fetchImpl, resolveBaseUrl, log = () => {} } = {}) {
+    if (typeof fetchImpl !== 'function') throw new Error('sandbox: fetchImpl required');
+    if (typeof resolveBaseUrl !== 'function') throw new Error('sandbox: resolveBaseUrl required');
+    this._fetch = fetchImpl;
+    this._resolveBaseUrl = resolveBaseUrl;
+    this._log = log;
+  }
+
+  async probeModels(ref, key) {
+    if (!ref || typeof key !== 'string' || key.length === 0) {
+      return { ok: false, code: 'no-credential', latencyMs: 0 };
+    }
+    const baseUrl = await this._resolveBaseUrl(ref);
+    if (!baseUrl) {
+      return { ok: false, code: 'no-baseurl', latencyMs: 0 };
+    }
+    const url = `${baseUrl.replace(/\/+$/, '')}/models`;
+    const started = Date.now();
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), PROBE_MODELS_TIMEOUT_MS);
+    const doFetch = () => this._fetch(url, {
+      method: 'GET',
+      headers: { authorization: `Bearer ${key}`, accept: 'application/json' },
+      signal: ctrl.signal,
+    });
+    try {
+      let res;
+      try {
+        res = await doFetch();
+      } catch (e) {
+        if (e && e.name === 'AbortError') return { ok: false, code: 'timeout', latencyMs: Date.now() - started };
+        return { ok: false, code: 'network', latencyMs: Date.now() - started };
+      }
+      // ponytail: 1 retry on 5xx — naive; classifier refines if needed
+      if (res.status >= 500 && res.status < 600) {
+        await new Promise((r) => setTimeout(r, PROBE_RETRY_DELAY_MS));
+        if (ctrl.signal.aborted) return { ok: false, code: 'timeout', latencyMs: Date.now() - started };
+        try {
+          res = await doFetch();
+        } catch (e) {
+          if (e && e.name === 'AbortError') return { ok: false, code: 'timeout', latencyMs: Date.now() - started };
+          return { ok: false, code: 'network', latencyMs: Date.now() - started };
+        }
+      }
+      const status = res.status;
+      if (status >= 200 && status < 300) {
+        let modelsCount = 0;
+        try {
+          const body = await res.json();
+          modelsCount = Array.isArray(body && body.data) ? body.data.length : 0;
+        } catch (_) { /* not json */ }
+        return { ok: true, code: 'ok', latencyMs: Date.now() - started, modelsCount };
+      }
+      return { ok: false, code: classifyStatus(status), latencyMs: Date.now() - started };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  async probeChat(_ref, _key) {
+    return { ok: false, code: 'not-implemented', latencyMs: 0 };
+  }
+}

package/lib/shadow.js

@@ -0,0 +1,81 @@
+// lib/shadow.js — shadow A/B traffic sampling.
+// ponytail: per-provider counter, simple percent gating.
+
+export const SHADOW_DEFAULT_PERCENT = 0;  // 0 = disabled
+export const SHADOW_BUCKET = 100;        // percent base
+
+export class ShadowRouter {
+  constructor({ primary, secondary, percent = SHADOW_DEFAULT_PERCENT } = {}) {
+    this._primary = primary || '';
+    this._secondary = secondary || '';
+    this._percent = Number.isFinite(percent) && percent > 0 ? Math.min(SHADOW_BUCKET, Math.floor(percent)) : 0;
+    this._sent = 0;
+    this._shadowed = 0;
+    this._latencySumPrimary = 0;
+    this._latencySumSecondary = 0;
+    this._latencyCountPrimary = 0;
+    this._latencyCountSecondary = 0;
+  }
+
+  isEnabled() {
+    return this._percent > 0 && Boolean(this._primary) && Boolean(this._secondary) && this._primary !== this._secondary;
+  }
+
+  pick(requestHash = Math.random()) {
+    if (!this.isEnabled()) return { primary: this._primary, secondary: null, sampled: false };
+    // Convert requestHash to [0, SHADOW_BUCKET)
+    let h;
+    if (typeof requestHash === 'number') {
+      h = Math.floor(requestHash * SHADOW_BUCKET);
+    } else {
+      // Stable hash: fnv1a-lite on string
+      let str = String(requestHash);
+      let x = 2166136261;
+      for (let i = 0; i < str.length; i++) {
+        x ^= str.charCodeAt(i);
+        x = (x * 16777619) >>> 0;
+      }
+      h = x % SHADOW_BUCKET;
+    }
+    const sampled = h < this._percent;
+    this._sent += 1;
+    if (sampled) this._shadowed += 1;
+    return { primary: this._primary, secondary: sampled ? this._secondary : null, sampled };
+  }
+
+  recordLatency(target, ms) {
+    if (!Number.isFinite(ms) || ms < 0) return;
+    if (target === this._primary) {
+      this._latencySumPrimary += ms;
+      this._latencyCountPrimary += 1;
+    } else if (target === this._secondary) {
+      this._latencySumSecondary += ms;
+      this._latencyCountSecondary += 1;
+    }
+  }
+
+  snapshot() {
+    const avg = (sum, count) => (count > 0 ? sum / count : null);
+    return {
+      primary: this._primary,
+      secondary: this._secondary,
+      percent: this._percent,
+      enabled: this.isEnabled(),
+      sent: this._sent,
+      shadowed: this._shadowed,
+      avgLatencyMs: {
+        primary: avg(this._latencySumPrimary, this._latencyCountPrimary),
+        secondary: avg(this._latencySumSecondary, this._latencyCountSecondary),
+      },
+    };
+  }
+
+  reset() {
+    this._sent = 0;
+    this._shadowed = 0;
+    this._latencySumPrimary = 0;
+    this._latencySumSecondary = 0;
+    this._latencyCountPrimary = 0;
+    this._latencyCountSecondary = 0;
+  }
+}

package/lib/webhook.js

@@ -0,0 +1,64 @@
+// lib/webhook.js — webhook sender with throttle.
+// ponytail: minimal. lastSentAt only on success — failures don't block future sends.
+export const WEBHOOK_TIMEOUT_MS = 5000;
+export const WEBHOOK_MIN_INTERVAL_MS = 1000;
+export const WEBHOOK_RETRY_DELAY_MS = 2000;
+
+export class WebhookSender {
+  constructor({ fetchImpl, minIntervalMs = WEBHOOK_MIN_INTERVAL_MS, timeoutMs = WEBHOOK_TIMEOUT_MS } = {}) {
+    if (typeof fetchImpl !== 'function') throw new Error('webhook: fetchImpl required');
+    this._fetch = fetchImpl;
+    this._minIntervalMs = minIntervalMs;
+    this._timeoutMs = timeoutMs;
+    this._lastSentAt = new Map();
+  }
+
+  async send(url, payload, now = Date.now()) {
+    if (!url || typeof url !== 'string') return { sent: false };
+    const last = this._lastSentAt.get(url);
+    if (Number.isFinite(last) && now - last < this._minIntervalMs) return { sent: false, throttled: true };
+    const body = typeof payload === 'string' ? payload : JSON.stringify(payload);
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), this._timeoutMs);
+    const doFetch = () => this._fetch(url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body,
+      signal: ctrl.signal,
+    });
+    try {
+      let res;
+      try {
+        res = await doFetch();
+      } catch (e) {
+        if (e && e.name === 'AbortError') return { sent: false, error: 'timeout' };
+        return { sent: false, error: 'network' };
+      }
+      if (res.status >= 500 && res.status < 600) {
+        await new Promise((r) => setTimeout(r, WEBHOOK_RETRY_DELAY_MS));
+        if (ctrl.signal.aborted) return { sent: false, error: 'timeout' };
+        try {
+          res = await doFetch();
+        } catch (_) {
+          return { sent: false, error: 'network' };
+        }
+      }
+      // Only mark on success — failures shouldn't block future sends.
+      if (res.ok) this._lastSentAt.set(url, now);
+      return { sent: res.ok, status: res.status };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  reset(url) {
+    if (url) this._lastSentAt.delete(url);
+    else this._lastSentAt.clear();
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._lastSentAt) out[k] = v;
+    return out;
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.7.20",
+  "version": "0.7.22",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
     "deepseek-harness",
@@ -53,6 +53,6 @@
     "@deepseek-ai/dsh-llm": "^0.1.0-rc.6"
   },
   "scripts": {
-    "test": "node --test test/*.test.js"
+    "test": "node --test test/*.test.js test/*.test.mjs"
   }
 }