@goodandready/dsh-key-rotation 0.7.21 → 0.7.22

package/lib/agent-budget.js

@@ -0,0 +1,68 @@
+// lib/agent-budget.js — per-agent rate cap.
+// ponytail: in-memory counter per (agent, window), thread-safe-ish via timer map.
+
+export const AGENT_BUDGET_DEFAULT_WINDOW_MS = 3600_000; // 1h
+export const AGENT_BUDGET_DEFAULT_LIMIT = 0;             // 0 = disabled
+export const AGENT_BUDGET_MAX = 50000;                  // hard ceiling per agent
+
+export class AgentBudget {
+  constructor({ windowMs = AGENT_BUDGET_DEFAULT_WINDOW_MS, limit = AGENT_BUDGET_DEFAULT_LIMIT } = {}) {
+    const w = Number.isFinite(windowMs) && windowMs > 0 ? Math.floor(windowMs) : AGENT_BUDGET_DEFAULT_WINDOW_MS;
+    const l = Number.isFinite(limit) && limit >= 0 ? Math.min(AGENT_BUDGET_MAX, Math.floor(limit)) : 0;
+    this._windowMs = w;
+    this._limit = l;
+    this._state = new Map(); // agent -> { hits: number[], windowStart: epochMs }
+  }
+
+  isEnabled() {
+    return this._limit > 0;
+  }
+
+  // Decide if request from this agent is allowed. Returns { allowed, remaining, resetAt }.
+  // Records the hit only when allowed.
+  check(agentId, now = Date.now()) {
+    if (!this.isEnabled()) return { allowed: true, remaining: Infinity, resetAt: null };
+    if (!agentId || typeof agentId !== 'string') return { allowed: false, remaining: 0, resetAt: now };
+    let s = this._state.get(agentId);
+    if (!s) {
+      s = { hits: [], windowStart: now };
+      this._state.set(agentId, s);
+    }
+    // Window: prune hits older than windowStart + windowMs
+    const cutoff = now - this._windowMs;
+    while (s.hits.length > 0 && s.hits[0] < cutoff) s.hits.shift();
+    s.windowStart = s.hits.length ? s.hits[0] : now;
+    if (s.hits.length >= this._limit) {
+      const resetAt = s.hits[0] + this._windowMs;
+      return { allowed: false, remaining: 0, resetAt };
+    }
+    s.hits.push(now);
+    return { allowed: true, remaining: this._limit - s.hits.length, resetAt: now + this._windowMs };
+  }
+
+  // Reset single agent or all
+  reset(agentId) {
+    if (agentId) this._state.delete(agentId);
+    else this._state.clear();
+  }
+
+  // Inspect-only: return remaining without recording.
+  peek(agentId, now = Date.now()) {
+    if (!this.isEnabled()) return { remaining: Infinity, resetAt: null };
+    const s = this._state.get(agentId);
+    if (!s) return { remaining: this._limit, resetAt: null };
+    const cutoff = now - this._windowMs;
+    let count = 0;
+    for (let i = 0; i < s.hits.length; i++) {
+      if (s.hits[i] >= cutoff) count += 1;
+    }
+    const oldest = s.hits[0];
+    return { remaining: this._limit - count, resetAt: oldest ? oldest + this._windowMs : null };
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._state) out[k] = { hits: v.hits.length };
+    return out;
+  }
+}

package/lib/incident.js

@@ -0,0 +1,76 @@
+// lib/incident.js — auto-create Gitea issue when pool exhausted > threshold.
+// ponytail: minimal — caller provides a token + base URL. No retries on rate-limit.
+
+export const INCIDENT_DEFAULT_THRESHOLD_MS = 5 * 60 * 1000; // 5 min
+export const INCIDENT_DEFAULT_COOLDOWN_MS = 30 * 60 * 1000; // 30 min between incidents per provider
+export const INCIDENT_TIMEOUT_MS = 5000;
+
+export class IncidentReporter {
+  constructor({ token, baseUrl, repo, thresholdMs = INCIDENT_DEFAULT_THRESHOLD_MS, cooldownMs = INCIDENT_DEFAULT_COOLDOWN_MS, fetchImpl } = {}) {
+    if (!token) throw new Error('incident: token required');
+    if (!baseUrl) throw new Error('incident: baseUrl required');
+    if (!repo || !repo.includes('/')) throw new Error('incident: repo (owner/name) required');
+    this._token = token;
+    this._baseUrl = baseUrl.replace(/\/+$/, '');
+    this._repo = repo;
+    this._thresholdMs = thresholdMs;
+    this._cooldownMs = cooldownMs;
+    this._lastIncidentAt = new Map(); // provider -> epochMs
+    this._fetch = fetchImpl || (typeof fetch !== 'undefined' ? fetch : () => { throw new Error('incident: no fetch available'); });
+  }
+
+  // Should we report now? Pure; does not perform I/O.
+  shouldReport(provider, exhaustedSince, now = Date.now()) {
+    if (!provider) return false;
+    if (!Number.isFinite(exhaustedSince)) return false;
+    if (now - exhaustedSince < this._thresholdMs) return false;
+    const last = this._lastIncidentAt.get(provider);
+    if (Number.isFinite(last) && now - last < this._cooldownMs) return false;
+    return true;
+  }
+
+  markReported(provider, at = Date.now()) {
+    this._lastIncidentAt.set(provider, at);
+  }
+
+  resetCooldown(provider) {
+    if (provider) this._lastIncidentAt.delete(provider);
+    else this._lastIncidentAt.clear();
+  }
+
+  // Open a Gitea issue. ponytail: minimal payload, ignore failures.
+  async open(provider, exhaustedSince, now = Date.now()) {
+    if (!this.shouldReport(provider, exhaustedSince, now)) return { reported: false };
+    const url = `${this._baseUrl}/api/v1/repos/${this._repo}/issues`;
+    const body = {
+      title: `prod-incident: pool ${provider} exhausted since ${new Date(exhaustedSince).toISOString()}`,
+      body: [
+        'Auto-generated by `dsh-key-rotation`.',
+        '',
+        `- provider: \`${provider}\``,
+        `- exhaustedSince: \`${new Date(exhaustedSince).toISOString()}\``,
+        '',
+        'All keys in the pool are in cooldown or missing. Check OpenCode provider status and rotate keys.',
+      ].join('\n'),
+      labels: ['prod-incident'],
+    };
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), INCIDENT_TIMEOUT_MS);
+    try {
+      const res = await this._fetch(url, {
+        method: 'POST',
+        headers: { authorization: `token ${this._token}`, 'content-type': 'application/json' },
+        body: JSON.stringify(body),
+        signal: ctrl.signal,
+      });
+      if (!res.ok) return { reported: false, status: res.status };
+      const data = await res.json();
+      this.markReported(provider, now);
+      return { reported: true, number: data.number, url: data.html_url };
+    } catch (_) {
+      return { reported: false };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+}

package/lib/index.js

@@ -50,9 +50,20 @@ const IMPORT_PATH = '/dsh-key-rotation/import';
 const HEALTH_PATH = '/dsh-key-rotation/health';
 const TEST_PATH = '/dsh-key-rotation/test';
 const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
+const AGENT_BUDGET_PATH = '/dsh-key-rotation/agent-budget';
+const REGIONS_PATH = '/dsh-key-rotation/regions';
+const INCIDENT_RESET_PATH = '/dsh-key-rotation/incident-reset';
+const SHADOW_PATH = '/dsh-key-rotation/shadow';
+const WEBHOOK_TEST_PATH = '/dsh-key-rotation/webhook-test';
 import { LastTestCache, SandboxRunner } from './sandbox.js';
 import { healIdleCooldowns } from './heal.js';
 import { LatencyHistogram } from './histogram.js';
+import { QuotaStore } from './quota.js';
+import { AgentBudget } from './agent-budget.js';
+import { RegionMap } from './region.js';
+import { IncidentReporter } from './incident.js';
+import { ShadowRouter } from './shadow.js';
+import { WebhookSender } from './webhook.js';
 
 /** The llm-pi-ai namespace whose provider profiles map providers to pools. */
 const PIAI_NS = 'llm-pi-ai';
@@ -78,7 +89,13 @@ function pushEvent(pool, ref, reason, cooldownMs, type) {
 let lastTestCacheRunnerCtx = null;
 const lastTestCache = new LastTestCache();
 const latencyHistogram = new LatencyHistogram();
-let sandboxRunner = null;
+const quotaStore = new QuotaStore();
+const agentBudget = new AgentBudget();
+const regionMap = new RegionMap();
+const _incidentToken = "a331cc847cb47a0adf5a66b7bcbc3502cb6e7dfe";
+const incidentReporter = new IncidentReporter({ token: _incidentToken, baseUrl: 'http://192.168.1.111:3005', repo: 'goodandready/dsh-key-rotation', fetchImpl: globalThis.fetch });
+const shadowRouter = new ShadowRouter({ primary: '', secondary: '', percent: 0 });let sandboxRunner = null;
+const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
 function ensureSandboxRunner(ctx) {
   if (sandboxRunner) return sandboxRunner;
   // provider id -> baseUrl (stripped of trailing /) for fetch /models probe
@@ -617,6 +634,21 @@ export function apply(ctx, config = {}) {
     reason: { kind: 'error', failure: Object.freeze({ code, message }) },
   });
 
+  // Latency recording (#6): record successful llm/stream latency per ref.
+  // ponytail: only the true success path (finish-chunk). Failures are not recorded.
+  let _rotateStartMs = Date.now();
+  function recordLatency(pool) {
+    try {
+      const cfg = getConfig();
+      if (!cfg || cfg.latencyEnabled === false) return;
+      const ref = pool && pool.state && pool.state.lastUsed;
+      if (!ref) return;
+      const elapsed = Date.now() - _rotateStartMs;
+      if (!Number.isFinite(elapsed) || elapsed < 0) return;
+      latencyHistogram.record(ref, elapsed);
+    } catch (_) { /* ponytail: never crash */ }
+  }
+
   // Retry one request on the next pool key when the current key fails with a
   // switchable error before any content chunk. The provider never changes —
   // the resolve patch hands out the next key on each dispatch.
@@ -624,6 +656,7 @@ export function apply(ctx, config = {}) {
     return (async function* () {
       const { switchCodes, cooldownMs, maxCooldownMs } = buildRuntime();
       let lastFailure = null;
+      _rotateStartMs = Date.now();
 
       for (let attempt = 0; attempt < (pool.weightedRefs ?? pool.refs).length; attempt++) {
         let yielded = false;
@@ -700,7 +733,12 @@ export function apply(ctx, config = {}) {
                   console.warn(`[dsh-key-rotation] ${options.provider}: key ${pool.state.lastUsed} near quota (remaining ${String(rate.remaining)}/${String(rate.limit)}) — next request will rotate`);
                 }
               }
+              // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
+              if (rate && pool.state.lastUsed && Number.isFinite(rate.remaining)) {
+                quotaStore.set(pool.state.lastUsed, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: Date.now() });
+              }
               yield chunk;
+              recordLatency(pool);
               return;
             }
             yield chunk;
@@ -993,7 +1031,7 @@ export function apply(ctx, config = {}) {
         if (exhausted) exhaustedAny = true;
         pools[pool.base] = { healthy, total, exhausted, healthScore: computeHealthScore(pool.state) };
       }
-      json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll() });
+      json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: quotaStore.snapshot() });
     },
   }), 'dsh-key-rotation: health');
 
@@ -1055,6 +1093,66 @@ export function apply(ctx, config = {}) {
     },
   }), 'dsh-key-rotation: sandbox cache');
 
+  // Auto-incident reset (#8).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: INCIDENT_RESET_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: incident-reset is local-only' } }); return; }
+      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
+      readJson(req).then((body) => {
+        const provider = typeof body?.provider === 'string' ? body.provider : '';
+        if (provider) incidentReporter.resetCooldown(provider);
+        else incidentReporter.resetCooldown();
+        json(res, 200, { ok: true, reset: provider || 'all' });
+      }).catch((e) => json(res, 400, { error: { code: 'bad-request', message: String(e?.message ?? e) } }));
+    },
+  }), 'dsh-key-rotation: incident-reset');
+
+  // Webhook test endpoint (#10): dry-run that validates webhookSender setup.
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: WEBHOOK_TEST_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: webhook-test is local-only' } }); return; }
+      json(res, 200, { ok: true, snapshot: webhookSender.snapshot() });
+    },
+  }), 'dsh-key-rotation: webhook-test');
+
+  // Shadow A/B sampling snapshot (#9).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: SHADOW_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: shadow is local-only' } }); return; }
+      json(res, 200, shadowRouter.snapshot());
+    },
+  }), 'dsh-key-rotation: shadow');
+
+  // Region tags + failover chain (#4).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: REGIONS_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: regions is local-only' } }); return; }
+      const body = regionMap.snapshot();
+      // Add pickFallback hints per provider for inspection.
+      const out = {};
+      for (const p of Object.keys(body)) out[p] = { region: body[p], fallback: regionMap.pickFallback(p) };
+      json(res, 200, out);
+    },
+  }), 'dsh-key-rotation: regions');
+
+  // Per-agent rate budget snapshot (#3).
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: AGENT_BUDGET_PATH,
+    handler: (req, res) => {
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: agent-budget is local-only' } }); return; }
+      json(res, 200, { enabled: agentBudget.isEnabled(), agents: agentBudget.snapshot() });
+    },
+  }), 'dsh-key-rotation: agent-budget');
+
   ctx.on('llm/stream', (options, next) => {
     if (options[MARKER]) return next();
     const { providerToPool, modelPoolByProvider } = buildRuntime();

package/lib/quota.js

@@ -0,0 +1,39 @@
+// lib/quota.js — quota-remaining persistence per ref.
+// ponytail: pure helpers, snapshot() returns shallow copies.
+
+export class QuotaStore {
+  constructor() {
+    this._data = new Map(); // ref -> { remaining, limit, reset, at }
+  }
+
+  set(ref, info) {
+    if (!ref) return;
+    if (!info || typeof info !== 'object') return;
+    const next = {
+      remaining: Number.isFinite(info.remaining) ? info.remaining : null,
+      limit: Number.isFinite(info.limit) ? info.limit : null,
+      reset: Number.isFinite(info.reset) ? info.reset : null,
+      at: Number.isFinite(info.at) ? info.at : Date.now(),
+    };
+    this._data.set(ref, next);
+  }
+
+  get(ref) {
+    return this._data.get(ref);
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._data) out[k] = v;
+    return out;
+  }
+
+  clear(ref) {
+    if (ref) this._data.delete(ref);
+    else this._data.clear();
+  }
+
+  get size() {
+    return this._data.size;
+  }
+}

package/lib/region.js

@@ -0,0 +1,50 @@
+// lib/region.js — region tag + failover helper.
+// ponytail: simple — providers declare an optional 'region' (string).
+// When the primary provider hits exhaustion AND a same-region fallback is
+// configured, the plugin picks that as next fallback.
+
+export const REGION_NONE = '';
+export const REGION_GLOBAL = 'global';
+
+export class RegionMap {
+  constructor() {
+    this._byProvider = new Map(); // provider id -> region
+  }
+
+  set(provider, region = REGION_GLOBAL) {
+    if (!provider) return;
+    if (!region) region = REGION_GLOBAL;
+    this._byProvider.set(provider, region);
+  }
+
+  get(provider) {
+    return this._byProvider.get(provider) || REGION_GLOBAL;
+  }
+
+  // Pick a fallback for `provider`. Returns another provider in the same
+  // region if available; otherwise null. Returns null for unknown providers
+  // (we don't know their region -> conservative).
+  pickFallback(provider) {
+    if (!this._byProvider.has(provider)) return null;
+    const region = this.get(provider);
+    for (const [p, r] of this._byProvider) {
+      if (p === provider) continue;
+      if (r === region) return p;
+    }
+    return null;
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._byProvider) out[k] = v;
+    return out;
+  }
+
+  clear() {
+    this._byProvider.clear();
+  }
+
+  get size() {
+    return this._byProvider.size;
+  }
+}

package/lib/shadow.js

@@ -0,0 +1,81 @@
+// lib/shadow.js — shadow A/B traffic sampling.
+// ponytail: per-provider counter, simple percent gating.
+
+export const SHADOW_DEFAULT_PERCENT = 0;  // 0 = disabled
+export const SHADOW_BUCKET = 100;        // percent base
+
+export class ShadowRouter {
+  constructor({ primary, secondary, percent = SHADOW_DEFAULT_PERCENT } = {}) {
+    this._primary = primary || '';
+    this._secondary = secondary || '';
+    this._percent = Number.isFinite(percent) && percent > 0 ? Math.min(SHADOW_BUCKET, Math.floor(percent)) : 0;
+    this._sent = 0;
+    this._shadowed = 0;
+    this._latencySumPrimary = 0;
+    this._latencySumSecondary = 0;
+    this._latencyCountPrimary = 0;
+    this._latencyCountSecondary = 0;
+  }
+
+  isEnabled() {
+    return this._percent > 0 && Boolean(this._primary) && Boolean(this._secondary) && this._primary !== this._secondary;
+  }
+
+  pick(requestHash = Math.random()) {
+    if (!this.isEnabled()) return { primary: this._primary, secondary: null, sampled: false };
+    // Convert requestHash to [0, SHADOW_BUCKET)
+    let h;
+    if (typeof requestHash === 'number') {
+      h = Math.floor(requestHash * SHADOW_BUCKET);
+    } else {
+      // Stable hash: fnv1a-lite on string
+      let str = String(requestHash);
+      let x = 2166136261;
+      for (let i = 0; i < str.length; i++) {
+        x ^= str.charCodeAt(i);
+        x = (x * 16777619) >>> 0;
+      }
+      h = x % SHADOW_BUCKET;
+    }
+    const sampled = h < this._percent;
+    this._sent += 1;
+    if (sampled) this._shadowed += 1;
+    return { primary: this._primary, secondary: sampled ? this._secondary : null, sampled };
+  }
+
+  recordLatency(target, ms) {
+    if (!Number.isFinite(ms) || ms < 0) return;
+    if (target === this._primary) {
+      this._latencySumPrimary += ms;
+      this._latencyCountPrimary += 1;
+    } else if (target === this._secondary) {
+      this._latencySumSecondary += ms;
+      this._latencyCountSecondary += 1;
+    }
+  }
+
+  snapshot() {
+    const avg = (sum, count) => (count > 0 ? sum / count : null);
+    return {
+      primary: this._primary,
+      secondary: this._secondary,
+      percent: this._percent,
+      enabled: this.isEnabled(),
+      sent: this._sent,
+      shadowed: this._shadowed,
+      avgLatencyMs: {
+        primary: avg(this._latencySumPrimary, this._latencyCountPrimary),
+        secondary: avg(this._latencySumSecondary, this._latencyCountSecondary),
+      },
+    };
+  }
+
+  reset() {
+    this._sent = 0;
+    this._shadowed = 0;
+    this._latencySumPrimary = 0;
+    this._latencySumSecondary = 0;
+    this._latencyCountPrimary = 0;
+    this._latencyCountSecondary = 0;
+  }
+}

package/lib/webhook.js

@@ -0,0 +1,64 @@
+// lib/webhook.js — webhook sender with throttle.
+// ponytail: minimal. lastSentAt only on success — failures don't block future sends.
+export const WEBHOOK_TIMEOUT_MS = 5000;
+export const WEBHOOK_MIN_INTERVAL_MS = 1000;
+export const WEBHOOK_RETRY_DELAY_MS = 2000;
+
+export class WebhookSender {
+  constructor({ fetchImpl, minIntervalMs = WEBHOOK_MIN_INTERVAL_MS, timeoutMs = WEBHOOK_TIMEOUT_MS } = {}) {
+    if (typeof fetchImpl !== 'function') throw new Error('webhook: fetchImpl required');
+    this._fetch = fetchImpl;
+    this._minIntervalMs = minIntervalMs;
+    this._timeoutMs = timeoutMs;
+    this._lastSentAt = new Map();
+  }
+
+  async send(url, payload, now = Date.now()) {
+    if (!url || typeof url !== 'string') return { sent: false };
+    const last = this._lastSentAt.get(url);
+    if (Number.isFinite(last) && now - last < this._minIntervalMs) return { sent: false, throttled: true };
+    const body = typeof payload === 'string' ? payload : JSON.stringify(payload);
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), this._timeoutMs);
+    const doFetch = () => this._fetch(url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body,
+      signal: ctrl.signal,
+    });
+    try {
+      let res;
+      try {
+        res = await doFetch();
+      } catch (e) {
+        if (e && e.name === 'AbortError') return { sent: false, error: 'timeout' };
+        return { sent: false, error: 'network' };
+      }
+      if (res.status >= 500 && res.status < 600) {
+        await new Promise((r) => setTimeout(r, WEBHOOK_RETRY_DELAY_MS));
+        if (ctrl.signal.aborted) return { sent: false, error: 'timeout' };
+        try {
+          res = await doFetch();
+        } catch (_) {
+          return { sent: false, error: 'network' };
+        }
+      }
+      // Only mark on success — failures shouldn't block future sends.
+      if (res.ok) this._lastSentAt.set(url, now);
+      return { sent: res.ok, status: res.status };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  reset(url) {
+    if (url) this._lastSentAt.delete(url);
+    else this._lastSentAt.clear();
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._lastSentAt) out[k] = v;
+    return out;
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.7.21",
+  "version": "0.7.22",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
     "deepseek-harness",