@goodandready/dsh-key-rotation 0.7.29 → 0.7.31

package/lib/heal.js

@@ -1,35 +1,35 @@
-// heal.js — self-healing idle cooldowns.
-// ponytail: pure function, easy to test, no side effects beyond mutation of passed-in state.
-
-// Returns array of { ref, poolBase } entries that were healed in this tick.
-// Mutates `pools` (removes from failedUntil, pushes heal event into events).
-// `now` parameter is injectable for tests.
-export function healIdleCooldowns(pools, idleMs, now = Date.now()) {
-  if (!Array.isArray(pools) || pools.length === 0) return [];
-  if (!Number.isFinite(idleMs) || idleMs <= 0) return [];
-  const healed = [];
-  for (const pool of pools) {
-    if (!pool || !pool.state || !pool.base) continue;
-    const fu = pool.state.failedUntil;
-    const lu = pool.state.lastUsed;
-    if (!fu || fu.size === 0) continue;
-    const expiredRefs = [];
-    for (const [ref, until] of fu.entries()) {
-      if (!Number.isFinite(until)) continue;
-      if (until > now) continue; // cooldown still active
-      const last = lu ? lu.get(ref) : undefined;
-      if (!Number.isFinite(last)) continue; // never used → no signal, skip
-      if (now - last < idleMs) continue; // used recently → don't heal
-      expiredRefs.push(ref);
-    }
-    for (const ref of expiredRefs) {
-      fu.delete(ref);
-      if (Array.isArray(pool.state.events)) {
-        pool.state.events.push({ at: now, ref, reason: 'self-heal', cooldownMs: 0, type: 'heal' });
-        if (pool.state.events.length > 50) pool.state.events.shift();
-      }
-      healed.push({ ref, poolBase: pool.base });
-    }
-  }
-  return healed;
-}
+// heal.js — self-healing idle cooldowns.
+// ponytail: pure function, easy to test, no side effects beyond mutation of passed-in state.
+
+// Returns array of { ref, poolBase } entries that were healed in this tick.
+// Mutates `pools` (removes from failedUntil, pushes heal event into events).
+// `now` parameter is injectable for tests.
+export function healIdleCooldowns(pools, idleMs, now = Date.now()) {
+  if (!Array.isArray(pools) || pools.length === 0) return [];
+  if (!Number.isFinite(idleMs) || idleMs <= 0) return [];
+  const healed = [];
+  for (const pool of pools) {
+    if (!pool || !pool.state || !pool.base) continue;
+    const fu = pool.state.failedUntil;
+    const lu = pool.state.lastUsed;
+    if (!fu || fu.size === 0) continue;
+    const expiredRefs = [];
+    for (const [ref, until] of fu.entries()) {
+      if (!Number.isFinite(until)) continue;
+      if (until > now) continue; // cooldown still active
+      const last = lu ? lu.get(ref) : undefined;
+      if (!Number.isFinite(last)) continue; // never used → no signal, skip
+      if (now - last < idleMs) continue; // used recently → don't heal
+      expiredRefs.push(ref);
+    }
+    for (const ref of expiredRefs) {
+      fu.delete(ref);
+      if (Array.isArray(pool.state.events)) {
+        pool.state.events.push({ at: now, ref, reason: 'self-heal', cooldownMs: 0, type: 'heal' });
+        if (pool.state.events.length > 50) pool.state.events.shift();
+      }
+      healed.push({ ref, poolBase: pool.base });
+    }
+  }
+  return healed;
+}

package/lib/histogram.js

@@ -1,66 +1,66 @@
-// histogram.js — per-ref latency ring buffer + percentile.
-// ponytail: ring buffer of fixed size, sort-on-read for percentile, no libraries.
-
-export const LATENCY_DEFAULT_WINDOW = 200;
-
-export class LatencyHistogram {
-  constructor({ window = LATENCY_DEFAULT_WINDOW } = {}) {
-    const w = Number.isFinite(window) && window > 0 ? Math.floor(window) : LATENCY_DEFAULT_WINDOW;
-    this._window = w;
-    this._buffers = new Map(); // ref -> Float64Array of size w, plus index/count
-    this._lastAt = new Map();  // ref -> epochMs of last sample
-  }
-
-  record(ref, ms) {
-    if (!ref || !Number.isFinite(ms) || ms < 0) return;
-    let entry = this._buffers.get(ref);
-    if (!entry) {
-      entry = { buf: new Float64Array(this._window), head: 0, count: 0 };
-      this._buffers.set(ref, entry);
-    }
-    entry.buf[entry.head] = ms;
-    entry.head = (entry.head + 1) % this._window;
-    if (entry.count < this._window) entry.count += 1;
-    this._lastAt.set(ref, Date.now());
-  }
-
-  // Returns p50/p95/p99 in milliseconds, plus count and lastAt. Sorted copy.
-  snapshot(ref) {
-    const entry = this._buffers.get(ref);
-    const lastAt = this._lastAt.get(ref);
-    if (!entry || entry.count === 0) {
-      return { count: 0, lastAt: lastAt || null };
-    }
-    const arr = entry.buf.subarray(0, entry.count);
-    const sorted = Array.from(arr).sort((a, b) => a - b);
-    const n = sorted.length;
-    return {
-      count: n,
-      lastAt: lastAt || null,
-      p50: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.5))],
-      p95: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.95))],
-      p99: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.99))],
-    };
-  }
-
-  // Returns { [ref]: snapshot }
-  snapshotAll() {
-    const out = {};
-    for (const ref of this._buffers.keys()) out[ref] = this.snapshot(ref);
-    return out;
-  }
-
-  clear(ref) {
-    if (ref) {
-      this._buffers.delete(ref);
-      this._lastAt.delete(ref);
-    } else {
-      this._buffers.clear();
-      this._lastAt.clear();
-    }
-  }
-
-  get size() {
-    return this._buffers.size;
-  }
-}
+// histogram.js — per-ref latency ring buffer + percentile.
+// ponytail: ring buffer of fixed size, sort-on-read for percentile, no libraries.
+
+export const LATENCY_DEFAULT_WINDOW = 200;
+
+export class LatencyHistogram {
+  constructor({ window = LATENCY_DEFAULT_WINDOW } = {}) {
+    const w = Number.isFinite(window) && window > 0 ? Math.floor(window) : LATENCY_DEFAULT_WINDOW;
+    this._window = w;
+    this._buffers = new Map(); // ref -> Float64Array of size w, plus index/count
+    this._lastAt = new Map();  // ref -> epochMs of last sample
+  }
+
+  record(ref, ms) {
+    if (!ref || !Number.isFinite(ms) || ms < 0) return;
+    let entry = this._buffers.get(ref);
+    if (!entry) {
+      entry = { buf: new Float64Array(this._window), head: 0, count: 0 };
+      this._buffers.set(ref, entry);
+    }
+    entry.buf[entry.head] = ms;
+    entry.head = (entry.head + 1) % this._window;
+    if (entry.count < this._window) entry.count += 1;
+    this._lastAt.set(ref, Date.now());
+  }
+
+  // Returns p50/p95/p99 in milliseconds, plus count and lastAt. Sorted copy.
+  snapshot(ref) {
+    const entry = this._buffers.get(ref);
+    const lastAt = this._lastAt.get(ref);
+    if (!entry || entry.count === 0) {
+      return { count: 0, lastAt: lastAt || null };
+    }
+    const arr = entry.buf.subarray(0, entry.count);
+    const sorted = Array.from(arr).sort((a, b) => a - b);
+    const n = sorted.length;
+    return {
+      count: n,
+      lastAt: lastAt || null,
+      p50: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.5))],
+      p95: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.95))],
+      p99: sorted[Math.min(n - 1, Math.floor((n - 1) * 0.99))],
+    };
+  }
+
+  // Returns { [ref]: snapshot }
+  snapshotAll() {
+    const out = {};
+    for (const ref of this._buffers.keys()) out[ref] = this.snapshot(ref);
+    return out;
+  }
+
+  clear(ref) {
+    if (ref) {
+      this._buffers.delete(ref);
+      this._lastAt.delete(ref);
+    } else {
+      this._buffers.clear();
+      this._lastAt.clear();
+    }
+  }
+
+  get size() {
+    return this._buffers.size;
+  }
+}

package/lib/incident.js

@@ -1,76 +1,76 @@
-// lib/incident.js — auto-create Gitea issue when pool exhausted > threshold.
-// ponytail: minimal — caller provides a token + base URL. No retries on rate-limit.
-
-export const INCIDENT_DEFAULT_THRESHOLD_MS = 5 * 60 * 1000; // 5 min
-export const INCIDENT_DEFAULT_COOLDOWN_MS = 30 * 60 * 1000; // 30 min between incidents per provider
-export const INCIDENT_TIMEOUT_MS = 5000;
-
-export class IncidentReporter {
-  constructor({ token, baseUrl, repo, thresholdMs = INCIDENT_DEFAULT_THRESHOLD_MS, cooldownMs = INCIDENT_DEFAULT_COOLDOWN_MS, fetchImpl } = {}) {
-    if (!token) throw new Error('incident: token required');
-    if (!baseUrl) throw new Error('incident: baseUrl required');
-    if (!repo || !repo.includes('/')) throw new Error('incident: repo (owner/name) required');
-    this._token = token;
-    this._baseUrl = baseUrl.replace(/\/+$/, '');
-    this._repo = repo;
-    this._thresholdMs = thresholdMs;
-    this._cooldownMs = cooldownMs;
-    this._lastIncidentAt = new Map(); // provider -> epochMs
-    this._fetch = fetchImpl || (typeof fetch !== 'undefined' ? fetch : () => { throw new Error('incident: no fetch available'); });
-  }
-
-  // Should we report now? Pure; does not perform I/O.
-  shouldReport(provider, exhaustedSince, now = Date.now()) {
-    if (!provider) return false;
-    if (!Number.isFinite(exhaustedSince)) return false;
-    if (now - exhaustedSince < this._thresholdMs) return false;
-    const last = this._lastIncidentAt.get(provider);
-    if (Number.isFinite(last) && now - last < this._cooldownMs) return false;
-    return true;
-  }
-
-  markReported(provider, at = Date.now()) {
-    this._lastIncidentAt.set(provider, at);
-  }
-
-  resetCooldown(provider) {
-    if (provider) this._lastIncidentAt.delete(provider);
-    else this._lastIncidentAt.clear();
-  }
-
-  // Open a Gitea issue. ponytail: minimal payload, ignore failures.
-  async open(provider, exhaustedSince, now = Date.now()) {
-    if (!this.shouldReport(provider, exhaustedSince, now)) return { reported: false };
-    const url = `${this._baseUrl}/api/v1/repos/${this._repo}/issues`;
-    const body = {
-      title: `prod-incident: pool ${provider} exhausted since ${new Date(exhaustedSince).toISOString()}`,
-      body: [
-        'Auto-generated by `dsh-key-rotation`.',
-        '',
-        `- provider: \`${provider}\``,
-        `- exhaustedSince: \`${new Date(exhaustedSince).toISOString()}\``,
-        '',
-        'All keys in the pool are in cooldown or missing. Check OpenCode provider status and rotate keys.',
-      ].join('\n'),
-      labels: ['prod-incident'],
-    };
-    const ctrl = new AbortController();
-    const timer = setTimeout(() => ctrl.abort(), INCIDENT_TIMEOUT_MS);
-    try {
-      const res = await this._fetch(url, {
-        method: 'POST',
-        headers: { authorization: `token ${this._token}`, 'content-type': 'application/json' },
-        body: JSON.stringify(body),
-        signal: ctrl.signal,
-      });
-      if (!res.ok) return { reported: false, status: res.status };
-      const data = await res.json();
-      this.markReported(provider, now);
-      return { reported: true, number: data.number, url: data.html_url };
-    } catch (_) {
-      return { reported: false };
-    } finally {
-      clearTimeout(timer);
-    }
-  }
-}
+// lib/incident.js — auto-create Gitea issue when pool exhausted > threshold.
+// ponytail: minimal — caller provides a token + base URL. No retries on rate-limit.
+
+export const INCIDENT_DEFAULT_THRESHOLD_MS = 5 * 60 * 1000; // 5 min
+export const INCIDENT_DEFAULT_COOLDOWN_MS = 30 * 60 * 1000; // 30 min between incidents per provider
+export const INCIDENT_TIMEOUT_MS = 5000;
+
+export class IncidentReporter {
+  constructor({ token, baseUrl, repo, thresholdMs = INCIDENT_DEFAULT_THRESHOLD_MS, cooldownMs = INCIDENT_DEFAULT_COOLDOWN_MS, fetchImpl } = {}) {
+    if (!token) throw new Error('incident: token required');
+    if (!baseUrl) throw new Error('incident: baseUrl required');
+    if (!repo || !repo.includes('/')) throw new Error('incident: repo (owner/name) required');
+    this._token = token;
+    this._baseUrl = baseUrl.replace(/\/+$/, '');
+    this._repo = repo;
+    this._thresholdMs = thresholdMs;
+    this._cooldownMs = cooldownMs;
+    this._lastIncidentAt = new Map(); // provider -> epochMs
+    this._fetch = fetchImpl || (typeof fetch !== 'undefined' ? fetch : () => { throw new Error('incident: no fetch available'); });
+  }
+
+  // Should we report now? Pure; does not perform I/O.
+  shouldReport(provider, exhaustedSince, now = Date.now()) {
+    if (!provider) return false;
+    if (!Number.isFinite(exhaustedSince)) return false;
+    if (now - exhaustedSince < this._thresholdMs) return false;
+    const last = this._lastIncidentAt.get(provider);
+    if (Number.isFinite(last) && now - last < this._cooldownMs) return false;
+    return true;
+  }
+
+  markReported(provider, at = Date.now()) {
+    this._lastIncidentAt.set(provider, at);
+  }
+
+  resetCooldown(provider) {
+    if (provider) this._lastIncidentAt.delete(provider);
+    else this._lastIncidentAt.clear();
+  }
+
+  // Open a Gitea issue. ponytail: minimal payload, ignore failures.
+  async open(provider, exhaustedSince, now = Date.now()) {
+    if (!this.shouldReport(provider, exhaustedSince, now)) return { reported: false };
+    const url = `${this._baseUrl}/api/v1/repos/${this._repo}/issues`;
+    const body = {
+      title: `prod-incident: pool ${provider} exhausted since ${new Date(exhaustedSince).toISOString()}`,
+      body: [
+        'Auto-generated by `dsh-key-rotation`.',
+        '',
+        `- provider: \`${provider}\``,
+        `- exhaustedSince: \`${new Date(exhaustedSince).toISOString()}\``,
+        '',
+        'All keys in the pool are in cooldown or missing. Check OpenCode provider status and rotate keys.',
+      ].join('\n'),
+      labels: ['prod-incident'],
+    };
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), INCIDENT_TIMEOUT_MS);
+    try {
+      const res = await this._fetch(url, {
+        method: 'POST',
+        headers: { authorization: `token ${this._token}`, 'content-type': 'application/json' },
+        body: JSON.stringify(body),
+        signal: ctrl.signal,
+      });
+      if (!res.ok) return { reported: false, status: res.status };
+      const data = await res.json();
+      this.markReported(provider, now);
+      return { reported: true, number: data.number, url: data.html_url };
+    } catch (_) {
+      return { reported: false };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+}

package/lib/index.js

@@ -86,6 +86,8 @@ let rotationDisabled = false;
 const expiryNotifiedAt = new Map();
 const budgetNotifiedAt = new Map();
 const switchNotifiedAt = new Map();
+const lowHealthNotifiedAt = new Map();
+const sloNotifiedAt = new Map();
 const DAY_MS = 86400000;
 
 // #216: one webhook per switch, deduped to at most one message per provider
@@ -215,6 +217,8 @@ export const Config = Schema.object({
   expiryWarnDays: Schema.number().default(7),
   switchNotify: Schema.boolean().default(false),
   switchNotifyThrottleMs: Schema.number().default(60000),
+  warnBelowHealthy: Schema.number().default(0),
+  latencySloMs: Schema.number().default(0),
   providers: Schema.array(Schema.object({
     provider: Schema.string().required(),
     keys: Schema.array(Schema.string()).default([]),
@@ -633,6 +637,55 @@ export function apply(ctx, config = {}) {
               if ((pool.state.failedUntil.get(ref) ?? 0) < until) pool.state.failedUntil.set(ref, until);
             }
           }
+          // #221: pool running low - webhook while healthy < warnBelowHealthy
+          const warnBelow = runtime.warnBelowHealthy ?? 0;
+          if (warnBelow > 0) {
+            let healthy = 0;
+            for (const ref of pool.refs) {
+              const fu = pool.state.failedUntil.get(ref);
+              if (fu !== undefined && fu > now) continue;
+              const exp = pool.expiresAt?.[ref];
+              if (exp !== undefined && now >= exp) continue;
+              healthy++;
+            }
+            if (healthy < warnBelow && shouldNotifyDaily(lowHealthNotifiedAt, pool.base, now)) {
+              console.warn(`[dsh-key-rotation] ${pool.base}: pool running low - ${healthy}/${pool.refs.length} healthy`);
+              if (runtime.notifyWebhook) {
+                const token = runtime.webhookActionToken ?? '';
+                webhookSender.send(runtime.notifyWebhook, {
+                  title: `Pool running low: ${pool.base}`,
+                  text: `${healthy}/${pool.refs.length} keys healthy (alert below ${warnBelow})`,
+                  provider: pool.base,
+                  kind: 'low-health',
+                  healthy,
+                  total: pool.refs.length,
+                  actionToken: token || undefined,
+                  actions: token ? [{ id: `reset-${pool.base}`, label: 'Reset cooldown' }] : undefined,
+                });
+              }
+            }
+          }
+          // #225: latency SLO - webhook when a key's p95 exceeds the threshold
+          const slo = runtime.latencySloMs ?? 0;
+          if (slo > 0) {
+            for (const ref of pool.refs) {
+              const snap = latencyHistogram.snapshot(ref);
+              if (!snap.p95 || snap.p95 <= slo) continue;
+              if (!shouldNotifyDaily(sloNotifiedAt, pool.base + ':' + ref + ':slo', now)) continue;
+              console.warn(`[dsh-key-rotation] ${pool.base}: ${ref} p95 ${Math.round(snap.p95)}ms > SLO ${slo}ms`);
+              if (runtime.notifyWebhook) {
+                webhookSender.send(runtime.notifyWebhook, {
+                  title: `Latency SLO exceeded: ${pool.base}`,
+                  text: `${ref} p95 ${Math.round(snap.p95)}ms > ${slo}ms (${snap.count} samples)`,
+                  provider: pool.base,
+                  kind: 'latency-slo',
+                  ref,
+                  p95: Math.round(snap.p95),
+                  slo,
+                });
+              }
+            }
+          }
         }
       } catch (_) { /* maintenance must never crash the sweep */ }
     }, 30000);
@@ -776,7 +829,7 @@ export function apply(ctx, config = {}) {
       const weekly = typeof p.costBudgetWeekly === 'number' ? p.costBudgetWeekly : 0;
       if (daily > 0 || weekly > 0) providerBudgets.set(p.provider, { costBudgetDaily: daily, costBudgetWeekly: weekly, pauseOnBudget: p.pauseOnBudget ?? false });
     }
-    return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, concurrencyLimit, canaryProbingEnabled, canaryIntervalMs, cascade, quotaResetWindow, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
+    return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, concurrencyLimit, canaryProbingEnabled, canaryIntervalMs, cascade, quotaResetWindow, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
   }
 
   // ── patch credentials.resolve: pool refs resolve to the next healthy key ──
@@ -1104,6 +1157,7 @@ export function apply(ctx, config = {}) {
       const { poolByRef, providerTags, providerBudgets } = buildRuntime();
       const base = ctx.get('credentials');
       const now = Date.now();
+      const latencySloMs = buildRuntime().latencySloMs;
       const seen = new Set();
       const providers = [];
       for (const pool of poolByRef.values()) {
@@ -1171,6 +1225,12 @@ export function apply(ctx, config = {}) {
           lastExhaustionAt: pool.state.lastExhaustionAt ?? null,
           exhaustionCount: pool.state.exhaustionCount ?? 0,
           totalUsage: [...(pool.state.usageCounts?.values() ?? [])].reduce((a, b) => a + b, 0),
+          // #225: aggregate p95 across the pool's keys
+          p95: (() => {
+            const vals = (pool.refs ?? []).map((r) => latencyHistogram.snapshot(r)).filter((s) => s && s.p95 != null).map((s) => s.p95);
+            return vals.length ? Math.round(Math.max(...vals)) : null;
+          })(),
+          latencySloMs,
           events: (pool.state.events ?? []).slice(-50),
           healthScore: computeHealthScore(pool.state),
           // #208: today/week spend + configured budget for the card
@@ -1592,6 +1652,28 @@ export function apply(ctx, config = {}) {
       if (!action && typeof body?.callback_data === 'string') {
         try { action = String(JSON.parse(body.callback_data)?.id ?? ''); } catch { action = ''; }
       }
+      // #222: Telegram update envelope {update_id, callback_query:{data}}
+      if (!action && typeof body?.callback_query?.data === 'string') {
+        try { action = String(JSON.parse(body.callback_query.data)?.id ?? ''); } catch { action = ''; }
+      }
+      // #222: Telegram setWebhook registration helper
+      if (typeof body?.setWebhook === 'object' && body.setWebhook) {
+        const botToken = typeof body.setWebhook.botToken === 'string' ? body.setWebhook.botToken : '';
+        if (!botToken) { json(res, 400, { error: { code: 'bad-request', message: 'dsh-key-rotation: setWebhook.botToken required' } }); return; }
+        // derive the public URL from request headers; explicit URL wins
+        const url = typeof body.setWebhook.url === 'string' && body.setWebhook.url ? body.setWebhook.url : `https://${String(req.headers.host ?? '')}/dsh-key-rotation/webhook-action`;
+        try {
+          const hookRes = await fetch(`https://api.telegram.org/bot${botToken}/setWebhook`, {
+            method: 'POST', headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({ url, allowed_updates: ['callback_query'] }),
+          });
+          const hookData = await hookRes.json().catch(() => ({}));
+          json(res, 200, { ok: hookRes.ok, url, telegram: hookData });
+        } catch (e) {
+          json(res, 502, { error: { code: 'telegram-failed', message: String(e?.message ?? e) } });
+        }
+        return;
+      }
       if (!action) { json(res, 400, { error: { code: 'bad-action', message: 'dsh-key-rotation: no action in payload' } }); return; }
       const provider = action.startsWith('pause-') || action.startsWith('reset-') ? action.replace(/^(pause|reset)-/, '') : '';
       try {