@goodandready/dsh-key-rotation 0.7.35 → 0.7.37

package/lib/pool.js

@@ -44,6 +44,10 @@ export const SWITCHABLE_MESSAGE_PATTERN = new RegExp([
   /\bout[\s_-]+of[\s_-]+(?:credits?|budget)\b/i,
   /\b(?:exceeded|exhausted)[\s_-]+(?:quota|limit|budget)\b/i,
   /\bbilling\b/i,
+  /\bresource[\s_-]+exhausted\b/i,
+  /\bcapacity[\s_-]+(?:limit|exceeded|reached)\b/i,
+  /\bfree[\s_-]+tier[\s_-]+(?:limit|exceeded)\b/i,
+  /\bconcurrent[\s_-]+(?:requests?|limit)[\s_-]+exceeded\b/i,
   /\b429\b|\b5\d\d\b/i,
   /\btime(?:d)?\s*out\b|timeout/i,
   /\b(?:network|connection|socket|fetch|ECONN[A-Z]+)\b/i,
@@ -52,6 +56,7 @@ export const SWITCHABLE_MESSAGE_PATTERN = new RegExp([
   /\b(?:invalid|expired|revoked|unauthorized)[\s_-]+(?:api[\s_-]?key|token)\b/i,
   /\bapi[\s_-]?key[\s_-]+(?:is[\s_-]+)?(?:invalid|expired|revoked|unauthorized)\b/i,
   /\b(?:authentication|unauthorized|not[\s_-]+authorized)\b/i,
+  /\b(?:overloaded|server[\s_-]+busy)\b/i,
 ].map((r) => r.source).join('|'), 'i');
 
 /** Default switch codes used by lib/index.js. */
@@ -261,4 +266,108 @@ export function isRateLimited(rate, threshold = 0.1) {
   if (rate.remaining === undefined) return false;
   if (rate.limit && rate.limit > 0) return rate.remaining < rate.limit * threshold;
   return rate.remaining <= 0;
-}
+}
+
+/**
+ * Unified check whether an error or payload represents a switchable failure.
+ * Prioritizes explicit HTTP status codes and gRPC codes before text matching.
+ */
+export function isSwitchableError(failureOrPayload, switchCodes = new Set(DEFAULT_SWITCH_CODES)) {
+  if (!failureOrPayload) return false;
+  const failure = failureOrPayload.failure ?? failureOrPayload;
+  const status = Number(failure.status ?? failure.statusCode ?? failure.httpStatus ?? 0);
+  const code = String(failure.code ?? failure.reason ?? '').toUpperCase();
+  const message = String(failure.message ?? failureOrPayload.message ?? '');
+
+  // 1. Direct HTTP status codes
+  if (status === 429 && (switchCodes.has('RATE_LIMIT') || switchCodes.has('QUOTA') || switchCodes.has('429'))) return true;
+  if ((status === 401 || status === 403) && (switchCodes.has('AUTH') || switchCodes.has('401'))) return true;
+  if ((status >= 500 && status <= 504) && (switchCodes.has('SERVER') || switchCodes.has(String(status)))) return true;
+
+  // 2. Standard gRPC / cloud error codes
+  if (code === 'RESOURCE_EXHAUSTED' && (switchCodes.has('QUOTA') || switchCodes.has('RATE_LIMIT'))) return true;
+  if ((code === 'UNAVAILABLE' || code === 'INTERNAL') && switchCodes.has('SERVER')) return true;
+  if (code === 'DEADLINE_EXCEEDED' && switchCodes.has('TIMEOUT')) return true;
+  if ((code === 'UNAUTHENTICATED' || code === 'PERMISSION_DENIED') && switchCodes.has('AUTH')) return true;
+
+  // 3. Named switch code matching
+  if (code && switchCodes.has(code)) return true;
+
+  // 4. Fallback text pattern matching
+  return SWITCHABLE_MESSAGE_PATTERN.test(message);
+}
+
+/**
+ * Format an informative user-facing exhaustion message with recovery countdown.
+ */
+export function formatExhaustionMessage(provider, pool, now = Date.now()) {
+  const list = pool.refs ?? [];
+  const total = list.length;
+  let minWaitMs = Infinity;
+  if (pool.state && pool.state.failedUntil) {
+    for (const ref of list) {
+      const until = pool.state.failedUntil.get(ref) ?? 0;
+      if (until > now) {
+        const wait = until - now;
+        if (wait < minWaitMs) minWaitMs = wait;
+      }
+    }
+  }
+  const sec = Number.isFinite(minWaitMs) && minWaitMs > 0 ? Math.ceil(minWaitMs / 1000) : 60;
+  return `[dsh-key-rotation] All ${total} keys for provider '${provider}' are temporarily exhausted. Next key recovers in ~${sec}s.`;
+}
+
+
+/**
+ * Keys of pool whose expiresAt falls within the next warnDays.
+ * Returns [{ ref, expiresInDays, expiresAt }], soonest first.
+ */
+export function expiringSoon(pool, warnDays = 7, now = Date.now()) {
+  if (!pool || !pool.expiresAt) return [];
+  const DAY_MS = 86400000;
+  const horizon = now + Math.max(1, warnDays) * DAY_MS;
+  return Object.entries(pool.expiresAt)
+    .filter(([, at]) => at > now && at <= horizon)
+    .map(([ref, at]) => ({ ref, expiresAt: at, expiresInDays: Math.max(0, Math.floor((at - now) / DAY_MS)) }))
+    .sort((a, b) => a.expiresAt - b.expiresAt);
+}
+
+/** Dedupe: true when a day-level notification for key is due. */
+export function shouldNotifyDaily(lastNotified, key, now = Date.now()) {
+  const DAY_MS = 86400000;
+  if (!lastNotified.has(key)) {
+    lastNotified.set(key, now);
+    return true;
+  }
+  const last = lastNotified.get(key);
+  if (now - last < DAY_MS) return false;
+  lastNotified.set(key, now);
+  return true;
+}
+
+/** Total spend of a pool on ISO day day (defaults to today) across costDays Map<ref, Map<day, cost>>. */
+export function costForDay(costDays, day) {
+  const d = day ?? new Date().toISOString().slice(0, 10);
+  let total = 0;
+  for (const perRef of (costDays?.values() ?? [])) {
+    total += perRef.get(d) ?? 0;
+  }
+  return total;
+}
+
+/** Total spend over the last 7 ISO days ending today. */
+export function costForWeek(costDays, now = Date.now()) {
+  let total = 0;
+  for (let i = 0; i < 7; i++) {
+    const d = new Date(now - i * 86400000).toISOString().slice(0, 10);
+    total += costForDay(costDays, d);
+  }
+  return total;
+}
+
+/** Budget verdict for a pool: { spend, budget, ratio, warn, exceeded }. */
+export function budgetVerdict(spend, budget) {
+  if (!budget || budget <= 0) return { spend, budget: 0, ratio: 0, warn: false, exceeded: false };
+  const ratio = spend / budget;
+  return { spend, budget, ratio, warn: ratio >= 0.8, exceeded: ratio >= 1 };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.7.35",
+  "version": "0.7.37",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [

package/lib/agent-budget.js

@@ -1,68 +0,0 @@
-// lib/agent-budget.js — per-agent rate cap.
-// ponytail: in-memory counter per (agent, window), thread-safe-ish via timer map.
-
-export const AGENT_BUDGET_DEFAULT_WINDOW_MS = 3600_000; // 1h
-export const AGENT_BUDGET_DEFAULT_LIMIT = 0;             // 0 = disabled
-export const AGENT_BUDGET_MAX = 50000;                  // hard ceiling per agent
-
-export class AgentBudget {
-  constructor({ windowMs = AGENT_BUDGET_DEFAULT_WINDOW_MS, limit = AGENT_BUDGET_DEFAULT_LIMIT } = {}) {
-    const w = Number.isFinite(windowMs) && windowMs > 0 ? Math.floor(windowMs) : AGENT_BUDGET_DEFAULT_WINDOW_MS;
-    const l = Number.isFinite(limit) && limit >= 0 ? Math.min(AGENT_BUDGET_MAX, Math.floor(limit)) : 0;
-    this._windowMs = w;
-    this._limit = l;
-    this._state = new Map(); // agent -> { hits: number[], windowStart: epochMs }
-  }
-
-  isEnabled() {
-    return this._limit > 0;
-  }
-
-  // Decide if request from this agent is allowed. Returns { allowed, remaining, resetAt }.
-  // Records the hit only when allowed.
-  check(agentId, now = Date.now()) {
-    if (!this.isEnabled()) return { allowed: true, remaining: Infinity, resetAt: null };
-    if (!agentId || typeof agentId !== 'string') return { allowed: false, remaining: 0, resetAt: now };
-    let s = this._state.get(agentId);
-    if (!s) {
-      s = { hits: [], windowStart: now };
-      this._state.set(agentId, s);
-    }
-    // Window: prune hits older than windowStart + windowMs
-    const cutoff = now - this._windowMs;
-    while (s.hits.length > 0 && s.hits[0] < cutoff) s.hits.shift();
-    s.windowStart = s.hits.length ? s.hits[0] : now;
-    if (s.hits.length >= this._limit) {
-      const resetAt = s.hits[0] + this._windowMs;
-      return { allowed: false, remaining: 0, resetAt };
-    }
-    s.hits.push(now);
-    return { allowed: true, remaining: this._limit - s.hits.length, resetAt: now + this._windowMs };
-  }
-
-  // Reset single agent or all
-  reset(agentId) {
-    if (agentId) this._state.delete(agentId);
-    else this._state.clear();
-  }
-
-  // Inspect-only: return remaining without recording.
-  peek(agentId, now = Date.now()) {
-    if (!this.isEnabled()) return { remaining: Infinity, resetAt: null };
-    const s = this._state.get(agentId);
-    if (!s) return { remaining: this._limit, resetAt: null };
-    const cutoff = now - this._windowMs;
-    let count = 0;
-    for (let i = 0; i < s.hits.length; i++) {
-      if (s.hits[i] >= cutoff) count += 1;
-    }
-    const oldest = s.hits[0];
-    return { remaining: this._limit - count, resetAt: oldest ? oldest + this._windowMs : null };
-  }
-
-  snapshot() {
-    const out = {};
-    for (const [k, v] of this._state) out[k] = { hits: v.hits.length };
-    return out;
-  }
-}

package/lib/canary.js

@@ -1,63 +0,0 @@
-// canary.js — canary probing before releasing a key from cooldown (issue #196, #7).
-
-export const CANARY_PROBE_TIMEOUT_MS = 5000;
-export const CANARY_DEFAULT_INTERVAL_MS = 30 * 1000;
-
-export class CanaryProber {
-  constructor(opts) {
-    opts = opts || {};
-    if (!opts.sandboxRunner) throw new Error('canary: sandboxRunner required');
-    this._runner = opts.sandboxRunner;
-    this._intervalMs = opts.intervalMs || CANARY_DEFAULT_INTERVAL_MS;
-    this._probeTargetModel = Boolean(opts.probeTargetModel);
-    this._results = new Map();
-    this._inProgress = new Set();
-  }
-
-  get intervalMs() { return this._intervalMs; }
-  get probeTargetModel() { return this._probeTargetModel; }
-
-  async probe(ref, key, targetModel = null) {
-    if (!ref || this._inProgress.has(ref)) return null;
-    this._inProgress.add(ref);
-    try {
-      let result;
-      if (this._probeTargetModel && targetModel && typeof this._runner.probeChatCompletion === 'function') {
-        result = await this._runner.probeChatCompletion(ref, key, targetModel);
-      } else {
-        result = await this._runner.probeModels(ref, key);
-      }
-      this._results.set(ref, Object.assign({}, result, { at: Date.now() }));
-      return result;
-    } catch (e) {
-      return { ok: false, code: 'error', at: Date.now() };
-    } finally {
-      this._inProgress.delete(ref);
-    }
-  }
-
-  lastResult(ref) {
-    return this._results.get(ref) || null;
-  }
-
-  isHealthy(ref) {
-    const r = this._results.get(ref);
-    return Boolean(r && r.ok);
-  }
-
-  clear(ref) {
-    if (ref) {
-      this._results.delete(ref);
-      this._inProgress.delete(ref);
-    } else {
-      this._results.clear();
-      this._inProgress.clear();
-    }
-  }
-
-  snapshot() {
-    const out = {};
-    for (const [k, v] of this._results) out[k] = v;
-    return out;
-  }
-}

package/lib/incident.js

@@ -1,76 +0,0 @@
-// lib/incident.js — auto-create Gitea issue when pool exhausted > threshold.
-// ponytail: minimal — caller provides a token + base URL. No retries on rate-limit.
-
-export const INCIDENT_DEFAULT_THRESHOLD_MS = 5 * 60 * 1000; // 5 min
-export const INCIDENT_DEFAULT_COOLDOWN_MS = 30 * 60 * 1000; // 30 min between incidents per provider
-export const INCIDENT_TIMEOUT_MS = 5000;
-
-export class IncidentReporter {
-  constructor({ token, baseUrl, repo, thresholdMs = INCIDENT_DEFAULT_THRESHOLD_MS, cooldownMs = INCIDENT_DEFAULT_COOLDOWN_MS, fetchImpl } = {}) {
-    if (!token) throw new Error('incident: token required');
-    if (!baseUrl) throw new Error('incident: baseUrl required');
-    if (!repo || !repo.includes('/')) throw new Error('incident: repo (owner/name) required');
-    this._token = token;
-    this._baseUrl = baseUrl.replace(/\/+$/, '');
-    this._repo = repo;
-    this._thresholdMs = thresholdMs;
-    this._cooldownMs = cooldownMs;
-    this._lastIncidentAt = new Map(); // provider -> epochMs
-    this._fetch = fetchImpl || (typeof fetch !== 'undefined' ? fetch : () => { throw new Error('incident: no fetch available'); });
-  }
-
-  // Should we report now? Pure; does not perform I/O.
-  shouldReport(provider, exhaustedSince, now = Date.now()) {
-    if (!provider) return false;
-    if (!Number.isFinite(exhaustedSince)) return false;
-    if (now - exhaustedSince < this._thresholdMs) return false;
-    const last = this._lastIncidentAt.get(provider);
-    if (Number.isFinite(last) && now - last < this._cooldownMs) return false;
-    return true;
-  }
-
-  markReported(provider, at = Date.now()) {
-    this._lastIncidentAt.set(provider, at);
-  }
-
-  resetCooldown(provider) {
-    if (provider) this._lastIncidentAt.delete(provider);
-    else this._lastIncidentAt.clear();
-  }
-
-  // Open a Gitea issue. ponytail: minimal payload, ignore failures.
-  async open(provider, exhaustedSince, now = Date.now()) {
-    if (!this.shouldReport(provider, exhaustedSince, now)) return { reported: false };
-    const url = `${this._baseUrl}/api/v1/repos/${this._repo}/issues`;
-    const body = {
-      title: `prod-incident: pool ${provider} exhausted since ${new Date(exhaustedSince).toISOString()}`,
-      body: [
-        'Auto-generated by `dsh-key-rotation`.',
-        '',
-        `- provider: \`${provider}\``,
-        `- exhaustedSince: \`${new Date(exhaustedSince).toISOString()}\``,
-        '',
-        'All keys in the pool are in cooldown or missing. Check OpenCode provider status and rotate keys.',
-      ].join('\n'),
-      labels: ['prod-incident'],
-    };
-    const ctrl = new AbortController();
-    const timer = setTimeout(() => ctrl.abort(), INCIDENT_TIMEOUT_MS);
-    try {
-      const res = await this._fetch(url, {
-        method: 'POST',
-        headers: { authorization: `token ${this._token}`, 'content-type': 'application/json' },
-        body: JSON.stringify(body),
-        signal: ctrl.signal,
-      });
-      if (!res.ok) return { reported: false, status: res.status };
-      const data = await res.json();
-      this.markReported(provider, now);
-      return { reported: true, number: data.number, url: data.html_url };
-    } catch (_) {
-      return { reported: false };
-    } finally {
-      clearTimeout(timer);
-    }
-  }
-}

package/lib/maintenance.js

@@ -1,64 +0,0 @@
-// lib/maintenance.js - pure helpers for #207 (expiry pre-warning) and
-// #208 (cost budget). No I/O; the timers in index.js call these and decide
-// whether to send webhooks.
-
-const DAY_MS = 86400000;
-
-/**
- * #207: keys of `pool` whose expiresAt falls within the next `warnDays`.
- * Returns [{ ref, expiresInDays, expiresAt }], soonest first.
- */
-export function expiringSoon(pool, warnDays, now = Date.now()) {
-  if (!pool || !pool.expiresAt) return [];
-  const horizon = now + Math.max(1, warnDays ?? 7) * DAY_MS;
-  return Object.entries(pool.expiresAt)
-    .filter(([, at]) => at > now && at <= horizon)
-    .map(([ref, at]) => ({ ref, expiresAt: at, expiresInDays: Math.max(0, Math.floor((at - now) / DAY_MS)) }))
-    .sort((a, b) => a.expiresAt - b.expiresAt);
-}
-
-/** #207 dedupe: true when a day-level notification for `key` is due. */
-export function shouldNotifyDaily(lastNotified, key, now = Date.now()) {
-  if (!lastNotified.has(key)) {
-    lastNotified.set(key, now);
-    return true;
-  }
-  const last = lastNotified.get(key);
-  if (now - last < DAY_MS) return false;
-  lastNotified.set(key, now);
-  return true;
-}
-
-/**
- * #208: total spend of a pool on ISO day `day` (defaults to today)
- * across costDays Map<ref, Map<day, cost>>.
- */
-export function costForDay(costDays, day) {
-  const d = day ?? new Date().toISOString().slice(0, 10);
-  let total = 0;
-  for (const perRef of (costDays?.values() ?? [])) {
-    total += perRef.get(d) ?? 0;
-  }
-  return total;
-}
-
-/** #208: total spend over the last 7 ISO days ending today. */
-export function costForWeek(costDays, now = Date.now()) {
-  let total = 0;
-  for (let i = 0; i < 7; i++) {
-    const d = new Date(now - i * 86400000).toISOString().slice(0, 10);
-    total += costForDay(costDays, d);
-  }
-  return total;
-}
-
-/**
- * #208 budget verdict for a pool: { spend, budget, ratio, warn, exceeded }.
- * budget <= 0 -> never warn.
- */
-export function budgetVerdict(spend, budget) {
-  if (!budget || budget <= 0) return { spend, budget: 0, ratio: 0, warn: false, exceeded: false };
-  const ratio = spend / budget;
-  // warn from 80%, exceeded at 100%
-  return { spend, budget, ratio, warn: ratio >= 0.8, exceeded: ratio >= 1 };
-}

package/lib/region.js

@@ -1,50 +0,0 @@
-// lib/region.js — region tag + failover helper.
-// ponytail: simple — providers declare an optional 'region' (string).
-// When the primary provider hits exhaustion AND a same-region fallback is
-// configured, the plugin picks that as next fallback.
-
-export const REGION_NONE = '';
-export const REGION_GLOBAL = 'global';
-
-export class RegionMap {
-  constructor() {
-    this._byProvider = new Map(); // provider id -> region
-  }
-
-  set(provider, region = REGION_GLOBAL) {
-    if (!provider) return;
-    if (!region) region = REGION_GLOBAL;
-    this._byProvider.set(provider, region);
-  }
-
-  get(provider) {
-    return this._byProvider.get(provider) || REGION_GLOBAL;
-  }
-
-  // Pick a fallback for `provider`. Returns another provider in the same
-  // region if available; otherwise null. Returns null for unknown providers
-  // (we don't know their region -> conservative).
-  pickFallback(provider) {
-    if (!this._byProvider.has(provider)) return null;
-    const region = this.get(provider);
-    for (const [p, r] of this._byProvider) {
-      if (p === provider) continue;
-      if (r === region) return p;
-    }
-    return null;
-  }
-
-  snapshot() {
-    const out = {};
-    for (const [k, v] of this._byProvider) out[k] = v;
-    return out;
-  }
-
-  clear() {
-    this._byProvider.clear();
-  }
-
-  get size() {
-    return this._byProvider.size;
-  }
-}

package/lib/shadow.js

@@ -1,81 +0,0 @@
-// lib/shadow.js — shadow A/B traffic sampling.
-// ponytail: per-provider counter, simple percent gating.
-
-export const SHADOW_DEFAULT_PERCENT = 0;  // 0 = disabled
-export const SHADOW_BUCKET = 100;        // percent base
-
-export class ShadowRouter {
-  constructor({ primary, secondary, percent = SHADOW_DEFAULT_PERCENT } = {}) {
-    this._primary = primary || '';
-    this._secondary = secondary || '';
-    this._percent = Number.isFinite(percent) && percent > 0 ? Math.min(SHADOW_BUCKET, Math.floor(percent)) : 0;
-    this._sent = 0;
-    this._shadowed = 0;
-    this._latencySumPrimary = 0;
-    this._latencySumSecondary = 0;
-    this._latencyCountPrimary = 0;
-    this._latencyCountSecondary = 0;
-  }
-
-  isEnabled() {
-    return this._percent > 0 && Boolean(this._primary) && Boolean(this._secondary) && this._primary !== this._secondary;
-  }
-
-  pick(requestHash = Math.random()) {
-    if (!this.isEnabled()) return { primary: this._primary, secondary: null, sampled: false };
-    // Convert requestHash to [0, SHADOW_BUCKET)
-    let h;
-    if (typeof requestHash === 'number') {
-      h = Math.floor(requestHash * SHADOW_BUCKET);
-    } else {
-      // Stable hash: fnv1a-lite on string
-      let str = String(requestHash);
-      let x = 2166136261;
-      for (let i = 0; i < str.length; i++) {
-        x ^= str.charCodeAt(i);
-        x = (x * 16777619) >>> 0;
-      }
-      h = x % SHADOW_BUCKET;
-    }
-    const sampled = h < this._percent;
-    this._sent += 1;
-    if (sampled) this._shadowed += 1;
-    return { primary: this._primary, secondary: sampled ? this._secondary : null, sampled };
-  }
-
-  recordLatency(target, ms) {
-    if (!Number.isFinite(ms) || ms < 0) return;
-    if (target === this._primary) {
-      this._latencySumPrimary += ms;
-      this._latencyCountPrimary += 1;
-    } else if (target === this._secondary) {
-      this._latencySumSecondary += ms;
-      this._latencyCountSecondary += 1;
-    }
-  }
-
-  snapshot() {
-    const avg = (sum, count) => (count > 0 ? sum / count : null);
-    return {
-      primary: this._primary,
-      secondary: this._secondary,
-      percent: this._percent,
-      enabled: this.isEnabled(),
-      sent: this._sent,
-      shadowed: this._shadowed,
-      avgLatencyMs: {
-        primary: avg(this._latencySumPrimary, this._latencyCountPrimary),
-        secondary: avg(this._latencySumSecondary, this._latencyCountSecondary),
-      },
-    };
-  }
-
-  reset() {
-    this._sent = 0;
-    this._shadowed = 0;
-    this._latencySumPrimary = 0;
-    this._latencySumSecondary = 0;
-    this._latencyCountPrimary = 0;
-    this._latencyCountSecondary = 0;
-  }
-}