@goodandready/dsh-key-rotation 0.7.39 → 0.8.0

package/lib/notify-queue.js

@@ -0,0 +1,75 @@
+// lib/notify-queue.js — non-blocking webhook dispatch with backoff (#263).
+// rotate() must never await webhook I/O.
+
+export class NotifyQueue {
+  /**
+   * @param {{ send: (url:string, payload:object)=>Promise<any>, maxDepth?: number, baseBackoffMs?: number, maxBackoffMs?: number }} opts
+   */
+  constructor({ send, maxDepth = 50, baseBackoffMs = 2000, maxBackoffMs = 60000 } = {}) {
+    if (typeof send !== 'function') throw new Error('NotifyQueue: send required');
+    this._send = send;
+    this.maxDepth = maxDepth;
+    this.baseBackoffMs = baseBackoffMs;
+    this.maxBackoffMs = maxBackoffMs;
+    this._q = [];
+    this._busy = false;
+    this._backoffByUrl = new Map();
+    this._dropped = 0;
+    this._sent = 0;
+    this._failed = 0;
+  }
+
+  enqueue(url, payload) {
+    if (!url) return { queued: false, reason: 'no-url' };
+    if (this._q.length >= this.maxDepth) {
+      this._dropped += 1;
+      return { queued: false, reason: 'full', dropped: this._dropped };
+    }
+    this._q.push({ url, payload });
+    this._pump();
+    return { queued: true };
+  }
+
+  _pump() {
+    if (this._busy) return;
+    const job = this._q.shift();
+    if (!job) return;
+    this._busy = true;
+    const delay = this._backoffByUrl.get(job.url) ?? 0;
+    const run = async () => {
+      try {
+        const res = await this._send(job.url, job.payload);
+        if (res && res.sent === false) {
+          this._failed += 1;
+          this._bumpBackoff(job.url);
+        } else {
+          this._sent += 1;
+          this._backoffByUrl.delete(job.url);
+        }
+      } catch {
+        this._failed += 1;
+        this._bumpBackoff(job.url);
+      } finally {
+        this._busy = false;
+        if (this._q.length > 0) this._pump();
+      }
+    };
+    if (delay > 0) {
+      const t = setTimeout(run, delay);
+      if (typeof t.unref === 'function') t.unref();
+    } else {
+      // fire and forget — do not return a promise to callers
+      run();
+    }
+  }
+
+  _bumpBackoff(url) {
+    const cur = this._backoffByUrl.get(url) ?? 0;
+    const next = cur === 0 ? this.baseBackoffMs : Math.min(cur * 2, this.maxBackoffMs);
+    this._backoffByUrl.set(url, next);
+  }
+
+  stats() {
+    return { depth: this._q.length, dropped: this._dropped, sent: this._sent, failed: this._failed, busy: this._busy };
+  }
+}

package/lib/pool.js

@@ -288,7 +288,8 @@ export function isSwitchableError(failureOrPayload, switchCodes = new Set(DEFAUL
   const code = String(failure.code ?? failure.reason ?? '').toUpperCase();
   const message = String(failure.message ?? failureOrPayload.message ?? '');
 
-  // 1. Direct HTTP status codes
+  // 1. Direct HTTP status codes (#267: also 408/425 transient)
+  if ((status === 429 || status === 408 || status === 425) && (switchCodes.has('RATE_LIMIT') || switchCodes.has('QUOTA') || switchCodes.has('429') || switchCodes.has('TIMEOUT') || switchCodes.has(String(status)))) return true;
   if (status === 429 && (switchCodes.has('RATE_LIMIT') || switchCodes.has('QUOTA') || switchCodes.has('429'))) return true;
   if ((status === 401 || status === 403) && (switchCodes.has('AUTH') || switchCodes.has('401'))) return true;
   if ((status >= 500 && status <= 504) && (switchCodes.has('SERVER') || switchCodes.has(String(status)))) return true;

package/lib/rotate.js

@@ -0,0 +1,261 @@
+// lib/rotate.js — stream rotation / failover generator (#253).
+// Pure wiring helper: dependencies are injected so unit tests can supply mocks.
+import { nowMono } from './clock.js';
+import { classifyFailure } from './error-taxonomy.js';
+import {
+  isSwitchableError,
+  recordFailure,
+  parseRetryAfter,
+  extractRateLimit,
+  isRateLimited,
+  formatExhaustionMessage,
+} from './pool.js';
+import { pickCascadeFallback } from './cascade.js';
+
+/**
+ * Create the rotate(options, pool) async generator used by llm/stream.
+ * @param {object} deps
+ */
+export function createRotate(deps) {
+  const {
+    ctx,
+    dispatchStorage,
+    buildRuntime,
+    pushEvent,
+    notifySwitch,
+    notifyExhaustion,
+    recordLatency,
+    concurrencyTracker,
+    MARKER,
+    finishError,
+    setRotateStartMs,
+    quotaStore,
+    circuitBreaker,
+    now = nowMono,
+  } = deps;
+
+  function rotate(options, pool) {
+    return (async function* () {
+      const runtime0 = buildRuntime();
+      const { switchCodes, cooldownMs, maxCooldownMs, switchNotify, rateLimitThreshold } = runtime0;
+      let lastFailure = null;
+      const reqStore = { pool, pickedRef: undefined, startMs: now() };
+      setRotateStartMs(reqStore.startMs);
+      let attemptList = (pool.weightedRefs ?? pool.refs).slice();
+      if (runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
+        // #193: prefer least-loaded key within limit
+        const available = attemptList.filter((r) => {
+          const fu = pool.state.failedUntil.get(r) ?? 0;
+          if (fu > now()) return false;
+          const exp = pool.expiresAt ? pool.expiresAt[r] : undefined;
+          if (exp !== undefined && now() >= exp) return false;
+          return true;
+        });
+        const preferred = concurrencyTracker.pickLeastLoaded(available);
+        if (preferred && attemptList[0] !== preferred) {
+          const list = attemptList.slice();
+          const i = list.indexOf(preferred);
+          if (i > 0) { list.splice(i, 1); list.unshift(preferred); }
+          attemptList = list;
+        }
+      }
+
+      const penalizeRef = (targetRef, errCode, errMsg) => {
+        if (!targetRef) return;
+        const _retry = parseRetryAfter(errMsg);
+        const _base = pool.cooldownMs ?? cooldownMs;
+        const _max = pool.maxCooldownMs ?? maxCooldownMs;
+        const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base;
+        const cls = classifyFailure({ code: errCode, message: errMsg });
+        const _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft);
+        if (circuitBreaker) circuitBreaker.onFailure(pool.base ?? options?.provider);
+        pushEvent(pool, targetRef, errCode ?? 'UNKNOWN', _b);
+        if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
+        if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
+        const _cStr = String(errCode ?? '');
+        if (_cStr === 'AUTH' || /auth/i.test(errMsg)) {
+          const _c2 = (pool.state.authFailCounts.get(targetRef) ?? 0) + 1;
+          pool.state.authFailCounts.set(targetRef, _c2);
+          if (_c2 >= 3) {
+            pool.state.brokenUntil.set(targetRef, now() + 86400000 * 30);
+            pool.state.failedUntil.set(targetRef, now() + 86400000 * 30);
+          }
+        } else {
+          pool.state.authFailCounts.delete(targetRef);
+        }
+      };
+
+      // #260: fail fast when provider circuit is open
+      if (circuitBreaker && !circuitBreaker.canRequest(options.provider)) {
+        console.warn(`[dsh-key-rotation] ${options.provider}: circuit open — skipping dispatch`);
+        yield finishError('CIRCUIT_OPEN', `[dsh-key-rotation] provider '${options.provider}' circuit is open`);
+        return;
+      }
+
+      for (let attempt = 0; attempt < attemptList.length; attempt++) {
+        let yielded = false;
+        let switching = false;
+        let inner;
+        try {
+          // mark the internal dispatch so the interceptor does not re-rotate
+          inner = dispatchStorage.run(reqStore, () => ctx.llm.stream({ ...options, [MARKER]: true }));
+        } catch (e) {
+          const curRef = reqStore.pickedRef ?? pool.state.lastUsed;
+          penalizeRef(curRef, e?.code ?? 'TRANSPORT', String(e?.message ?? ''));
+          lastFailure = finishError(e?.code ?? 'TRANSPORT',
+            `dsh-key-rotation: dispatch failed: ${String(e?.message ?? e)}`);
+          console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(curRef ?? '?')} threw ${String(e?.code ?? e?.message ?? e)}`);
+          continue;
+        }
+
+        const _pickedRef = reqStore.pickedRef ?? pool.state.lastUsed;
+        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.acquire(_pickedRef);
+        try {
+          for await (const chunk of inner) {
+            // Only actual content deltas lock the stream (no more rotation).
+            // Structural/metadata chunks (block-start/end, usage) do not.
+            if (chunk && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta')) {
+              yielded = true;
+              yield chunk;
+              continue;
+            }
+            if (chunk && chunk.type === 'finish') {
+              const kind = chunk.reason?.kind;
+              const failure = chunk.reason?.failure;
+              const code = failure?.code;
+              const message = failure?.message ?? '';
+              const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
+              const switchable = !yielded && kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes);
+              const activeRef = reqStore.pickedRef ?? pool.state.lastUsed;
+              if (switchable) {
+                penalizeRef(activeRef, code ?? 'UNKNOWN', message);
+                pool.state.switches = (pool.state.switches ?? 0) + 1;
+                pool.state.lastReason = String(code ?? 'UNKNOWN');
+                pool.state.lastSwitchAt = now();
+                lastFailure = chunk;
+                console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) - next key`);
+                // #216: per-switch webhook (opt-in switchNotify), deduped per provider
+                if (switchNotify && activeRef) {
+                  notifySwitch(runtime0, pool, {
+                    provider: options.provider,
+                    from: activeRef,
+                    code: String(code ?? 'UNKNOWN'),
+                    at: pool.state.lastSwitchAt,
+                  });
+                }
+                switching = true;
+                break;
+              }
+              // cost tracking if provider returns usage.cost
+              const todayIso = activeRef ? new Date().toISOString().slice(0, 10) : undefined;
+              if (chunk.usage?.cost != null && activeRef) {
+                const c = Number(chunk.usage.cost);
+                if (!isNaN(c)) {
+                  if (!pool.state.costPerKey) pool.state.costPerKey = new Map();
+                  pool.state.costPerKey.set(activeRef, (pool.state.costPerKey.get(activeRef) ?? 0) + c);
+                  // #208: cost per day per key (mirrors usageDays) for budget checks
+                  if (!pool.state.costDays) pool.state.costDays = new Map();
+                  const cMap = pool.state.costDays.get(activeRef) || new Map();
+                  cMap.set(todayIso, (cMap.get(todayIso) ?? 0) + c);
+                  pool.state.costDays.set(activeRef, cMap);
+                }
+              }
+              // Usage by day (#119)
+              if (activeRef) {
+                if (!pool.state.usageDays) pool.state.usageDays = new Map();
+                const dayMap = pool.state.usageDays.get(activeRef) || new Map();
+                dayMap.set(todayIso, (dayMap.get(todayIso) ?? 0) + 1);
+                pool.state.usageDays.set(activeRef, dayMap);
+              }
+              // Per-model request detail (#121)
+              if (activeRef && options.model) {
+                if (!pool.state.byModel) pool.state.byModel = new Map();
+                let byRef = pool.state.byModel.get(activeRef);
+                if (!byRef) { byRef = new Map(); pool.state.byModel.set(activeRef, byRef); }
+                byRef.set(options.model, (byRef.get(options.model) ?? 0) + 1);
+              }
+              // Proactive rate-limit (#115): if response headers say this key is near
+              // its quota, cool it down so the NEXT request starts on a different key.
+              // We do NOT re-run this (already successful) request — that would double-send.
+              const rate = extractRateLimit(chunk?.metadata?.headers ?? chunk?.headers);
+              if (rate && activeRef) {
+                if (isRateLimited(rate, rateLimitThreshold ?? 0.1)) {
+                  const cool = rate.reset && rate.reset > now() ? (rate.reset - now()) : pool.cooldownMs;
+                  recordFailure(pool, activeRef, now(), cool, pool.maxCooldownMs);
+                  pushEvent(pool, activeRef, 'RATE_LIMIT', cool);
+                  console.warn(`[dsh-key-rotation] ${options.provider}: key ${activeRef} near quota (remaining ${String(rate.remaining)}/${String(rate.limit)}) — next request will rotate`);
+                }
+              }
+              // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
+              if (rate && activeRef && Number.isFinite(rate.remaining) && quotaStore) {
+                quotaStore.set(activeRef, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: now() });
+              }
+              yield chunk;
+              recordLatency(pool, reqStore);
+              if (circuitBreaker) circuitBreaker.onSuccess(options.provider);
+              return;
+            }
+            yield chunk;
+          }
+        } catch (e) {
+          if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
+          const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
+          const activeRef = _pickedRef ?? reqStore.pickedRef ?? pool.state.lastUsed;
+          if (!yielded && isSwitchableError(e, effectiveSwitchCodes)) {
+            penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+            pool.state.switches = (pool.state.switches ?? 0) + 1;
+            pool.state.lastReason = String(e?.code ?? 'TRANSPORT');
+            pool.state.lastSwitchAt = now();
+            lastFailure = finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+            console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} stream threw ${String(e?.code ?? e?.message ?? e)} - failover to next key`);
+            if (switchNotify && activeRef) {
+              notifySwitch(runtime0, pool, {
+                provider: options.provider,
+                from: activeRef,
+                code: String(e?.code ?? 'TRANSPORT'),
+                at: pool.state.lastSwitchAt,
+              });
+            }
+            continue; // Failover to next key!
+          }
+          yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+          return;
+        }
+
+        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
+        if (switching) continue; // try the next key
+        return; // clean end — served
+      }
+
+      // pool exhausted — all keys cooling or missing
+      pool.state.lastExhaustionAt = now();
+      pool.state.exhaustionCount = (pool.state.exhaustionCount ?? 0) + 1;
+      console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — all ${pool.refs.length} keys cooling`);
+      const runtime = buildRuntime();
+      // notify via extracted helper (see notifyExhaustion above)
+      notifyExhaustion(runtime, pool, { provider: options.provider });
+
+      // #194: cross-provider cascade failover (guarded against infinite recursion)
+      if (!options.__isCascade && Array.isArray(runtime.cascade) && runtime.cascade.length > 0) {
+        const pools = runtime.providerToPool;
+        const fb = pickCascadeFallback(options.provider, runtime, pools);
+        if (fb && fb.pool && fb.pool !== pool) {
+          console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — cascading to ${fb.provider}`);
+          pool.state.lastReason = 'CASCADE';
+          pool.state.lastSwitchAt = now();
+          // Re-dispatch on the fallback pool (depth-1 via __isCascade guard)
+          const innerCascade = rotate({ ...options, provider: fb.provider, __isCascade: true }, fb.pool);
+          for await (const chunk of innerCascade) {
+            yield chunk;
+          }
+          return;
+        }
+      }
+
+      const exhaustionMsg = formatExhaustionMessage(options.provider, pool);
+      yield lastFailure ?? finishError('QUOTA', exhaustionMsg);
+    })();
+  }
+
+  return rotate;
+}