@goodandready/dsh-key-rotation 0.8.7 → 0.8.10

package/lib/index.js

@@ -27,8 +27,9 @@ const HEALTH_PATH = '/dsh-key-rotation/health';
 const USAGE_PATH = '/dsh-key-rotation/usage';
 const TEST_PATH = '/dsh-key-rotation/test';
 const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
+import { sortAttemptList } from './pool.js';
 import { LastTestCache, SandboxRunner } from './sandbox.js';
-import { healIdleCooldowns } from './heal.js';
+import { healIdleCooldowns, autoUnbreakBrokenKeys } from './heal.js';
 import { LatencyHistogram } from './histogram.js';
 import { pickCascadeFallback } from './cascade.js';
 import { ConcurrencyTracker } from './concurrency.js';
@@ -199,6 +200,9 @@ export const Config = Schema.object({
     hour: Schema.number().default(0),
   }),
   rateLimitThreshold: Schema.number().default(0.1),
+  proactiveRateLimitGuard: Schema.boolean().default(true),
+  selfHealingIntervalMinutes: Schema.number().default(30),
+  routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency']).default('round-robin'),
   rpmLimit: Schema.number().default(0),
   webhookActionToken: Schema.string().role('secret').default(''),
   expiryWarnDays: Schema.number().default(7),
@@ -230,6 +234,8 @@ export const Config = Schema.object({
     })).default({}),
     cooldownMs: Schema.number(),
     maxCooldownMs: Schema.number(),
+    routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency']),
+    proactiveRateLimitGuard: Schema.boolean(),
   })).default([...DEFAULT_PROVIDERS]),
 });
 
@@ -339,6 +345,29 @@ export function apply(ctx, config = {}) {
     if (snap) statePersistence.save(snap);
   }
 
+    // Auto-healing / auto-unbreak for broken keys (#303)
+  ctx.effect(() => {
+    const cfg = getConfig();
+    const intervalMin = cfg?.selfHealingIntervalMinutes ?? 30;
+    if (!intervalMin || intervalMin <= 0) return () => {};
+    const intervalMs = intervalMin * 60 * 1000;
+    const timer = setInterval(async () => {
+      try {
+        const c = getConfig();
+        if (!c || !c.selfHealingIntervalMinutes || c.selfHealingIntervalMinutes <= 0) return;
+        const { pools } = buildRuntime();
+        const runner = ensureSandboxRunner(ctx);
+        await autoUnbreakBrokenKeys(pools, async (ref) => {
+          let val = (await ctx.credentials?.resolve?.(ref))?.value;
+          if (!val) return { ok: false };
+          return runner.probeModels(ref, val);
+        });
+      } catch (_) {}
+    }, intervalMs);
+    if (typeof timer.unref === 'function') timer.unref();
+    return () => clearInterval(timer);
+  }, 'dsh-key-rotation: auto-unbreak');
+
   ctx.effect(() => {
     const timer = setInterval(() => { try { schedulePersist(); } catch (_) {} }, 15000);
     if (typeof timer.unref === 'function') timer.unref();
@@ -372,9 +401,10 @@ export function apply(ctx, config = {}) {
           }
         }
       }
-      const n = sweepExpired(poolState, now);
-      if (n > 0) console.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
       const runtime = buildRuntime();
+      const allActiveRefs = Array.from(runtime.poolByRef.keys());
+      const n = sweepExpired(poolState, now, allActiveRefs);
+      if (n > 0) console.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
       for (const pool of runtime.poolByRef.values()) {
         compactUsage(pool, 30, now);
       }
@@ -564,7 +594,7 @@ export function apply(ctx, config = {}) {
       if (typeof v === 'string' && v.length > 0) { const ts = Date.parse(v); return Number.isNaN(ts) ? undefined : ts; }
       return undefined;
     };
-    const buildPool = (base, keys, weights, poolCooldown, poolMax, expiresAt) => {
+    const buildPool = (base, keys, weights, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard) => {
       const refs = (keys ?? []).filter((ref) => typeof ref === 'string' && ref.length > 0);
       if (refs.length === 0) return null;
       const w = Array.isArray(weights) ? weights : [];
@@ -582,7 +612,7 @@ export function apply(ctx, config = {}) {
       }
       return { base, refs, weights: refs.map((_, i) => (typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1)),
                weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs,
-               state: makeState(base), cooldownMs: poolCooldown, maxCooldownMs: poolMax, expiresAt: parsedExpiry, rpmLimit };
+               state: makeState(base), cooldownMs: poolCooldown, maxCooldownMs: poolMax, expiresAt: parsedExpiry, rpmLimit, routingStrategy: poolStrategy, proactiveRateLimitGuard: poolGuard };
     };
     for (const p of cfg.providers ?? []) {
       const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
@@ -656,7 +686,10 @@ export function apply(ctx, config = {}) {
         }
       }
     }
-    cachedRuntime = { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, concurrencyLimit, cascade, quotaResetWindow, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, verboseLogging: cfg.verboseLogging ?? false, switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds, expectedClones,
+    cachedRuntime = { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, concurrencyLimit, cascade, quotaResetWindow, rateLimitThreshold, rpmLimit, webhookActionToken, expiryWarnDays: cfg.expiryWarnDays ?? 7, switchNotify: cfg.switchNotify ?? false, verboseLogging: cfg.verboseLogging ?? false,
+      proactiveRateLimitGuard: cfg.proactiveRateLimitGuard ?? true,
+      selfHealingIntervalMinutes: cfg.selfHealingIntervalMinutes ?? 30,
+      routingStrategy: cfg.routingStrategy ?? 'round-robin', switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000, warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0, providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider, cloneIds, expectedClones,
       circuitBreakerEnabled: cfg.circuitBreakerEnabled ?? true,
       circuitBreakerThreshold: cfg.circuitBreakerThreshold ?? 5,
       circuitBreakerOpenMs: cfg.circuitBreakerOpenMs ?? 30000,
@@ -684,8 +717,12 @@ export function apply(ctx, config = {}) {
       const pool = poolByRef.get(ref);
       if (!pool) return original(ref);
       const now = Date.now();
-      const list = pool.weightedRefs ?? pool.refs;
-      const start = pool.state.pointer ?? 0;
+      const strat = pool.routingStrategy ?? buildRuntime().routingStrategy ?? 'round-robin';
+      let list = pool.weightedRefs ?? pool.refs;
+      if (strat === 'lowest-latency' || strat === 'least-loaded') {
+        list = sortAttemptList(list, strat, { latencyHistogram, concurrencyTracker });
+      }
+      const start = (strat === 'round-robin') ? (pool.state.pointer ?? 0) : 0;
       for (let i = 0; i < list.length; i++) {
         const index = (start + i) % list.length;
         const candidate = list[index];
@@ -805,6 +842,7 @@ export function apply(ctx, config = {}) {
     notifySwitch: (runtime, pool, info) => notifySwitch(runtime, pool, info, { webhookSender, notifyQueue: moduleNotifyQueue, now: () => Date.now() }),
     notifyExhaustion,
     recordLatency,
+    latencyHistogram,
     concurrencyTracker,
     MARKER,
     finishError,
@@ -827,6 +865,7 @@ export function apply(ctx, config = {}) {
     getRotationDisabled: () => rotationDisabled,
     setRotationDisabled: (v) => { rotationDisabled = v; },
     circuitBreaker: moduleBreaker,
+    quotaStore,
   });
 
   ctx.effect(() => ctx.on('llm/stream', (options, next) => {

package/lib/pool.js

@@ -184,9 +184,10 @@ export function envValue(ref) {
   return typeof v === 'string' && v.length > 0 ? v : undefined;
 }
 
-/** Sweep expired cooldown entries from poolState. Returns count of cleared refs. */
-export function sweepExpired(poolState, now = Date.now()) {
+/** Sweep expired cooldown entries and prune stale deleted refs from poolState. Returns count of cleared refs. */
+export function sweepExpired(poolState, now = Date.now(), activeRefs = null) {
   let cleared = 0;
+  const activeSet = activeRefs ? new Set(activeRefs) : null;
   for (const st of poolState.values()) {
     for (const [ref, until] of [...st.failedUntil.entries()]) {
       if (until <= now) {
@@ -195,6 +196,16 @@ export function sweepExpired(poolState, now = Date.now()) {
         cleared++;
       }
     }
+    // Prune stale entries for keys no longer in active configuration
+    if (activeSet) {
+      for (const map of [st.failedUntil, st.failCounts, st.authFailCounts, st.brokenUntil, st.costPerKey, st.lastUsedAt, st.usageCounts, st.byModel, st.usageDays, st.quotaWindows, st.rpmWindows, st.probedAt]) {
+        if (map && typeof map.keys === 'function') {
+          for (const k of [...map.keys()]) {
+            if (!activeSet.has(k)) map.delete(k);
+          }
+        }
+      }
+    }
   }
   return cleared;
 }
@@ -248,33 +259,105 @@ export function computeHealthScore(state) {
 /** Extract rate-limit info from an object that may carry response headers. */
 export function extractRateLimit(headers) {
   if (!headers || typeof headers !== 'object') return null;
-  let remaining, limit, reset;
+  let remaining, limit, reset, retryAfter;
   for (const k in headers) {
-    const lk = k.length;
-    if (lk === 21 || lk === 16 || lk === 17) {
-      const lower = k.toLowerCase();
-      if (lower === 'x-ratelimit-remaining') {
-        const v = headers[k];
-        if (v != null) remaining = Number(String(v));
-      } else if (lower === 'x-ratelimit-limit') {
-        const v = headers[k];
-        if (v != null) limit = Number(String(v));
-      } else if (lower === 'x-ratelimit-reset') {
-        const v = headers[k];
-        if (v != null) reset = Number(String(v));
+    const lower = k.toLowerCase();
+    const v = headers[k];
+    if (v == null) continue;
+    if (lower === 'retry-after') {
+      const num = Number(v);
+      if (Number.isFinite(num) && num >= 0) {
+        retryAfter = Math.ceil(num);
+      } else {
+        const parsed = Date.parse(v);
+        if (!Number.isNaN(parsed)) {
+          retryAfter = Math.max(0, Math.ceil((parsed - Date.now()) / 1000));
+        }
+      }
+    } else if (
+      lower === 'x-ratelimit-remaining' ||
+      lower === 'x-ratelimit-remaining-requests' ||
+      lower === 'anthropic-ratelimit-requests-remaining' ||
+      lower === 'openai-ratelimit-remaining-requests' ||
+      lower === 'ratelimit-remaining'
+    ) {
+      const n = Number(String(v));
+      if (Number.isFinite(n)) remaining = n;
+    } else if (
+      (lower === 'x-ratelimit-remaining-tokens' || lower === 'anthropic-ratelimit-tokens-remaining') &&
+      remaining === undefined
+    ) {
+      const n = Number(String(v));
+      if (Number.isFinite(n)) remaining = n;
+    } else if (
+      lower === 'x-ratelimit-limit' ||
+      lower === 'x-ratelimit-limit-requests' ||
+      lower === 'anthropic-ratelimit-requests-limit' ||
+      lower === 'openai-ratelimit-limit-requests' ||
+      lower === 'ratelimit-limit'
+    ) {
+      const n = Number(String(v));
+      if (Number.isFinite(n)) limit = n;
+    } else if (
+      lower === 'x-ratelimit-reset' ||
+      lower === 'x-ratelimit-reset-requests' ||
+      lower === 'anthropic-ratelimit-requests-reset' ||
+      lower === 'ratelimit-reset'
+    ) {
+      const n = Number(String(v));
+      if (Number.isFinite(n)) {
+        reset = n;
+      } else {
+        const parsed = Date.parse(v);
+        if (!Number.isNaN(parsed)) reset = parsed;
       }
     }
   }
-  if (remaining === undefined && limit === undefined) return null;
-  return { remaining, limit, reset };
+  if (remaining === undefined && limit === undefined && retryAfter === undefined && reset === undefined) return null;
+  const out = { remaining, limit, reset };
+  if (retryAfter !== undefined) out.retryAfter = retryAfter;
+  return out;
 }
 
-/** True if remaining is below the given threshold fraction of limit (e.g. 0.1). */
+/** True if remaining is below the given threshold fraction of limit (e.g. 0.1) or retry-after is active. */
 export function isRateLimited(rate, threshold = 0.1) {
   if (!rate) return false;
-  if (rate.remaining === undefined) return false;
-  if (rate.limit && rate.limit > 0) return rate.remaining < rate.limit * threshold;
-  return rate.remaining <= 0;
+  if (typeof rate.retryAfter === 'number' && rate.retryAfter > 0) return true;
+  if (rate.remaining !== undefined && rate.remaining <= 1) return true;
+  if (rate.limit && rate.limit > 0 && rate.remaining !== undefined) {
+    return rate.remaining < rate.limit * threshold;
+  }
+  if (rate.remaining !== undefined) return rate.remaining <= 0;
+  return false;
+}
+
+/**
+ * Sort or prioritize an attempt list of refs according to routing strategy.
+ * Strategies:
+ * - 'round-robin': preserves order
+ * - 'least-loaded': keys with lowest active concurrency count first
+ * - 'lowest-latency': keys with lowest p95 (or avg) latency first; unsampled keys neutral
+ */
+export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
+  if (!Array.isArray(refs) || refs.length <= 1) return (refs ?? []).slice();
+  const list = refs.slice();
+  if (strategy === 'least-loaded' && deps.concurrencyTracker && typeof deps.concurrencyTracker.getActive === 'function') {
+    return list.sort((a, b) => {
+      const ca = deps.concurrencyTracker.getActive(a) ?? 0;
+      const cb = deps.concurrencyTracker.getActive(b) ?? 0;
+      return ca - cb;
+    });
+  }
+  if (strategy === 'lowest-latency' && deps.latencyHistogram && typeof deps.latencyHistogram.snapshot === 'function') {
+    return list.sort((a, b) => {
+      const sa = deps.latencyHistogram.snapshot(a);
+      const sb = deps.latencyHistogram.snapshot(b);
+      const la = (sa && Number.isFinite(sa.p95)) ? sa.p95 : ((sa && Number.isFinite(sa.avg)) ? sa.avg : 500);
+      const lb = (sb && Number.isFinite(sb.p95)) ? sb.p95 : ((sb && Number.isFinite(sb.avg)) ? sb.avg : 500);
+      return la - lb;
+    });
+  }
+  return list;
 }
 
 /**

package/lib/rotate.js

@@ -9,6 +9,7 @@ import {
   extractRateLimit,
   isRateLimited,
   formatExhaustionMessage,
+  sortAttemptList,
 } from './pool.js';
 import { pickCascadeFallback } from './cascade.js';
 
@@ -25,6 +26,7 @@ export function createRotate(deps) {
     notifySwitch,
     notifyExhaustion,
     recordLatency,
+    latencyHistogram,
     concurrencyTracker,
     MARKER,
     finishError,
@@ -42,7 +44,11 @@ export function createRotate(deps) {
       const reqStore = { pool, pickedRef: undefined, startMs: now() };
       setRotateStartMs(reqStore.startMs);
       let attemptList = (pool.weightedRefs ?? pool.refs).slice();
-      if (runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
+      const strategy = pool.routingStrategy ?? runtime0.routingStrategy ?? 'round-robin';
+      if (strategy !== 'round-robin') {
+        attemptList = sortAttemptList(attemptList, strategy, { latencyHistogram, concurrencyTracker });
+      }
+      if (strategy !== 'least-loaded' && runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
         // #193: prefer least-loaded key within limit
         const available = attemptList.filter((r) => {
           const fu = pool.state.failedUntil.get(r) ?? 0;
@@ -115,122 +121,136 @@ export function createRotate(deps) {
         }
 
         const _pickedRef = reqStore.pickedRef ?? pool.state.lastUsed;
-        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.acquire(_pickedRef);
+        const acquired = (_pickedRef && runtime0.concurrencyLimit > 0) ? concurrencyTracker.acquire(_pickedRef) : false;
         try {
-          for await (const chunk of inner) {
-            // Only actual content deltas lock the stream (no more rotation).
-            // Structural/metadata chunks (block-start/end, usage) do not.
-            if (chunk && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta')) {
-              yielded = true;
-              yield chunk;
-              continue;
-            }
-            if (chunk && chunk.type === 'finish') {
-              const kind = chunk.reason?.kind;
-              const failure = chunk.reason?.failure;
-              const code = failure?.code;
-              const message = failure?.message ?? '';
-              const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-              const switchable = !yielded && kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes);
-              const activeRef = reqStore.pickedRef ?? pool.state.lastUsed;
-              if (switchable) {
-                penalizeRef(activeRef, code ?? 'UNKNOWN', message);
-                pool.state.switches = (pool.state.switches ?? 0) + 1;
-                pool.state.lastReason = String(code ?? 'UNKNOWN');
-                pool.state.lastSwitchAt = now();
-                lastFailure = chunk;
-                console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) - next key`);
-                // #216: per-switch webhook (opt-in switchNotify), deduped per provider
-                if (switchNotify && activeRef) {
-                  notifySwitch(runtime0, pool, {
-                    provider: options.provider,
-                    from: activeRef,
-                    code: String(code ?? 'UNKNOWN'),
-                    at: pool.state.lastSwitchAt,
-                  });
-                }
-                switching = true;
-                break;
+          try {
+            for await (const chunk of inner) {
+              // Only actual content deltas lock the stream (no more rotation).
+              // Structural/metadata chunks (block-start/end, usage) do not.
+              if (chunk && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta')) {
+                yielded = true;
+                yield chunk;
+                continue;
               }
-              // cost tracking if provider returns usage.cost
-              const todayIso = activeRef ? new Date().toISOString().slice(0, 10) : undefined;
-              if (chunk.usage?.cost != null && activeRef) {
-                const c = Number(chunk.usage.cost);
-                if (!isNaN(c)) {
-                  if (!pool.state.costPerKey) pool.state.costPerKey = new Map();
-                  pool.state.costPerKey.set(activeRef, (pool.state.costPerKey.get(activeRef) ?? 0) + c);
-                  // #208: cost per day per key (mirrors usageDays) for budget checks
-                  if (!pool.state.costDays) pool.state.costDays = new Map();
-                  const cMap = pool.state.costDays.get(activeRef) || new Map();
-                  cMap.set(todayIso, (cMap.get(todayIso) ?? 0) + c);
-                  pool.state.costDays.set(activeRef, cMap);
+              if (chunk && chunk.type === 'finish') {
+                const kind = chunk.reason?.kind;
+                const failure = chunk.reason?.failure;
+                const code = failure?.code;
+                const message = failure?.message ?? '';
+                const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
+                const switchable = !yielded && kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes);
+                const activeRef = reqStore.pickedRef ?? pool.state.lastUsed;
+                if (switchable) {
+                  penalizeRef(activeRef, code ?? 'UNKNOWN', message);
+                  pool.state.switches = (pool.state.switches ?? 0) + 1;
+                  pool.state.lastReason = String(code ?? 'UNKNOWN');
+                  pool.state.lastSwitchAt = now();
+                  lastFailure = chunk;
+                  console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) - next key`);
+                  // #216: per-switch webhook (opt-in switchNotify), deduped per provider
+                  if (switchNotify && activeRef) {
+                    notifySwitch(runtime0, pool, {
+                      provider: options.provider,
+                      from: activeRef,
+                      code: String(code ?? 'UNKNOWN'),
+                      at: pool.state.lastSwitchAt,
+                    });
+                  }
+                  switching = true;
+                  break;
                 }
-              }
-              // Usage by day (#119)
-              if (activeRef) {
-                if (!pool.state.usageDays) pool.state.usageDays = new Map();
-                const dayMap = pool.state.usageDays.get(activeRef) || new Map();
-                dayMap.set(todayIso, (dayMap.get(todayIso) ?? 0) + 1);
-                pool.state.usageDays.set(activeRef, dayMap);
-              }
-              // Per-model request detail (#121)
-              if (activeRef && options.model) {
-                if (!pool.state.byModel) pool.state.byModel = new Map();
-                let byRef = pool.state.byModel.get(activeRef);
-                if (!byRef) { byRef = new Map(); pool.state.byModel.set(activeRef, byRef); }
-                byRef.set(options.model, (byRef.get(options.model) ?? 0) + 1);
-              }
-              // Proactive rate-limit (#115): if response headers say this key is near
-              // its quota, cool it down so the NEXT request starts on a different key.
-              // We do NOT re-run this (already successful) request — that would double-send.
-              const rate = extractRateLimit(chunk?.metadata?.headers ?? chunk?.headers);
-              if (rate && activeRef) {
-                if (isRateLimited(rate, rateLimitThreshold ?? 0.1)) {
-                  const cool = rate.reset && rate.reset > now() ? (rate.reset - now()) : pool.cooldownMs;
-                  recordFailure(pool, activeRef, now(), cool, pool.maxCooldownMs);
-                  pushEvent(pool, activeRef, 'RATE_LIMIT', cool);
-                  console.warn(`[dsh-key-rotation] ${options.provider}: key ${activeRef} near quota (remaining ${String(rate.remaining)}/${String(rate.limit)}) — next request will rotate`);
+                // cost tracking if provider returns usage.cost
+                const todayIso = activeRef ? new Date().toISOString().slice(0, 10) : undefined;
+                if (chunk.usage?.cost != null && activeRef) {
+                  const c = Number(chunk.usage.cost);
+                  if (!isNaN(c)) {
+                    if (!pool.state.costPerKey) pool.state.costPerKey = new Map();
+                    pool.state.costPerKey.set(activeRef, (pool.state.costPerKey.get(activeRef) ?? 0) + c);
+                    // #208: cost per day per key (mirrors usageDays) for budget checks
+                    if (!pool.state.costDays) pool.state.costDays = new Map();
+                    const cMap = pool.state.costDays.get(activeRef) || new Map();
+                    cMap.set(todayIso, (cMap.get(todayIso) ?? 0) + c);
+                    pool.state.costDays.set(activeRef, cMap);
+                  }
                 }
-              }
-              // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
-              if (rate && activeRef && Number.isFinite(rate.remaining) && quotaStore) {
-                quotaStore.set(activeRef, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: now() });
+                // Usage by day (#119)
+                if (activeRef) {
+                  if (!pool.state.usageDays) pool.state.usageDays = new Map();
+                  const dayMap = pool.state.usageDays.get(activeRef) || new Map();
+                  dayMap.set(todayIso, (dayMap.get(todayIso) ?? 0) + 1);
+                  pool.state.usageDays.set(activeRef, dayMap);
+                }
+                // Per-model request detail (#121)
+                if (activeRef && options.model) {
+                  if (!pool.state.byModel) pool.state.byModel = new Map();
+                  let byRef = pool.state.byModel.get(activeRef);
+                  if (!byRef) { byRef = new Map(); pool.state.byModel.set(activeRef, byRef); }
+                  byRef.set(options.model, (byRef.get(options.model) ?? 0) + 1);
+                }
+                // Proactive rate-limit (#115, #303): if response headers say this key is near
+                // its quota or has retry-after, cool it down so the NEXT request starts on a different key.
+                // We do NOT re-run this (already successful) request — that would double-send.
+                const guardEnabled = pool.proactiveRateLimitGuard ?? runtime0.proactiveRateLimitGuard ?? true;
+                const rate = extractRateLimit(chunk?.metadata?.headers ?? chunk?.headers);
+                if (guardEnabled && rate && activeRef) {
+                  if (isRateLimited(rate, rateLimitThreshold ?? 0.1)) {
+                    let cool;
+                    if (typeof rate.retryAfter === 'number' && rate.retryAfter > 0) {
+                      cool = rate.retryAfter * 1000;
+                    } else if (rate.reset && rate.reset > now()) {
+                      cool = rate.reset - now();
+                    } else {
+                      cool = pool.cooldownMs ?? cooldownMs;
+                    }
+                    const maxCool = pool.maxCooldownMs ?? maxCooldownMs;
+                    const effCool = Math.min(cool, maxCool ?? cool);
+                    recordFailure(pool, activeRef, now(), effCool, maxCool);
+                    pushEvent(pool, activeRef, 'RATE_LIMIT', effCool);
+                    console.warn(`[dsh-key-rotation] ${options.provider}: key ${activeRef} proactive pause (remaining ${String(rate.remaining ?? '?')}/${String(rate.limit ?? '?')}, cool ${Math.round(effCool / 1000)}s) — next request will rotate`);
+                  }
+                }
+                // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
+                if (rate && activeRef && Number.isFinite(rate.remaining) && quotaStore) {
+                  quotaStore.set(activeRef, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: now() });
+                }
+                yield chunk;
+                recordLatency(pool, reqStore);
+                if (circuitBreaker) circuitBreaker.onSuccess(options.provider);
+                return;
               }
               yield chunk;
-              recordLatency(pool, reqStore);
-              if (circuitBreaker) circuitBreaker.onSuccess(options.provider);
-              return;
             }
-            yield chunk;
-          }
-        } catch (e) {
-          if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
-          const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-          const activeRef = _pickedRef ?? reqStore.pickedRef ?? pool.state.lastUsed;
-          if (!yielded && isSwitchableError(e, effectiveSwitchCodes)) {
-            penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e));
-            pool.state.switches = (pool.state.switches ?? 0) + 1;
-            pool.state.lastReason = String(e?.code ?? 'TRANSPORT');
-            pool.state.lastSwitchAt = now();
-            lastFailure = finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
-            console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} stream threw ${String(e?.code ?? e?.message ?? e)} - failover to next key`);
-            if (switchNotify && activeRef) {
-              notifySwitch(runtime0, pool, {
-                provider: options.provider,
-                from: activeRef,
-                code: String(e?.code ?? 'TRANSPORT'),
-                at: pool.state.lastSwitchAt,
-              });
+          } catch (e) {
+            const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
+            const activeRef = _pickedRef ?? reqStore.pickedRef ?? pool.state.lastUsed;
+            if (!yielded && isSwitchableError(e, effectiveSwitchCodes)) {
+              penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+              pool.state.switches = (pool.state.switches ?? 0) + 1;
+              pool.state.lastReason = String(e?.code ?? 'TRANSPORT');
+              pool.state.lastSwitchAt = now();
+              lastFailure = finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+              console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} stream threw ${String(e?.code ?? e?.message ?? e)} - failover to next key`);
+              if (switchNotify && activeRef) {
+                notifySwitch(runtime0, pool, {
+                  provider: options.provider,
+                  from: activeRef,
+                  code: String(e?.code ?? 'TRANSPORT'),
+                  at: pool.state.lastSwitchAt,
+                });
+              }
+              continue; // Failover to next key!
             }
-            continue; // Failover to next key!
+            yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+            return;
           }
-          yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
-          return;
-        }
 
-        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
-        if (switching) continue; // try the next key
-        return; // clean end — served
+          if (switching) continue; // try the next key
+          return; // clean end — served
+        } finally {
+          if (acquired && _pickedRef) {
+            concurrencyTracker.release(_pickedRef);
+          }
+        }
       }
 
       // pool exhausted — all keys cooling or missing