@goodandready/dsh-key-rotation 0.8.8 → 0.8.11

package/lib/pool.js

@@ -1,3 +1,4 @@
+import { bestEffort } from './best-effort.js';
 // lib/pool.js — pure pool arithmetic and selection logic for dsh-key-rotation.
 // Isolated from cordis/dsh runtime so unit tests run in vanilla Node.js.
 
@@ -71,11 +72,6 @@ export const SOFT_FAILURE_CODES = new Set([
 ]);
 
 /** True if failure code or message represents a soft/transient drop. */
-export function isSoftFailure(code, message = '') {
-  if (code && SOFT_FAILURE_CODES.has(String(code).toUpperCase())) return true;
-  if (/502|503|504|timeout|econnreset|econnrefused|socket hang up/i.test(message)) return true;
-  return false;
-}
 
 /** Ref name validator. Same rule lib/index.js enforces in PUT/DELETE /key. */
 const REF_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
@@ -184,9 +180,10 @@ export function envValue(ref) {
   return typeof v === 'string' && v.length > 0 ? v : undefined;
 }
 
-/** Sweep expired cooldown entries from poolState. Returns count of cleared refs. */
-export function sweepExpired(poolState, now = Date.now()) {
+/** Sweep expired cooldown entries and prune stale deleted refs from poolState. Returns count of cleared refs. */
+export function sweepExpired(poolState, now = Date.now(), activeRefs = null) {
   let cleared = 0;
+  const activeSet = activeRefs ? new Set(activeRefs) : null;
   for (const st of poolState.values()) {
     for (const [ref, until] of [...st.failedUntil.entries()]) {
       if (until <= now) {
@@ -195,6 +192,16 @@ export function sweepExpired(poolState, now = Date.now()) {
         cleared++;
       }
     }
+    // Prune stale entries for keys no longer in active configuration
+    if (activeSet) {
+      for (const map of [st.failedUntil, st.failCounts, st.authFailCounts, st.brokenUntil, st.costPerKey, st.lastUsedAt, st.usageCounts, st.byModel, st.usageDays, st.quotaWindows, st.rpmWindows, st.probedAt]) {
+        if (map && typeof map.keys === 'function') {
+          for (const k of [...map.keys()]) {
+            if (!activeSet.has(k)) map.delete(k);
+          }
+        }
+      }
+    }
   }
   return cleared;
 }
@@ -248,33 +255,105 @@ export function computeHealthScore(state) {
 /** Extract rate-limit info from an object that may carry response headers. */
 export function extractRateLimit(headers) {
   if (!headers || typeof headers !== 'object') return null;
-  let remaining, limit, reset;
+  let remaining, limit, reset, retryAfter;
   for (const k in headers) {
-    const lk = k.length;
-    if (lk === 21 || lk === 16 || lk === 17) {
-      const lower = k.toLowerCase();
-      if (lower === 'x-ratelimit-remaining') {
-        const v = headers[k];
-        if (v != null) remaining = Number(String(v));
-      } else if (lower === 'x-ratelimit-limit') {
-        const v = headers[k];
-        if (v != null) limit = Number(String(v));
-      } else if (lower === 'x-ratelimit-reset') {
-        const v = headers[k];
-        if (v != null) reset = Number(String(v));
+    const lower = k.toLowerCase();
+    const v = headers[k];
+    if (v == null) continue;
+    if (lower === 'retry-after') {
+      const num = Number(v);
+      if (Number.isFinite(num) && num >= 0) {
+        retryAfter = Math.ceil(num);
+      } else {
+        const parsed = Date.parse(v);
+        if (!Number.isNaN(parsed)) {
+          retryAfter = Math.max(0, Math.ceil((parsed - Date.now()) / 1000));
+        }
+      }
+    } else if (
+      lower === 'x-ratelimit-remaining' ||
+      lower === 'x-ratelimit-remaining-requests' ||
+      lower === 'anthropic-ratelimit-requests-remaining' ||
+      lower === 'openai-ratelimit-remaining-requests' ||
+      lower === 'ratelimit-remaining'
+    ) {
+      const n = Number(String(v));
+      if (Number.isFinite(n)) remaining = n;
+    } else if (
+      (lower === 'x-ratelimit-remaining-tokens' || lower === 'anthropic-ratelimit-tokens-remaining') &&
+      remaining === undefined
+    ) {
+      const n = Number(String(v));
+      if (Number.isFinite(n)) remaining = n;
+    } else if (
+      lower === 'x-ratelimit-limit' ||
+      lower === 'x-ratelimit-limit-requests' ||
+      lower === 'anthropic-ratelimit-requests-limit' ||
+      lower === 'openai-ratelimit-limit-requests' ||
+      lower === 'ratelimit-limit'
+    ) {
+      const n = Number(String(v));
+      if (Number.isFinite(n)) limit = n;
+    } else if (
+      lower === 'x-ratelimit-reset' ||
+      lower === 'x-ratelimit-reset-requests' ||
+      lower === 'anthropic-ratelimit-requests-reset' ||
+      lower === 'ratelimit-reset'
+    ) {
+      const n = Number(String(v));
+      if (Number.isFinite(n)) {
+        reset = n;
+      } else {
+        const parsed = Date.parse(v);
+        if (!Number.isNaN(parsed)) reset = parsed;
       }
     }
   }
-  if (remaining === undefined && limit === undefined) return null;
-  return { remaining, limit, reset };
+  if (remaining === undefined && limit === undefined && retryAfter === undefined && reset === undefined) return null;
+  const out = { remaining, limit, reset };
+  if (retryAfter !== undefined) out.retryAfter = retryAfter;
+  return out;
 }
 
-/** True if remaining is below the given threshold fraction of limit (e.g. 0.1). */
+/** True if remaining is below the given threshold fraction of limit (e.g. 0.1) or retry-after is active. */
 export function isRateLimited(rate, threshold = 0.1) {
   if (!rate) return false;
-  if (rate.remaining === undefined) return false;
-  if (rate.limit && rate.limit > 0) return rate.remaining < rate.limit * threshold;
-  return rate.remaining <= 0;
+  if (typeof rate.retryAfter === 'number' && rate.retryAfter > 0) return true;
+  if (rate.remaining !== undefined && rate.remaining <= 1) return true;
+  if (rate.limit && rate.limit > 0 && rate.remaining !== undefined) {
+    return rate.remaining < rate.limit * threshold;
+  }
+  if (rate.remaining !== undefined) return rate.remaining <= 0;
+  return false;
+}
+
+/**
+ * Sort or prioritize an attempt list of refs according to routing strategy.
+ * Strategies:
+ * - 'round-robin': preserves order
+ * - 'least-loaded': keys with lowest active concurrency count first
+ * - 'lowest-latency': keys with lowest p95 (or avg) latency first; unsampled keys neutral
+ */
+export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
+  if (!Array.isArray(refs) || refs.length <= 1) return (refs ?? []).slice();
+  const list = refs.slice();
+  if (strategy === 'least-loaded' && deps.concurrencyTracker && typeof deps.concurrencyTracker.getActive === 'function') {
+    return list.sort((a, b) => {
+      const ca = deps.concurrencyTracker.getActive(a) ?? 0;
+      const cb = deps.concurrencyTracker.getActive(b) ?? 0;
+      return ca - cb;
+    });
+  }
+  if (strategy === 'lowest-latency' && deps.latencyHistogram && typeof deps.latencyHistogram.snapshot === 'function') {
+    return list.sort((a, b) => {
+      const sa = deps.latencyHistogram.snapshot(a);
+      const sb = deps.latencyHistogram.snapshot(b);
+      const la = (sa && Number.isFinite(sa.p95)) ? sa.p95 : ((sa && Number.isFinite(sa.avg)) ? sa.avg : 500);
+      const lb = (sb && Number.isFinite(sb.p95)) ? sb.p95 : ((sb && Number.isFinite(sb.avg)) ? sb.avg : 500);
+      return la - lb;
+    });
+  }
+  return list;
 }
 
 /**
@@ -386,7 +465,7 @@ export function budgetVerdict(spend, budget) {
 /** Safely reset a provider's circuit breaker to closed state. */
 export function resetCircuitForProvider(circuitBreaker, provider) {
   if (!circuitBreaker || !provider) return false;
-  try {
+  const ok = bestEffort('resetCircuitForProvider', () => {
     if (typeof circuitBreaker.reset === 'function') {
       circuitBreaker.reset(provider);
       return true;
@@ -395,6 +474,7 @@ export function resetCircuitForProvider(circuitBreaker, provider) {
       circuitBreaker.onSuccess(provider);
       return true;
     }
-  } catch (_) {}
-  return false;
+    return false;
+  });
+  return ok === true;
 }

package/lib/rotate.js

@@ -9,6 +9,7 @@ import {
   extractRateLimit,
   isRateLimited,
   formatExhaustionMessage,
+  sortAttemptList,
 } from './pool.js';
 import { pickCascadeFallback } from './cascade.js';
 
@@ -25,6 +26,7 @@ export function createRotate(deps) {
     notifySwitch,
     notifyExhaustion,
     recordLatency,
+    latencyHistogram,
     concurrencyTracker,
     MARKER,
     finishError,
@@ -42,7 +44,11 @@ export function createRotate(deps) {
       const reqStore = { pool, pickedRef: undefined, startMs: now() };
       setRotateStartMs(reqStore.startMs);
       let attemptList = (pool.weightedRefs ?? pool.refs).slice();
-      if (runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
+      const strategy = pool.routingStrategy ?? runtime0.routingStrategy ?? 'round-robin';
+      if (strategy !== 'round-robin') {
+        attemptList = sortAttemptList(attemptList, strategy, { latencyHistogram, concurrencyTracker });
+      }
+      if (strategy !== 'least-loaded' && runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
         // #193: prefer least-loaded key within limit
         const available = attemptList.filter((r) => {
           const fu = pool.state.failedUntil.get(r) ?? 0;
@@ -115,122 +121,136 @@ export function createRotate(deps) {
         }
 
         const _pickedRef = reqStore.pickedRef ?? pool.state.lastUsed;
-        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.acquire(_pickedRef);
+        const acquired = (_pickedRef && runtime0.concurrencyLimit > 0) ? concurrencyTracker.acquire(_pickedRef) : false;
         try {
-          for await (const chunk of inner) {
-            // Only actual content deltas lock the stream (no more rotation).
-            // Structural/metadata chunks (block-start/end, usage) do not.
-            if (chunk && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta')) {
-              yielded = true;
-              yield chunk;
-              continue;
-            }
-            if (chunk && chunk.type === 'finish') {
-              const kind = chunk.reason?.kind;
-              const failure = chunk.reason?.failure;
-              const code = failure?.code;
-              const message = failure?.message ?? '';
-              const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-              const switchable = !yielded && kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes);
-              const activeRef = reqStore.pickedRef ?? pool.state.lastUsed;
-              if (switchable) {
-                penalizeRef(activeRef, code ?? 'UNKNOWN', message);
-                pool.state.switches = (pool.state.switches ?? 0) + 1;
-                pool.state.lastReason = String(code ?? 'UNKNOWN');
-                pool.state.lastSwitchAt = now();
-                lastFailure = chunk;
-                console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) - next key`);
-                // #216: per-switch webhook (opt-in switchNotify), deduped per provider
-                if (switchNotify && activeRef) {
-                  notifySwitch(runtime0, pool, {
-                    provider: options.provider,
-                    from: activeRef,
-                    code: String(code ?? 'UNKNOWN'),
-                    at: pool.state.lastSwitchAt,
-                  });
-                }
-                switching = true;
-                break;
+          try {
+            for await (const chunk of inner) {
+              // Only actual content deltas lock the stream (no more rotation).
+              // Structural/metadata chunks (block-start/end, usage) do not.
+              if (chunk && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta')) {
+                yielded = true;
+                yield chunk;
+                continue;
               }
-              // cost tracking if provider returns usage.cost
-              const todayIso = activeRef ? new Date().toISOString().slice(0, 10) : undefined;
-              if (chunk.usage?.cost != null && activeRef) {
-                const c = Number(chunk.usage.cost);
-                if (!isNaN(c)) {
-                  if (!pool.state.costPerKey) pool.state.costPerKey = new Map();
-                  pool.state.costPerKey.set(activeRef, (pool.state.costPerKey.get(activeRef) ?? 0) + c);
-                  // #208: cost per day per key (mirrors usageDays) for budget checks
-                  if (!pool.state.costDays) pool.state.costDays = new Map();
-                  const cMap = pool.state.costDays.get(activeRef) || new Map();
-                  cMap.set(todayIso, (cMap.get(todayIso) ?? 0) + c);
-                  pool.state.costDays.set(activeRef, cMap);
+              if (chunk && chunk.type === 'finish') {
+                const kind = chunk.reason?.kind;
+                const failure = chunk.reason?.failure;
+                const code = failure?.code;
+                const message = failure?.message ?? '';
+                const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
+                const switchable = !yielded && kind === 'error' && isSwitchableError(failure, effectiveSwitchCodes);
+                const activeRef = reqStore.pickedRef ?? pool.state.lastUsed;
+                if (switchable) {
+                  penalizeRef(activeRef, code ?? 'UNKNOWN', message);
+                  pool.state.switches = (pool.state.switches ?? 0) + 1;
+                  pool.state.lastReason = String(code ?? 'UNKNOWN');
+                  pool.state.lastSwitchAt = now();
+                  lastFailure = chunk;
+                  console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} failed (${String(code)} ${String(message).slice(0, 100)}) - next key`);
+                  // #216: per-switch webhook (opt-in switchNotify), deduped per provider
+                  if (switchNotify && activeRef) {
+                    notifySwitch(runtime0, pool, {
+                      provider: options.provider,
+                      from: activeRef,
+                      code: String(code ?? 'UNKNOWN'),
+                      at: pool.state.lastSwitchAt,
+                    });
+                  }
+                  switching = true;
+                  break;
                 }
-              }
-              // Usage by day (#119)
-              if (activeRef) {
-                if (!pool.state.usageDays) pool.state.usageDays = new Map();
-                const dayMap = pool.state.usageDays.get(activeRef) || new Map();
-                dayMap.set(todayIso, (dayMap.get(todayIso) ?? 0) + 1);
-                pool.state.usageDays.set(activeRef, dayMap);
-              }
-              // Per-model request detail (#121)
-              if (activeRef && options.model) {
-                if (!pool.state.byModel) pool.state.byModel = new Map();
-                let byRef = pool.state.byModel.get(activeRef);
-                if (!byRef) { byRef = new Map(); pool.state.byModel.set(activeRef, byRef); }
-                byRef.set(options.model, (byRef.get(options.model) ?? 0) + 1);
-              }
-              // Proactive rate-limit (#115): if response headers say this key is near
-              // its quota, cool it down so the NEXT request starts on a different key.
-              // We do NOT re-run this (already successful) request — that would double-send.
-              const rate = extractRateLimit(chunk?.metadata?.headers ?? chunk?.headers);
-              if (rate && activeRef) {
-                if (isRateLimited(rate, rateLimitThreshold ?? 0.1)) {
-                  const cool = rate.reset && rate.reset > now() ? (rate.reset - now()) : pool.cooldownMs;
-                  recordFailure(pool, activeRef, now(), cool, pool.maxCooldownMs);
-                  pushEvent(pool, activeRef, 'RATE_LIMIT', cool);
-                  console.warn(`[dsh-key-rotation] ${options.provider}: key ${activeRef} near quota (remaining ${String(rate.remaining)}/${String(rate.limit)}) — next request will rotate`);
+                // cost tracking if provider returns usage.cost
+                const todayIso = activeRef ? new Date().toISOString().slice(0, 10) : undefined;
+                if (chunk.usage?.cost != null && activeRef) {
+                  const c = Number(chunk.usage.cost);
+                  if (!isNaN(c)) {
+                    if (!pool.state.costPerKey) pool.state.costPerKey = new Map();
+                    pool.state.costPerKey.set(activeRef, (pool.state.costPerKey.get(activeRef) ?? 0) + c);
+                    // #208: cost per day per key (mirrors usageDays) for budget checks
+                    if (!pool.state.costDays) pool.state.costDays = new Map();
+                    const cMap = pool.state.costDays.get(activeRef) || new Map();
+                    cMap.set(todayIso, (cMap.get(todayIso) ?? 0) + c);
+                    pool.state.costDays.set(activeRef, cMap);
+                  }
                 }
-              }
-              // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
-              if (rate && activeRef && Number.isFinite(rate.remaining) && quotaStore) {
-                quotaStore.set(activeRef, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: now() });
+                // Usage by day (#119)
+                if (activeRef) {
+                  if (!pool.state.usageDays) pool.state.usageDays = new Map();
+                  const dayMap = pool.state.usageDays.get(activeRef) || new Map();
+                  dayMap.set(todayIso, (dayMap.get(todayIso) ?? 0) + 1);
+                  pool.state.usageDays.set(activeRef, dayMap);
+                }
+                // Per-model request detail (#121)
+                if (activeRef && options.model) {
+                  if (!pool.state.byModel) pool.state.byModel = new Map();
+                  let byRef = pool.state.byModel.get(activeRef);
+                  if (!byRef) { byRef = new Map(); pool.state.byModel.set(activeRef, byRef); }
+                  byRef.set(options.model, (byRef.get(options.model) ?? 0) + 1);
+                }
+                // Proactive rate-limit (#115, #303): if response headers say this key is near
+                // its quota or has retry-after, cool it down so the NEXT request starts on a different key.
+                // We do NOT re-run this (already successful) request — that would double-send.
+                const guardEnabled = pool.proactiveRateLimitGuard ?? runtime0.proactiveRateLimitGuard ?? true;
+                const rate = extractRateLimit(chunk?.metadata?.headers ?? chunk?.headers);
+                if (guardEnabled && rate && activeRef) {
+                  if (isRateLimited(rate, rateLimitThreshold ?? 0.1)) {
+                    let cool;
+                    if (typeof rate.retryAfter === 'number' && rate.retryAfter > 0) {
+                      cool = rate.retryAfter * 1000;
+                    } else if (rate.reset && rate.reset > now()) {
+                      cool = rate.reset - now();
+                    } else {
+                      cool = pool.cooldownMs ?? cooldownMs;
+                    }
+                    const maxCool = pool.maxCooldownMs ?? maxCooldownMs;
+                    const effCool = Math.min(cool, maxCool ?? cool);
+                    recordFailure(pool, activeRef, now(), effCool, maxCool);
+                    pushEvent(pool, activeRef, 'RATE_LIMIT', effCool);
+                    console.warn(`[dsh-key-rotation] ${options.provider}: key ${activeRef} proactive pause (remaining ${String(rate.remaining ?? '?')}/${String(rate.limit ?? '?')}, cool ${Math.round(effCool / 1000)}s) — next request will rotate`);
+                  }
+                }
+                // #7: persist quota snapshot regardless of threshold (so dashboard widget can show it).
+                if (rate && activeRef && Number.isFinite(rate.remaining) && quotaStore) {
+                  quotaStore.set(activeRef, { remaining: rate.remaining, limit: rate.limit, reset: rate.reset, at: now() });
+                }
+                yield chunk;
+                recordLatency(pool, reqStore);
+                if (circuitBreaker) circuitBreaker.onSuccess(options.provider);
+                return;
               }
               yield chunk;
-              recordLatency(pool, reqStore);
-              if (circuitBreaker) circuitBreaker.onSuccess(options.provider);
-              return;
             }
-            yield chunk;
-          }
-        } catch (e) {
-          if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
-          const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
-          const activeRef = _pickedRef ?? reqStore.pickedRef ?? pool.state.lastUsed;
-          if (!yielded && isSwitchableError(e, effectiveSwitchCodes)) {
-            penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e));
-            pool.state.switches = (pool.state.switches ?? 0) + 1;
-            pool.state.lastReason = String(e?.code ?? 'TRANSPORT');
-            pool.state.lastSwitchAt = now();
-            lastFailure = finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
-            console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} stream threw ${String(e?.code ?? e?.message ?? e)} - failover to next key`);
-            if (switchNotify && activeRef) {
-              notifySwitch(runtime0, pool, {
-                provider: options.provider,
-                from: activeRef,
-                code: String(e?.code ?? 'TRANSPORT'),
-                at: pool.state.lastSwitchAt,
-              });
+          } catch (e) {
+            const effectiveSwitchCodes = pool.switchCodes ?? switchCodes;
+            const activeRef = _pickedRef ?? reqStore.pickedRef ?? pool.state.lastUsed;
+            if (!yielded && isSwitchableError(e, effectiveSwitchCodes)) {
+              penalizeRef(activeRef, e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+              pool.state.switches = (pool.state.switches ?? 0) + 1;
+              pool.state.lastReason = String(e?.code ?? 'TRANSPORT');
+              pool.state.lastSwitchAt = now();
+              lastFailure = finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+              console.warn(`[dsh-key-rotation] ${options.provider}: key ${String(activeRef ?? '?')} stream threw ${String(e?.code ?? e?.message ?? e)} - failover to next key`);
+              if (switchNotify && activeRef) {
+                notifySwitch(runtime0, pool, {
+                  provider: options.provider,
+                  from: activeRef,
+                  code: String(e?.code ?? 'TRANSPORT'),
+                  at: pool.state.lastSwitchAt,
+                });
+              }
+              continue; // Failover to next key!
             }
-            continue; // Failover to next key!
+            yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
+            return;
           }
-          yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
-          return;
-        }
 
-        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
-        if (switching) continue; // try the next key
-        return; // clean end — served
+          if (switching) continue; // try the next key
+          return; // clean end — served
+        } finally {
+          if (acquired && _pickedRef) {
+            concurrencyTracker.release(_pickedRef);
+          }
+        }
       }
 
       // pool exhausted — all keys cooling or missing

package/lib/routes-ops.js

@@ -24,6 +24,7 @@ import { findSecrets, looksLikeApiSecret } from './keycheck.js';
 import { nextQuotaReset } from './quota-window.js';
 import { classifyFailure } from './error-taxonomy.js';
 import { sanitizeSnapshot } from './sanitize-snapshot.js';
+import { bestEffort } from './best-effort.js';
 
 const STATUS_PATH = '/dsh-key-rotation/status';
 const SNAPSHOT_PATH = '/dsh-key-rotation/snapshot';
@@ -163,6 +164,8 @@ export function registerOpsRoutes(ctx, deps) {
           budgetDaily: (providerBudgets?.get ? providerBudgets.get(pool.base) : providerBudgets?.[pool.base])?.costBudgetDaily ?? 0,
           budgetWeekly: (providerBudgets?.get ? providerBudgets.get(pool.base) : providerBudgets?.[pool.base])?.costBudgetWeekly ?? 0,
           pauseOnBudget: (providerBudgets?.get ? providerBudgets.get(pool.base) : providerBudgets?.[pool.base])?.pauseOnBudget ?? false,
+          routingStrategy: pool.routingStrategy ?? runtime.routingStrategy ?? 'round-robin',
+          proactiveRateLimitGuard: pool.proactiveRateLimitGuard ?? runtime.proactiveRateLimitGuard ?? true,
         });
         } catch (e) {
           console.warn(`[dsh-key-rotation] status: pool ${pool.base} failed: ${String(e?.message ?? e)} ${e?.stack ?? ''}`);
@@ -314,7 +317,7 @@ export function registerOpsRoutes(ctx, deps) {
             st.brokenUntil?.delete(ref);
             if (st.lastUsed === ref) st.lastUsed = undefined;
           }
-          try { lastTestCache?.delete?.(ref); } catch (_) {}
+          bestEffort('lastTestCache.delete', () => { lastTestCache?.delete?.(ref); }, ctx.logger);
           json(res, 200, { ok: true, ref });
           return;
         }
@@ -480,7 +483,7 @@ export function registerOpsRoutes(ctx, deps) {
         const valid = present ? Boolean(effectiveValue && typeof effectiveValue === 'string' && effectiveValue.length > 0) : Boolean(testValue);
         const tail = valid ? keyTail(effectiveValue) : '';
         let source = null;
-        try { const d = await base?.describe?.(ref); source = d?.source ?? null; } catch {}
+        await bestEffort('credentials.describe', async () => { const d = await base?.describe?.(ref); source = d?.source ?? null; }, ctx.logger);
         if (!present && !testValue) { json(res, 200, { ok: false, ref, code: 'no-credential', message: 'no such credential' }); return; }
         if (!present && testValue) { source = 'pre-save'; }
         else if (!present) {

package/lib/sandbox.js

@@ -83,7 +83,15 @@ export class SandboxRunner {
         res = await doFetch();
       } catch (e) {
         if (e && e.name === 'AbortError') return { ok: false, code: 'timeout', latencyMs: Date.now() - started };
-        return { ok: false, code: 'network', latencyMs: Date.now() - started };
+        // ponytail: 1 retry on transient socket network errors before marking broken
+        await new Promise((r) => setTimeout(r, PROBE_RETRY_DELAY_MS));
+        if (ctrl.signal.aborted) return { ok: false, code: 'timeout', latencyMs: Date.now() - started };
+        try {
+          res = await doFetch();
+        } catch (e2) {
+          if (e2 && e2.name === 'AbortError') return { ok: false, code: 'timeout', latencyMs: Date.now() - started };
+          return { ok: false, code: 'network', latencyMs: Date.now() - started };
+        }
       }
       // ponytail: 1 retry on 5xx — naive; classifier refines if needed
       if (res.status >= 500 && res.status < 600) {

package/lib/webhook.js

@@ -1,3 +1,4 @@
+import { bestEffort } from './best-effort.js';
 // lib/webhook.js — webhook sender with throttle, interactive actions + alert digest debouncer (#199, #10).
 export const WEBHOOK_TIMEOUT_MS = 5000;
 export const WEBHOOK_MIN_INTERVAL_MS = 1000;
@@ -187,7 +188,7 @@ export class AlertDebouncer {
 
     const res = await this._sender.send(url, payload);
     for (const cb of entry.callbacks) {
-      try { cb(res); } catch (_) {}
+      bestEffort('webhook.callback', () => { cb(res); }, this._logger);
     }
     return res;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.8",
+  "version": "0.8.11",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
@@ -41,7 +41,10 @@
     },
     "client": {
       "platform": "web",
-      "inject": []
+      "inject": [
+        "@deepseek-ai/dsh-client-locale",
+        "@deepseek-ai/dsh-client-ui-settings"
+      ]
     }
   },
   "license": "MIT",