@goodandready/dsh-key-rotation 0.8.42 → 0.8.44

package/CHANGELOG.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## 0.8.44 - 2026-10-01
+
+### Fixed
+- **Core & Persistence Integrity (#407, #408, #409, #410, #422, #427)**:
+  - Eliminated concurrency permit leaks in lazy streams by avoiding premature permit reservation in rotate dispatch (#407).
+  - Implemented physical credential revocation fan-out across all pools in runtime on 401 AUTH failures (#408).
+  - Supported nested `Map<ref, Map<day, number>>` in state persistence for `costDays` and `usageDays`, preserving daily financial spend and preventing provider budget bypasses across restarts (#409, #422).
+  - Preserved exponential backoff failure counts across cooldown expiration in `sweepExpired` (#410).
+  - Blocked quarantined credentials in candidate selection (`resolver.js`) and ensured auto-unbreak probes skip configured paused keys (#427).
+- **Transport, Circuit Breakers & Route Reliability (#411, #413, #415, #418, #428)**:
+  - Ensured safe dispatcher single-address lookup returns scalar `(err, address, family)` rather than array, conforming to standard Node.js `dns.lookup` contract (#411).
+  - Added non-mutating `isAvailable` inspection to `CircuitBreaker`, preventing cascade candidate checks from consuming trial half-open probes prior to dispatch (#413).
+  - Enforced fail-closed decryption validation in `/dsh-key-rotation/test`, preventing encrypted ciphertext from reaching upstream LLM endpoints when secrets cannot be decrypted (#415).
+  - Model sub-pools now accurately reflect their base provider circuit breaker state in `/status`, and `/health` correctly transitions to `degraded` when any circuit is open (#418).
+  - Streamlined `NotifyQueue` dispatch into debouncer pipeline with accurate delivery outcome tracking and backoff handling (#428).
+- **UI & Client Reliability (#419, #420, #421)**:
+  - Stripped master action token from Telegram interactive callbacks, bounded UTF-8 byte length to <= 64 bytes for unicode, and preserved ASCII action routing (#419).
+  - Fixed `keyStatus` to recognize paused status from server DTOs and reset optimistic reset flags upon active cooldown updates (#420).
+  - Memoized `UpdaterSection` load callback with ref to prevent duplicate background network checks on unrelated parent settings updates (#421).
+- **Packaging, CI & Documentation (#424, #425)**:
+  - Synchronized `package-lock.json` with `semver@7.8.5` and `undici@6.29.0` for clean `npm ci --ignore-scripts` reproducibility (#424).
+  - Updated `docs/deployment/github-npm-provenance.md` with mandatory pre-release MiniPC test-server acceptance and explicit user confirmation requirements (#425).
+
+## 0.8.43 - 2026-10-01
+
+### Fixed
+- **Sanitized GitHub mirror and provenance workflow documentation (#425)**:
+  - Updated `docs/deployment/github-npm-provenance.md` to accurately document the Gitea-first development workflow and sanitized GitHub mirror export (`./scripts/publish-github.sh --push <tag>`), eliminating contradictions regarding direct GitHub PRs and external CI triggers.
+- **LatencyHistogram configuration from latencyWindow setting (#426)**:
+  - Implemented dynamic `reconfigure({ window })` on `LatencyHistogram` (`lib/histogram.js`) and wired `cfg.latencyWindow` from configuration into `buildRuntime()` (`lib/index.js`), ensuring the configured latency sampling window is respected at runtime.
+- **Auto-unbreak quarantine separation and safety checks (#427)**:
+  - Updated `lib/rotate.js` to place keys into a recoverable 30-minute quarantine (`brokenUntil`) after 3 consecutive non-auth failures while protecting rate-limited/quota-exhausted keys from quarantine pollution.
+  - Updated `lib/heal.js` to strictly skip permanently revoked (`revokedRefs`) and paused (`pausedRefs`) credentials during auto-unbreak background probing, preventing probing or accidental reactivation of revoked credentials.
+- **Webhook Alert Digest pipeline integration (#428)**:
+  - Wired `AlertDebouncer` into the production notification pipeline in `lib/index.js`, batching rapid alert bursts into consolidated digest payloads before webhook dispatch as documented.
+
 ## 0.8.42 - 2026-10-01
 
 ### Fixed

package/lib/cascade.js

@@ -15,8 +15,12 @@ export function pickCascadeFallback(provider, cfg, pools, currentModel, modelPoo
     if (!fb || !fb.provider || fb.provider === provider) continue;
 
     const breaker = circuitBreaker || cfg?.circuitBreaker || null;
-    if (breaker && typeof breaker.canRequest === 'function' && !breaker.canRequest(fb.provider)) {
-      continue;
+    if (breaker) {
+      if (typeof breaker.isAvailable === 'function') {
+        if (!breaker.isAvailable(fb.provider)) continue;
+      } else if (typeof breaker.canRequest === 'function') {
+        if (!breaker.canRequest(fb.provider)) continue;
+      }
     }
 
     const mappedModel = (fb.modelMapping && currentModel && fb.modelMapping[currentModel]) || fb.model || currentModel || null;

package/lib/circuit-breaker.js

@@ -57,6 +57,22 @@ export class CircuitBreaker {
     return true;
   }
 
+  /** @returns {boolean} whether a request may be dispatched to this provider without mutating probe count */
+  isAvailable(provider) {
+    const e = this._entry(provider);
+    const now = this._now();
+    if (e.state === BREAKER_OPEN) {
+      if (now - e.openedAt >= this.openMs) {
+        return true;
+      }
+      return false;
+    }
+    if (e.state === BREAKER_HALF_OPEN) {
+      return e.probes < this.halfOpenProbes;
+    }
+    return true;
+  }
+
   // A permit is settled exactly once. Late outcomes from an older circuit
   // generation (or a removed provider) cannot close/reopen a newer circuit.
   acquire(provider) {

package/lib/client.js

@@ -753,23 +753,25 @@ window.__ModuleLoader__.load({
     }
 
     function UpdaterSection({ t }) {
+      const tRef = React.useRef(t);
+      tRef.current = t;
       const [state, setState] = React.useState({ phase: 'idle', current: '', latest: '', message: '' });
       const load = React.useCallback(async () => {
         setState((s) => ({ ...s, phase: 'loading', message: '' }));
         try {
           const res = await fetch('/api/dsh-key-rotation/update', { headers: { 'x-dsh-plugin-update': '1' } });
           const data = await res.json();
-          if (!res.ok) throw new Error(data?.error?.message || t('updateFailed'));
+          if (!res.ok) throw new Error(data?.error?.message || (tRef.current ? tRef.current('updateFailed') : 'Update check failed'));
           setState({
             phase: data.updateAvailable ? 'available' : 'current',
             current: data.currentVersion || '',
             latest: data.latestVersion || '',
-            message: data.updateAvailable ? t('updateAvailable') : t('updateNone'),
+            message: data.updateAvailable ? (tRef.current ? tRef.current('updateAvailable') : '') : (tRef.current ? tRef.current('updateNone') : ''),
           });
         } catch (e) {
-          setState({ phase: 'error', current: '', latest: '', message: e?.message || t('updateFailed') });
+          setState({ phase: 'error', current: '', latest: '', message: e?.message || (tRef.current ? tRef.current('updateFailed') : 'Update check failed') });
         }
-      }, [t]);
+      }, []);
       React.useEffect(() => { load(); }, [load]);
       const run = async () => {
         setState((s) => ({ ...s, phase: 'running', message: t('updateRunning') }));
@@ -1792,10 +1794,14 @@ window.__ModuleLoader__.load({
         if (isPaused) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyPaused'), ready: false, paused: true };
         const hit = keyInfo(providerId, ref);
         if (!hit) return null;
+        if (hit.paused) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyPaused'), ready: false, paused: true };
         if (hit.expired) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyExpired'), ready: false, expired: true };
         if (hit.broken || hit.revoked || hit.status === 'revoked') return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyRevoked'), ready: false, revoked: true };
         if (!hit.present) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyMissing'), ready: false, missing: true };
-        if (hit.cooldownMsLeft > 0 && !optimisticReset[ref]) {
+        if (hit.cooldownMsLeft > 0) {
+          if (optimisticReset && ref in optimisticReset) {
+            delete optimisticReset[ref];
+          }
           return {
             color: 'var(--dsw-alias-state-warning-primary)',
             text: t('keyCooling').replace('{s}', String(Math.ceil(hit.cooldownMsLeft / 1000))),

package/lib/heal.js

@@ -1,3 +1,4 @@
+import { isKeyRevoked, isKeyPaused } from './pool.js';
 // heal.js — self-healing idle cooldowns.
 // ponytail: pure function, easy to test, no side effects beyond mutation of passed-in state.
 
@@ -54,6 +55,7 @@ export async function autoUnbreakBrokenKeys(pools, probeFn, now = Date.now(), {
     if (!bu || bu.size === 0) continue;
     const brokenRefs = [];
     for (const [ref, until] of bu.entries()) {
+      if (isKeyRevoked(pool, ref) || isKeyPaused(pool, ref)) continue;
       if (Number.isFinite(until) && until > now) {
         brokenRefs.push(ref);
       }

package/lib/histogram.js

@@ -4,11 +4,43 @@
 export const LATENCY_DEFAULT_WINDOW = 200;
 
 export class LatencyHistogram {
-  constructor({ window = LATENCY_DEFAULT_WINDOW } = {}) {
+  constructor(opts = {}) {
+    const windowVal = typeof opts === 'number' ? opts : opts?.window;
+    this.reconfigure({ window: windowVal });
+  }
+
+  get window() {
+    return this._window;
+  }
+
+  reconfigure({ window = LATENCY_DEFAULT_WINDOW } = {}) {
     const w = Number.isFinite(window) && window > 0 ? Math.floor(window) : LATENCY_DEFAULT_WINDOW;
+    if (this._window === w && this._buffers) return;
+
+    if (!this._buffers) {
+      this._window = w;
+      this._buffers = new Map(); // ref -> { buf, head, count }
+      this._lastAt = new Map();  // ref -> epochMs
+      return;
+    }
+
+    // Resize existing ring buffers to new length w
+    for (const [ref, entry] of this._buffers.entries()) {
+      const samples = [];
+      const start = entry.count < entry.buf.length ? 0 : entry.head;
+      for (let i = 0; i < entry.count; i++) {
+        samples.push(entry.buf[(start + i) % entry.buf.length]);
+      }
+      const newSamples = samples.slice(-w);
+      const newBuf = new Float64Array(w);
+      for (let i = 0; i < newSamples.length; i++) {
+        newBuf[i] = newSamples[i];
+      }
+      entry.buf = newBuf;
+      entry.head = newSamples.length % w;
+      entry.count = newSamples.length;
+    }
     this._window = w;
-    this._buffers = new Map(); // ref -> Float64Array of size w, plus index/count
-    this._lastAt = new Map();  // ref -> epochMs of last sample
   }
 
   record(ref, ms) {

package/lib/index.js

@@ -20,7 +20,7 @@ import {
 import { LatencyHistogram } from './histogram.js';
 import { ConcurrencyTracker } from './concurrency.js';
 import { QuotaStore } from './quota.js';
-import { WebhookSender } from './webhook.js';
+import { WebhookSender, AlertDebouncer } from './webhook.js';
 import { compactUsage } from './usage-report.js';
 import { json, handleConfigBridge } from './http-bridge.js';
 import { createRotate } from './rotate.js';
@@ -80,7 +80,8 @@ const concurrencyTracker = new ConcurrencyTracker();
 
 let moduleBreaker = new CircuitBreaker({ threshold: 5, openMs: 30000, halfOpenProbes: 1, now: nowMono });
 const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
-let moduleNotifyQueue = new NotifyQueue({ send: (url, payload) => webhookSender.send(url, payload) });
+const alertDebouncer = new AlertDebouncer({ sender: webhookSender, debounceMs: 2000 });
+let moduleNotifyQueue = new NotifyQueue({ send: (url, payload) => alertDebouncer.enqueue(url, payload) });
 
 let getConfig = () => null;
 function verboseLoggingOn() {
@@ -270,6 +271,9 @@ export function apply(ctx, config = {}) {
       openMs: cfg.circuitBreakerOpenMs ?? 30000,
       halfOpenProbes: cfg.circuitBreakerHalfOpenProbes ?? 1,
     });
+    if (cfg.latencyWindow && latencyHistogram.window !== cfg.latencyWindow) {
+      latencyHistogram.reconfigure({ window: cfg.latencyWindow });
+    }
     const switchCodes = cfg.switchCodes ?? DEFAULT_SWITCH_CODES;
     const cooldownMs = cfg.cooldownMs ?? 60000;
     const maxCooldownMs = cfg.maxCooldownMs;

package/lib/notify-queue.js

@@ -31,35 +31,44 @@ export class NotifyQueue {
   }
 
   _pump() {
-    if (this._busy) return;
-    const job = this._q.shift();
-    if (!job) return;
-    this._busy = true;
-    const delay = this._backoffByUrl.get(job.url) ?? 0;
-    const run = async () => {
-      try {
-        const res = await this._send(job.url, job.payload);
-        if (res && res.sent === false) {
+    while (this._q.length > 0) {
+      const delay = this._backoffByUrl.get(this._q[0].url) ?? 0;
+      if (delay > 0 && this._busy) break;
+      const job = this._q.shift();
+      if (!job) break;
+      this._activeSends = (this._activeSends || 0) + 1;
+      this._busy = true;
+      const run = async () => {
+        try {
+          const res = await this._send(job.url, job.payload);
+          if (res && res.sent === false) {
+            this._failed += 1;
+            this._bumpBackoff(job.url);
+          } else if (res && res.sent === true) {
+            this._sent += 1;
+            this._backoffByUrl.delete(job.url);
+          } else if (res === undefined) {
+            this._sent += 1;
+            this._backoffByUrl.delete(job.url);
+          } else {
+            this._sent += 1;
+            this._backoffByUrl.delete(job.url);
+          }
+        } catch {
           this._failed += 1;
           this._bumpBackoff(job.url);
-        } else {
-          this._sent += 1;
-          this._backoffByUrl.delete(job.url);
+        } finally {
+          this._activeSends = Math.max(0, this._activeSends - 1);
+          this._busy = this._activeSends > 0;
+          if (this._q.length > 0) this._pump();
         }
-      } catch {
-        this._failed += 1;
-        this._bumpBackoff(job.url);
-      } finally {
-        this._busy = false;
-        if (this._q.length > 0) this._pump();
+      };
+      if (delay > 0) {
+        const t = setTimeout(run, delay);
+        if (typeof t.unref === 'function') t.unref();
+      } else {
+        run();
       }
-    };
-    if (delay > 0) {
-      const t = setTimeout(run, delay);
-      if (typeof t.unref === 'function') t.unref();
-    } else {
-      // fire and forget — do not return a promise to callers
-      run();
     }
   }
 

package/lib/ops-status.js

@@ -129,7 +129,11 @@ ctx.effect(() => ctx.webServer.register({
         circuit: (() => {
           const br = circuitBreaker ?? runtime.breaker;
           if (!br) return null;
-          const st = br.state(pool.base);
+          let st = br.state(pool.base);
+          if (st === 'closed' && pool.provider && pool.provider !== pool.base) {
+            const baseSt = br.state(pool.provider);
+            if (baseSt !== 'closed') st = baseSt;
+          }
           return { state: st, threshold: br.threshold, openMs: br.openMs };
         })(),
         switches: pool.state.switches ?? 0,
@@ -217,7 +221,19 @@ ctx.effect(() => ctx.webServer.register({
         modelQuota: quotaAware ? { configured: true, exhaustedKeys: quotaExhausted } : null,
       };
     }
-    json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: typeof quotaStore?.snapshot === 'function' ? quotaStore.snapshot() : null });
+    let circuitOpenAny = false;
+    const brH = circuitBreaker ?? runtimeH.breaker;
+    if (brH) {
+      for (const pool of (runtimeH.pools ?? runtimeH.poolByRef.values())) {
+        const provKey = pool.provider || pool.base;
+        if (brH.state(pool.base) === 'open' || brH.state(provKey) === 'open') {
+          circuitOpenAny = true;
+          break;
+        }
+      }
+    }
+    const isDegraded = exhaustedAny || circuitOpenAny;
+    json(res, 200, { status: isDegraded ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: typeof quotaStore?.snapshot === 'function' ? quotaStore.snapshot() : null });
   },
 }), 'dsh-key-rotation: health');
 

package/lib/ops-test.js

@@ -45,6 +45,13 @@ ctx.effect(() => ctx.webServer.register({
       let hit = await (base?.__dshKeyRotationOriginalResolve ?? base?.resolve)?.call(base, ref);
       let present = Boolean(hit && typeof hit.value === 'string' && hit.value.length > 0);
       const rawValue = testValue || hit?.value;
+      if (typeof rawValue === 'string' && rawValue.startsWith('enc:v1:')) {
+        const decrypted = decryptSecret(rawValue);
+        if (!decrypted || decrypted === rawValue || decrypted.startsWith('enc:v1:')) {
+          json(res, 200, { ok: false, ref, code: 'decryption-failed', message: 'dsh-key-rotation: failed to decrypt credential (missing or invalid DSH_KEY_SECRET)' });
+          return;
+        }
+      }
       const effectiveValue = rawValue ? decryptSecret(rawValue) : rawValue;
       const valid = present ? Boolean(effectiveValue && typeof effectiveValue === 'string' && effectiveValue.length > 0) : Boolean(testValue);
       const tail = valid ? keyTail(effectiveValue) : '';

package/lib/persistence.js

@@ -56,8 +56,22 @@ export class StatePersistence {
         }
         const costDays = {};
         if (st?.costDays) {
-          const src = st.costDays instanceof Map ? Object.fromEntries(st.costDays) : st.costDays;
-          if (typeof src === 'object' && src !== null) Object.assign(costDays, src);
+          const entries = st.costDays instanceof Map ? st.costDays.entries() : Object.entries(st.costDays);
+          for (const [k, val] of entries) {
+            if (!val) continue;
+            if (val instanceof Map || (typeof val === 'object' && !Array.isArray(val))) {
+              const innerObj = {};
+              const innerEntries = val instanceof Map ? val.entries() : Object.entries(val);
+              for (const [day, c] of innerEntries) {
+                const num = Number(c);
+                if (Number.isFinite(num) && num >= 0) innerObj[day] = num;
+              }
+              if (Object.keys(innerObj).length > 0) costDays[k] = innerObj;
+            } else if (Number.isFinite(Number(val))) {
+              const num = Number(val);
+              if (num >= 0) costDays[k] = num;
+            }
+          }
         }
         const costPerKey = {};
         if (st?.costPerKey) {
@@ -66,8 +80,22 @@ export class StatePersistence {
         }
         const usageDays = {};
         if (st?.usageDays) {
-          const src = st.usageDays instanceof Map ? Object.fromEntries(st.usageDays) : st.usageDays;
-          if (typeof src === 'object' && src !== null) Object.assign(usageDays, src);
+          const entries = st.usageDays instanceof Map ? st.usageDays.entries() : Object.entries(st.usageDays);
+          for (const [k, val] of entries) {
+            if (!val) continue;
+            if (val instanceof Map || (typeof val === 'object' && !Array.isArray(val))) {
+              const innerObj = {};
+              const innerEntries = val instanceof Map ? val.entries() : Object.entries(val);
+              for (const [day, c] of innerEntries) {
+                const num = Number(c);
+                if (Number.isFinite(num) && num >= 0) innerObj[day] = Math.floor(num);
+              }
+              if (Object.keys(innerObj).length > 0) usageDays[k] = innerObj;
+            } else if (Number.isFinite(Number(val))) {
+              const num = Number(val);
+              if (num >= 0) usageDays[k] = Math.floor(num);
+            }
+          }
         }
         const usageCounts = {};
         if (st?.usageCounts instanceof Map) {
@@ -182,10 +210,22 @@ export class StatePersistence {
             state.tokenUsage.set(ref, { used: Math.floor(used), resetAt: Math.floor(resetAt) });
           }
         }
-        if (saved.costDays && typeof saved.costDays === 'object') {
+        if (saved.costDays && typeof saved.costDays === 'object' && !Array.isArray(saved.costDays)) {
           if (!state.costDays) state.costDays = new Map();
-          for (const [day, cost] of Object.entries(saved.costDays)) {
-            if (Number.isFinite(cost)) state.costDays.set(day, cost);
+          for (const [k, val] of Object.entries(saved.costDays)) {
+            if (val && typeof val === 'object' && !Array.isArray(val)) {
+              let innerMap = state.costDays.get(k);
+              if (!(innerMap instanceof Map)) {
+                innerMap = new Map();
+                state.costDays.set(k, innerMap);
+              }
+              for (const [day, cost] of Object.entries(val)) {
+                const num = Number(cost);
+                if (Number.isFinite(num) && num >= 0) innerMap.set(day, num);
+              }
+            } else if (Number.isFinite(Number(val))) {
+              state.costDays.set(k, Number(val));
+            }
           }
         }
         if (saved.costPerKey && typeof saved.costPerKey === 'object') {
@@ -194,10 +234,22 @@ export class StatePersistence {
             if (Number.isFinite(c)) state.costPerKey.set(k, c);
           }
         }
-        if (saved.usageDays && typeof saved.usageDays === 'object') {
+        if (saved.usageDays && typeof saved.usageDays === 'object' && !Array.isArray(saved.usageDays)) {
           if (!state.usageDays) state.usageDays = new Map();
-          for (const [d, u] of Object.entries(saved.usageDays)) {
-            if (Number.isFinite(u)) state.usageDays.set(d, u);
+          for (const [k, val] of Object.entries(saved.usageDays)) {
+            if (val && typeof val === 'object' && !Array.isArray(val)) {
+              let innerMap = state.usageDays.get(k);
+              if (!(innerMap instanceof Map)) {
+                innerMap = new Map();
+                state.usageDays.set(k, innerMap);
+              }
+              for (const [day, count] of Object.entries(val)) {
+                const num = Number(count);
+                if (Number.isFinite(num) && num >= 0) innerMap.set(day, Math.floor(num));
+              }
+            } else if (Number.isFinite(Number(val))) {
+              state.usageDays.set(k, Math.floor(Number(val)));
+            }
           }
         }
         if (saved.usageCounts && typeof saved.usageCounts === 'object') {

package/lib/pool.js

@@ -210,7 +210,6 @@ export function sweepExpired(poolState, now = Date.now(), activeRefs = null, quo
     for (const [ref, until] of [...st.failedUntil.entries()]) {
       if (until <= now) {
         st.failedUntil.delete(ref);
-        st.failCounts?.delete(ref);
         st.quotaWindows?.delete(ref);
         cleared++;
       }
@@ -564,3 +563,12 @@ export function isKeyRevoked(pool, ref) {
   if (pool.basePool) return isKeyRevoked(pool.basePool, ref);
   return false;
 }
+
+/** Check if a key reference is in active recoverable or permanent quarantine (#427). */
+export function isKeyBroken(pool, ref, now = Date.now()) {
+  if (!pool || !ref) return false;
+  const bu = pool.state?.brokenUntil?.get(ref);
+  if (bu !== undefined && bu > now) return true;
+  if (pool.basePool) return isKeyBroken(pool.basePool, ref, now);
+  return false;
+}

package/lib/resolver.js

@@ -9,7 +9,7 @@
 // delegates to `original(ref)`.
 
 import { bucketAllow, bucketRetryMs, tpmAllow, tpmRetryMs } from './bucket.js';
-import { envValue, sortAttemptList, isKeyPaused, isKeyRevoked, costForDay, costForWeek, budgetVerdict } from './pool.js';
+import { envValue, sortAttemptList, isKeyPaused, isKeyRevoked, isKeyBroken, costForDay, costForWeek, budgetVerdict } from './pool.js';
 import { decryptSecret } from './crypto-storage.js';
 import { resolveRefPool, isQuotaManagedRequest } from './pool-index.js';
 import {
@@ -132,6 +132,7 @@ export function createResolver({
       const candidate = list[index];
       if (isKeyRevoked(pool, candidate)) continue;
       if (isKeyPaused(pool, candidate)) continue;
+      if (isKeyBroken(pool, candidate, at)) continue;
       const until = pool.state.failedUntil.get(candidate);
       if (until !== undefined && until > at) continue;
       const candidateExp = pool.expiresAt?.[candidate] ?? pool.basePool?.expiresAt?.[candidate];

package/lib/rotate.js

@@ -11,6 +11,7 @@ import {
   isRateLimited,
   isKeyPaused,
   isKeyRevoked,
+  isKeyBroken,
   formatExhaustionMessage,
   sortAttemptList,
 } from './pool.js';
@@ -26,6 +27,8 @@ import {
   formatModelQuotaExhaustion,
 } from './model-quota.js';
 
+import { LOCAL_POOL_EXHAUSTED_CODE } from './resolver.js';
+
 export const LOCAL_QUOTA_CODE = 'LOCAL_MODEL_QUOTA_EXHAUSTED';
 
 export async function* preserveDispatchContext(iterable, storage, store) {
@@ -85,7 +88,7 @@ export function createRotate(deps) {
       const wallNow = Date.now();
       const quotaManaged = hasModelQuotaConfig(pool);
       let attemptList = (pool.weightedRefs ?? pool.refs).slice();
-      attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r));
+      attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r) && !isKeyBroken(pool, r, wallNow));
       if (quotaManaged) {
         attemptList = attemptList.filter((ref) => isModelQuotaAvailable(pool, ref, wallNow));
         if (attemptList.length === 0) {
@@ -184,8 +187,43 @@ export function createRotate(deps) {
           pool.state.failedUntil.set(targetRef, Infinity);
           pushEvent(pool, targetRef, 'KEY_REVOKED', 0);
           logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently REVOKED on auth failure`);
+          // Fan-out revocation across all pools sharing this physical ref (#408):
+          const rAll = buildRuntime();
+          const candidatePools = [];
+          if (Array.isArray(rAll.pools)) candidatePools.push(...rAll.pools);
+          if (rAll.index?.allPools) candidatePools.push(...rAll.index.allPools.values());
+          if (rAll.providerToPool) candidatePools.push(...rAll.providerToPool.values());
+          if (rAll.modelPoolByProvider) {
+            for (const map of rAll.modelPoolByProvider.values()) {
+              if (map) candidatePools.push(...map.values());
+            }
+          }
+          const seenPools = new Set();
+          for (const other of candidatePools) {
+            if (!other || seenPools.has(other)) continue;
+            seenPools.add(other);
+            if (other === pool) continue;
+            const hasRef = (Array.isArray(other.refs) && other.refs.includes(targetRef)) ||
+              (other.weightsMap instanceof Map && other.weightsMap.has(targetRef));
+            if (hasRef && other.state) {
+              if (!other.state.revokedRefs) other.state.revokedRefs = new Set();
+              other.state.revokedRefs.add(targetRef);
+              if (other.state.failedUntil) other.state.failedUntil.set(targetRef, Infinity);
+              pushEvent(other, targetRef, 'KEY_REVOKED', 0);
+            }
+          }
         } else {
           pool.state.authFailCounts.delete(targetRef);
+          if (!isQuota) {
+            const _fc = (pool.state.failCounts.get(targetRef) ?? 0) + 1;
+            pool.state.failCounts.set(targetRef, _fc);
+            if (_fc >= 3 && !pool.state.brokenUntil.has(targetRef)) {
+              const quarantineMs = 1800000;
+              pool.state.brokenUntil.set(targetRef, Date.now() + quarantineMs);
+              pushEvent(pool, targetRef, 'KEY_QUARANTINED', quarantineMs);
+              logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} put in recoverable quarantine after ${_fc} failures`);
+            }
+          }
         }
       };
 
@@ -248,13 +286,16 @@ export function createRotate(deps) {
           }
 
           const _pickedRef = reqStore.pickedRef ?? pool.state.lastUsed;
-          if (_pickedRef && !reqStore.concurrencyRef && typeof concurrencyTracker?.acquire === 'function') {
-            const ok = concurrencyTracker.acquire(_pickedRef, now(), pool.concurrencyLimit ?? runtime0.concurrencyLimit ?? 0);
-            if (ok) reqStore.concurrencyRef = _pickedRef;
-          }
           try {
             try {
               for await (const chunk of preserveDispatchContext(inner, dispatchStorage, reqStore)) {
+                if (!reqStore.concurrencyRef && typeof concurrencyTracker?.acquire === 'function') {
+                  const fallbackRef = reqStore.pickedRef ?? _pickedRef;
+                  if (fallbackRef) {
+                    const ok = concurrencyTracker.acquire(fallbackRef, now(), pool.concurrencyLimit ?? runtime0.concurrencyLimit ?? 0);
+                    if (ok) reqStore.concurrencyRef = fallbackRef;
+                  }
+                }
                 // Only actual content deltas lock the stream (no more rotation).
                 // Structural/metadata chunks (block-start/end, usage) do not.
                 if (chunk && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta')) {
@@ -430,6 +471,7 @@ export function createRotate(deps) {
             concurrencyTracker?.release(reqStore.concurrencyRef);
             reqStore.concurrencyRef = null;
           }
+          reqStore.pickedRef = undefined;
           // Also runs on consumer return(), abort, empty EOF and local errors.
           activePermit?.release();
         }

package/lib/safe-fetch.js

@@ -109,7 +109,7 @@ export function createSafeDispatcher({ lookupImpl = nodeDns.lookup } = {}) {
     connect: {
       lookup: (hostname, options, callback) => {
         const cb = typeof options === 'function' ? options : callback;
-        const opts = typeof options === 'function' ? {} : options;
+        const opts = typeof options === 'function' ? {} : (typeof options === 'number' ? { family: options } : (options || {}));
         lookupImpl(hostname, { ...opts, all: true }, (err, addresses) => {
           if (err) return cb(err);
           const list = Array.isArray(addresses) ? addresses : [addresses];
@@ -121,10 +121,10 @@ export function createSafeDispatcher({ lookupImpl = nodeDns.lookup } = {}) {
               return cb(ssrfErr);
             }
           }
-          if (typeof options === 'function') {
+          if (!opts.all) {
             const first = list[0];
-            const addr = typeof first === 'string' ? first : first.address;
-            const fam = typeof first === 'object' && first.family ? first.family : (net.isIP(addr) || 4);
+            const addr = typeof first === 'string' ? first : first?.address;
+            const fam = typeof first === 'object' && first?.family ? first.family : (net.isIP(addr) || 4);
             return cb(null, addr, fam);
           }
           return cb(null, addresses);

package/lib/webhook.js

@@ -32,12 +32,17 @@ export function formatInteractive(url, payload, actionToken) {
       parse_mode: 'Markdown',
       reply_markup: {
         inline_keyboard: [actions.map((a) => {
-          let cbData = actionToken ? JSON.stringify({ id: a.id, token: actionToken }) : JSON.stringify({ id: a.id });
+          let cbData = JSON.stringify({ id: a.id });
           if (cbData.length > 64) {
-            cbData = JSON.stringify({ id: a.id });
+            cbData = String(a.id ?? '');
           }
-          if (cbData.length > 64) {
-            cbData = String(a.id).slice(0, 64);
+          if (Buffer.byteLength(cbData, 'utf8') > 64 && /[^\x00-\x7F]/.test(cbData)) {
+            const buf = Buffer.from(cbData, 'utf8');
+            let res = buf.subarray(0, 64).toString('utf8');
+            while (Buffer.byteLength(res, 'utf8') > 64) {
+              res = res.slice(0, -1);
+            }
+            cbData = res;
           }
           return {
             text: a.label,
@@ -189,7 +194,7 @@ export class AlertDebouncer {
   }
 
   enqueue(url, event, onFlushed = null) {
-    if (!url) return;
+    if (!url) return Promise.resolve({ sent: false });
     let entry = this._pending.get(url);
     if (!entry) {
       entry = { events: [], timer: null, callbacks: [] };
@@ -198,13 +203,19 @@ export class AlertDebouncer {
     entry.events.push(event);
     if (onFlushed) entry.callbacks.push(onFlushed);
 
+    const promise = new Promise((resolve) => {
+      entry.callbacks.push(resolve);
+    });
+
     if (entry.events.length >= this._maxBatch) {
-      return this.flush(url);
+      this.flush(url);
+      return promise;
     }
     if (!entry.timer) {
       entry.timer = setTimeout(() => this.flush(url), this._debounceMs);
       if (typeof entry.timer.unref === 'function') entry.timer.unref();
     }
+    return promise;
   }
 
   async flush(url) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.42",
+  "version": "0.8.44",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [