@goodandready/dsh-key-rotation 0.8.41 → 0.8.43

package/CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## 0.8.43 - 2026-10-01
+
+### Fixed
+- **Sanitized GitHub mirror and provenance workflow documentation (#425)**:
+  - Updated `docs/deployment/github-npm-provenance.md` to accurately document the Gitea-first development workflow and sanitized GitHub mirror export (`./scripts/publish-github.sh --push <tag>`), eliminating contradictions regarding direct GitHub PRs and external CI triggers.
+- **LatencyHistogram configuration from latencyWindow setting (#426)**:
+  - Implemented dynamic `reconfigure({ window })` on `LatencyHistogram` (`lib/histogram.js`) and wired `cfg.latencyWindow` from configuration into `buildRuntime()` (`lib/index.js`), ensuring the configured latency sampling window is respected at runtime.
+- **Auto-unbreak quarantine separation and safety checks (#427)**:
+  - Updated `lib/rotate.js` to place keys into a recoverable 30-minute quarantine (`brokenUntil`) after 3 consecutive non-auth failures while protecting rate-limited/quota-exhausted keys from quarantine pollution.
+  - Updated `lib/heal.js` to strictly skip permanently revoked (`revokedRefs`) and paused (`pausedRefs`) credentials during auto-unbreak background probing, preventing probing or accidental reactivation of revoked credentials.
+- **Webhook Alert Digest pipeline integration (#428)**:
+  - Wired `AlertDebouncer` into the production notification pipeline in `lib/index.js`, batching rapid alert bursts into consolidated digest payloads before webhook dispatch as documented.
+
+## 0.8.42 - 2026-10-01
+
+### Fixed
+- **Telemetry and budget monitor model pool coverage (#417)**:
+  - Fixed `/dsh-key-rotation/usage` export and background alert monitoring (`budget-monitor.js`) to iterate over `runtime.pools` instead of `poolByRef.values()`.
+  - Ensures model pools sharing physical credential refs (`provider::model`) are included in usage reports, CSV exports, and low-health/SLO alerts without double-counting aggregate credential totals.
+- **Exposed circuit breaker state and notifyQueue status (#418)**:
+  - Passed `notifyQueue` to operational routes and attached `breaker` and `notifyQueue` to `buildRuntime()`.
+  - `/dsh-key-rotation/status` now reports `circuit` state (`open`, `half-open`, `closed`) per provider and includes `notifyQueue` pending/sent metrics in `meta`.
+- **Bounded Telegram callback_data payload size (#419)**:
+  - Updated `formatInteractive` in `lib/webhook.js` so Telegram inline button `callback_data` payloads are strictly <= 64 bytes and omit raw master `actionToken`.
+  - Prevents Telegram API rejection (`BUTTON_DATA_INVALID`) while retaining Telegram `secret_token` header authentication and callback acknowledgement.
+- **Structured key eligibility in Settings UI (#420)**:
+  - Updated `keyStatus` in `lib/client-src/50-card-main.js` to return structured eligibility flags (`ready`, `paused`, `revoked`, `expired`, `missing`, `cooling`).
+  - Settings card ready counters, header summaries, and ready filter pills no longer miscount paused, revoked, expired, or missing keys as ready. `optimisticReset` state is cleared upon receiving server status updates.
+- **Single root UpdaterSection component (#421)**:
+  - Relocated `UpdaterSection` out of per-provider card row loops to the main card root container in `lib/client-src/50-card-main.js`.
+  - Ensures exactly one updater section is rendered regardless of provider count (0, 1, or N), enabling update checks even with 0 pools configured while eliminating duplicate GET requests.
+- **Quota-aware Prometheus active_keys metric (#423)**:
+  - Updated Prometheus metrics exporter in `lib/ops-metrics.js` to verify model token quota availability when calculating `dsh_key_rotation_active_keys`.
+  - Added new `dsh_key_rotation_quota_exhausted_keys` gauge metric, aligning exporter readiness metrics with `/dsh-key-rotation/health`.
+
 ## 0.8.41 - 2026-10-01
 
 ### Fixed

package/lib/budget-monitor.js

@@ -55,7 +55,8 @@ export function checkBudgetAndHealthAlerts({
     }
 
     const seen = new Set();
-    for (const pool of runtime.poolByRef.values()) {
+    const poolsToScan = runtime.pools ?? runtime.poolByRef.values();
+    for (const pool of poolsToScan) {
       if (seen.has(pool.base)) continue;
       seen.add(pool.base);
       // #207: keys expiring within expiryWarnDays -> one webhook per key/day

package/lib/client.js

@@ -1789,26 +1789,27 @@ window.__ModuleLoader__.load({
       // Key state dot: color and label read at a glance; "key not found"
       // catches an env-name typo that would otherwise stay silent.
       const keyStatus = (providerId, ref, isPaused = false) => {
-        if (isPaused) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyPaused') };
+        if (isPaused) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyPaused'), ready: false, paused: true };
         const hit = keyInfo(providerId, ref);
         if (!hit) return null;
-        if (hit.expired) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyExpired') };
-        if (hit.expiresAt && !hit.expired) {
-          const days = Math.ceil((hit.expiresAt - Date.now()) / 86400000);
-          const warnDays = Number(val?.expiryWarnDays) || 7; // #207: configurable horizon
-          if (days <= warnDays) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyExpiringSoon').replace('{n}', String(days)) };
-        }
-        if (hit.broken || hit.revoked || hit.status === 'revoked') return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyRevoked') };
-        if (hit.broken) return { color: 'var(--dsw-alias-state-error-primary)', text: t('brokenKey') };
-        if (!hit.present) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyMissing') };
+        if (hit.expired) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyExpired'), ready: false, expired: true };
+        if (hit.broken || hit.revoked || hit.status === 'revoked') return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyRevoked'), ready: false, revoked: true };
+        if (!hit.present) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyMissing'), ready: false, missing: true };
         if (hit.cooldownMsLeft > 0 && !optimisticReset[ref]) {
           return {
             color: 'var(--dsw-alias-state-warning-primary)',
             text: t('keyCooling').replace('{s}', String(Math.ceil(hit.cooldownMsLeft / 1000))),
+            ready: false,
+            cooling: true,
           };
         }
-        if (hit.active) return { color: 'var(--dsw-alias-state-success-primary)', text: t('keyActive') };
-        return { color: 'var(--dsw-alias-label-tertiary)', text: t('keyReady') };
+        if (hit.expiresAt && !hit.expired) {
+          const days = Math.ceil((hit.expiresAt - Date.now()) / 86400000);
+          const warnDays = Number(val?.expiryWarnDays) || 7;
+          if (days <= warnDays) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyExpiringSoon').replace('{n}', String(days)), ready: true, expiringSoon: true };
+        }
+        if (hit.active) return { color: 'var(--dsw-alias-state-success-primary)', text: t('keyActive'), ready: true, active: true };
+        return { color: 'var(--dsw-alias-label-tertiary)', text: t('keyReady'), ready: true };
       };
 
       const searchInput = h('input', { className: 'krot-in', placeholder: 'Search providers…', value: search, onChange: (e) => setSearch(e.target.value), style: { marginBottom: '8px' } });
@@ -1829,19 +1830,19 @@ window.__ModuleLoader__.load({
         for (let idx = 0; idx < keys.length; idx++) {
           const k = keys[idx];
           const st = keyStatus(entry.provider, k, Boolean(entryPaused[idx]));
-          const isCool = (st && (st.text === t('brokenKey') || (st.text && st.text.includes(t('keyCooling').slice(0, 4))))) && !optimisticReset[k];
+          const isCool = (st && st.cooling) && !optimisticReset[k];
           const tr = testResult[k];
           if (tr && !tr.ok) errorCount++;
           if (isCool) cooldownCount++;
-          else readyCount++;
+          if (st && st.ready) readyCount++;
         }
 
         const filteredIndices = keys.map((k, idx) => ({ key: k, kIndex: idx })).filter(({ key: k, kIndex: idx }) => {
           if (statusFilter === 'all') return true;
           const st = keyStatus(entry.provider, k, Boolean(entryPaused[idx]));
-          const isCool = (st && (st.text === t('brokenKey') || (st.text && st.text.includes(t('keyCooling').slice(0, 4))))) && !optimisticReset[k];
+          const isCool = (st && st.cooling) && !optimisticReset[k];
           const tr = testResult[k];
-          if (statusFilter === 'ready') return !isCool;
+          if (statusFilter === 'ready') return Boolean(st && st.ready);
           if (statusFilter === 'cooldown') return isCool;
           if (statusFilter === 'error') return tr && !tr.ok;
           return true;
@@ -2224,7 +2225,6 @@ window.__ModuleLoader__.load({
           loadChart,
           h('div', { className: 'krot-keys' }, keyRows),
           modelPools,
-          h(UpdaterSection, { t }),
       h('div', { className: 'krot-foot' },
             btn(t('addKey'), () => addKey(pIndex), { title: t('addKeyTitle'), disabled: !entry.provider.trim() }),
             btn('📋 ' + t('bulkImport'), () => setBulkOpen(cur => ({ ...cur, [pIndex]: !cur[pIndex] })), { disabled: !entry.provider.trim(), title: t('bulkImport') }),
@@ -2303,8 +2303,7 @@ window.__ModuleLoader__.load({
         totalKeysCount += pKeys.length;
         for (const k of pKeys) {
           const st = keyStatus(prov.provider, k);
-          const isDown = st && (st.text === t('brokenKey') || st.text === t('keyMissing') || (st.text && st.text.includes(t('keyCooling').slice(0, 4)))) && !optimisticReset[k];
-          if (!isDown) healthyKeysCount++;
+          if (st && st.ready) healthyKeysCount++;
         }
       }
 
@@ -2377,7 +2376,8 @@ window.__ModuleLoader__.load({
         h('div', { className: 'krot-foot', style: { marginTop: '10px' } },
           btn(t('addProvider'), addProvider, { primary: false }),
           noProviders
-        )
+        ),
+        h(UpdaterSection, { t })
       );
 
       const failoverSection = h('div', { className: 'krot-section-card' },

package/lib/heal.js

@@ -54,6 +54,7 @@ export async function autoUnbreakBrokenKeys(pools, probeFn, now = Date.now(), {
     if (!bu || bu.size === 0) continue;
     const brokenRefs = [];
     for (const [ref, until] of bu.entries()) {
+      if (pool.state.revokedRefs?.has(ref) || pool.state.pausedRefs?.has(ref)) continue;
       if (Number.isFinite(until) && until > now) {
         brokenRefs.push(ref);
       }

package/lib/histogram.js

@@ -4,11 +4,43 @@
 export const LATENCY_DEFAULT_WINDOW = 200;
 
 export class LatencyHistogram {
-  constructor({ window = LATENCY_DEFAULT_WINDOW } = {}) {
+  constructor(opts = {}) {
+    const windowVal = typeof opts === 'number' ? opts : opts?.window;
+    this.reconfigure({ window: windowVal });
+  }
+
+  get window() {
+    return this._window;
+  }
+
+  reconfigure({ window = LATENCY_DEFAULT_WINDOW } = {}) {
     const w = Number.isFinite(window) && window > 0 ? Math.floor(window) : LATENCY_DEFAULT_WINDOW;
+    if (this._window === w && this._buffers) return;
+
+    if (!this._buffers) {
+      this._window = w;
+      this._buffers = new Map(); // ref -> { buf, head, count }
+      this._lastAt = new Map();  // ref -> epochMs
+      return;
+    }
+
+    // Resize existing ring buffers to new length w
+    for (const [ref, entry] of this._buffers.entries()) {
+      const samples = [];
+      const start = entry.count < entry.buf.length ? 0 : entry.head;
+      for (let i = 0; i < entry.count; i++) {
+        samples.push(entry.buf[(start + i) % entry.buf.length]);
+      }
+      const newSamples = samples.slice(-w);
+      const newBuf = new Float64Array(w);
+      for (let i = 0; i < newSamples.length; i++) {
+        newBuf[i] = newSamples[i];
+      }
+      entry.buf = newBuf;
+      entry.head = newSamples.length % w;
+      entry.count = newSamples.length;
+    }
     this._window = w;
-    this._buffers = new Map(); // ref -> Float64Array of size w, plus index/count
-    this._lastAt = new Map();  // ref -> epochMs of last sample
   }
 
   record(ref, ms) {

package/lib/index.js

@@ -20,7 +20,7 @@ import {
 import { LatencyHistogram } from './histogram.js';
 import { ConcurrencyTracker } from './concurrency.js';
 import { QuotaStore } from './quota.js';
-import { WebhookSender } from './webhook.js';
+import { WebhookSender, AlertDebouncer } from './webhook.js';
 import { compactUsage } from './usage-report.js';
 import { json, handleConfigBridge } from './http-bridge.js';
 import { createRotate } from './rotate.js';
@@ -80,7 +80,8 @@ const concurrencyTracker = new ConcurrencyTracker();
 
 let moduleBreaker = new CircuitBreaker({ threshold: 5, openMs: 30000, halfOpenProbes: 1, now: nowMono });
 const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
-let moduleNotifyQueue = new NotifyQueue({ send: (url, payload) => webhookSender.send(url, payload) });
+const alertDebouncer = new AlertDebouncer({ sender: webhookSender, debounceMs: 2000 });
+let moduleNotifyQueue = new NotifyQueue({ send: (url, payload) => alertDebouncer.enqueue(url, payload) });
 
 let getConfig = () => null;
 function verboseLoggingOn() {
@@ -270,6 +271,9 @@ export function apply(ctx, config = {}) {
       openMs: cfg.circuitBreakerOpenMs ?? 30000,
       halfOpenProbes: cfg.circuitBreakerHalfOpenProbes ?? 1,
     });
+    if (cfg.latencyWindow && latencyHistogram.window !== cfg.latencyWindow) {
+      latencyHistogram.reconfigure({ window: cfg.latencyWindow });
+    }
     const switchCodes = cfg.switchCodes ?? DEFAULT_SWITCH_CODES;
     const cooldownMs = cfg.cooldownMs ?? 60000;
     const maxCooldownMs = cfg.maxCooldownMs;
@@ -336,6 +340,7 @@ export function apply(ctx, config = {}) {
       switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000,
       warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0,
       index, pools,
+      breaker: moduleBreaker, notifyQueue: moduleNotifyQueue,
       providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider,
       cloneIds, expectedClones,
       circuitBreakerEnabled: cfg.circuitBreakerEnabled ?? true,
@@ -426,6 +431,7 @@ export function apply(ctx, config = {}) {
     getRotationDisabled: () => rotationDisabled,
     setRotationDisabled: (v) => { rotationDisabled = v; },
     circuitBreaker: moduleBreaker,
+    notifyQueue: moduleNotifyQueue,
     quotaStore,
   });
 

package/lib/ops-metrics.js

@@ -1,6 +1,7 @@
 // lib/ops-metrics.js — Prometheus / OpenMetrics export route (#373).
 import { METRICS_PATH } from './ops-paths.js';
 import { isTrustedBridgeRequest, isKeyPaused, isKeyRevoked } from './pool.js';
+import { hasModelQuotaConfig, isModelQuotaAvailable } from './model-quota.js';
 
 /**
  * @param {object} ctx cordis context
@@ -50,6 +51,9 @@ export function registerMetricsRoutes(ctx, deps) {
       lines.push('# HELP dsh_key_rotation_expired_keys Number of expired keys');
       lines.push('# TYPE dsh_key_rotation_expired_keys gauge');
 
+      lines.push('# HELP dsh_key_rotation_quota_exhausted_keys Number of keys blocked by model token quota');
+      lines.push('# TYPE dsh_key_rotation_quota_exhausted_keys gauge');
+
       lines.push('# HELP dsh_key_rotation_proactive_switches_total Number of proactive rate-limit switches');
       lines.push('# TYPE dsh_key_rotation_proactive_switches_total counter');
 
@@ -59,11 +63,13 @@ export function registerMetricsRoutes(ctx, deps) {
       for (const pool of pools) {
         const p = pool.base;
         const st = poolState.get(p);
-        let active = 0, cooling = 0, paused = 0, revoked = 0, expired = 0;
+        const quotaAware = hasModelQuotaConfig(pool);
+        let active = 0, cooling = 0, paused = 0, revoked = 0, expired = 0, quotaExhausted = 0;
         for (const r of (pool.refs ?? [])) {
           if (isKeyRevoked(pool, r)) { revoked++; continue; }
           if (isKeyPaused(pool, r)) { paused++; continue; }
           if (pool.expiresAt?.[r] !== undefined && now >= pool.expiresAt[r]) { expired++; continue; }
+          if (quotaAware && !isModelQuotaAvailable(pool, r, now)) { quotaExhausted++; continue; }
           const fu = st?.failedUntil?.get(r) ?? 0;
           if (fu > now) cooling++;
           else active++;
@@ -73,6 +79,7 @@ export function registerMetricsRoutes(ctx, deps) {
         lines.push(`dsh_key_rotation_paused_keys{provider="${p}"} ${paused}`);
         lines.push(`dsh_key_rotation_revoked_keys{provider="${p}"} ${revoked}`);
         lines.push(`dsh_key_rotation_expired_keys{provider="${p}"} ${expired}`);
+        lines.push(`dsh_key_rotation_quota_exhausted_keys{provider="${p}"} ${quotaExhausted}`);
         lines.push(`dsh_key_rotation_proactive_switches_total{provider="${p}"} ${st?.proactiveSwitches ?? 0}`);
         lines.push(`dsh_key_rotation_cascade_switches_total{provider="${p}"} ${st?.cascadeCount ?? 0}`);
       }

package/lib/ops-status.js

@@ -29,6 +29,7 @@ export function registerStatusRoutes(ctx, deps) {
     buildRuntime,
     latencyHistogram,
     circuitBreaker,
+    notifyQueue,
     quotaStore,
   } = deps;
 
@@ -89,7 +90,7 @@ ctx.effect(() => ctx.webServer.register({
         } catch {
           /* describe is optional — the card falls back to editable */
         }
-        const until = pool.state.failedUntil.get(ref);
+        const until = pool.state.failedUntil?.get(ref);
         const modelQuota = getModelQuotaStatus(pool, ref, now);
         keys.push({
           ref,
@@ -123,10 +124,10 @@ ctx.effect(() => ctx.webServer.register({
         model: pool.model ?? null,
         quotasConfigured: hasModelQuotaConfig(pool),
         keys,
-        tags: providerTags.get(pool.base) ?? [],
+        tags: providerTags?.get?.(pool.base) ?? [],
         // #260 circuit breaker state (may be null if not yet tripped)
         circuit: (() => {
-          const br = runtime.breaker;
+          const br = circuitBreaker ?? runtime.breaker;
           if (!br) return null;
           const st = br.state(pool.base);
           return { state: st, threshold: br.threshold, openMs: br.openMs };
@@ -164,7 +165,7 @@ ctx.effect(() => ctx.webServer.register({
       // #266/#263 operational extras (additive)
       meta: {
         expectedClones: [...(runtime.expectedClones ?? [])],
-        notifyQueue: runtime.notifyQueue?.stats?.() ?? null,
+        notifyQueue: (notifyQueue ?? runtime.notifyQueue)?.stats?.() ?? (notifyQueue ?? runtime.notifyQueue)?.snapshot?.() ?? null,
         breakerEnabled: runtime.circuitBreakerEnabled !== false,
         at: now,
       },
@@ -197,7 +198,7 @@ ctx.effect(() => ctx.webServer.register({
       let quotaExhausted = 0;
       for (const ref of pool.refs) {
         if (isKeyRevoked(pool, ref) || isKeyPaused(pool, ref)) continue;
-        const until = pool.state.failedUntil.get(ref);
+        const until = pool.state.failedUntil?.get(ref);
         if (until !== undefined && until > now) continue;
         const exp = pool.expiresAt?.[ref];
         if (exp !== undefined && now >= exp) continue;

package/lib/ops-telemetry.js

@@ -40,10 +40,11 @@ ctx.effect(() => ctx.webServer.register({
     const now = Date.now();
     const seen = new Set();
     const report = [];
-    for (const pool of runtime.poolByRef.values()) {
+    const poolsToScan = runtime.pools ?? runtime.poolByRef.values();
+    for (const pool of poolsToScan) {
       if (seen.has(pool.base)) continue;
       seen.add(pool.base);
-      if (provider && pool.base !== provider) continue;
+      if (provider && pool.base !== provider && pool.provider !== provider) continue;
       report.push({ provider: pool.base, rows: usageRows(pool, days, now) });
     }
     if (csv) {

package/lib/rotate.js

@@ -26,6 +26,8 @@ import {
   formatModelQuotaExhaustion,
 } from './model-quota.js';
 
+import { LOCAL_POOL_EXHAUSTED_CODE } from './resolver.js';
+
 export const LOCAL_QUOTA_CODE = 'LOCAL_MODEL_QUOTA_EXHAUSTED';
 
 export async function* preserveDispatchContext(iterable, storage, store) {
@@ -186,6 +188,16 @@ export function createRotate(deps) {
           logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently REVOKED on auth failure`);
         } else {
           pool.state.authFailCounts.delete(targetRef);
+          if (!isQuota) {
+            const _fc = (pool.state.failCounts.get(targetRef) ?? 0) + 1;
+            pool.state.failCounts.set(targetRef, _fc);
+            if (_fc >= 3 && !pool.state.brokenUntil.has(targetRef)) {
+              const quarantineMs = 1800000;
+              pool.state.brokenUntil.set(targetRef, Date.now() + quarantineMs);
+              pushEvent(pool, targetRef, 'KEY_QUARANTINED', quarantineMs);
+              logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} put in recoverable quarantine after ${_fc} failures`);
+            }
+          }
         }
       };
 

package/lib/webhook.js

@@ -31,10 +31,19 @@ export function formatInteractive(url, payload, actionToken) {
       text: `*${title}*\n${text}`,
       parse_mode: 'Markdown',
       reply_markup: {
-        inline_keyboard: [actions.map((a) => ({
-          text: a.label,
-          callback_data: JSON.stringify({ id: a.id, token: actionToken }),
-        }))],
+        inline_keyboard: [actions.map((a) => {
+          let cbData = actionToken ? JSON.stringify({ id: a.id, token: actionToken }) : JSON.stringify({ id: a.id });
+          if (cbData.length > 64) {
+            cbData = JSON.stringify({ id: a.id });
+          }
+          if (cbData.length > 64) {
+            cbData = String(a.id).slice(0, 64);
+          }
+          return {
+            text: a.label,
+            callback_data: cbData,
+          };
+        })],
       },
     };
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.41",
+  "version": "0.8.43",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [