@goodandready/dsh-key-rotation 0.8.28 → 0.8.29

package/CHANGELOG.md

@@ -3,6 +3,27 @@
 All notable changes to `@goodandready/dsh-key-rotation` are documented here.
 User-facing feature notes also appear in README (en is source of truth).
 
+## 0.8.29 - 2026-09-27
+
+### Fixed & Hardened (#376)
+- **Decrypted secret resolution at rest**:
+  - `lib/index.js`: patched `credentials.resolve` now decrypts ciphertext (`enc:v1:...`) via `decryptSecret` for pooled hits, fallback environment variables, and unpooled standalone key queries, preventing raw encrypted strings from being passed to upstream LLM providers.
+- **Fail-safe candidate selection in rotation pool**:
+  - `lib/index.js`: `credentials.resolve` candidate check loop now explicitly enforces `!isKeyRevoked(pool, candidate)` and `!isKeyPaused(pool, candidate)` before selecting candidates.
+- **Predicate logic & dynamic state resolution**:
+  - `lib/pool.js`: corrected `isKeyPaused` predicate to ensure `pool.state.pausedRefs` is inspected when `pool.pausedRefs` Set or Array does not contain the ref, allowing runtime key pause mutations to take effect.
+  - `lib/cascade.js`: replaced raw `.has()` call with `isKeyRevoked(pool, ref)` and unified `isKeyPaused`, preventing `TypeError` crashes when `revokedRefs` is represented as an Array.
+- **Config & schema synchronization**:
+  - `lib/index.js`: propagated `p.revoked` into `buildPool` / `buildPoolItem` and added missing `paused` and `revoked` definitions to `SettingsSchema.providers`.
+- **Status & key administrative routes**:
+  - `lib/ops-status.js`: exposed `paused` and `revoked` booleans on key entries in `/dsh-key-rotation/status`.
+  - `lib/pool.js`: updated `keyTail` to transparently decrypt ciphertext before extracting the tail, displaying the actual credential mask instead of random ciphertext characters while preserving test safety contracts.
+  - `lib/ops-keys.js`: `/dsh-key-rotation/reset` now clears `st.revokedRefs` for provider resets and single ref resets, enabling operators to unblock fixed keys.
+- **Prometheus metrics & monitoring**:
+  - `lib/ops-metrics.js`: separated expired keys from active counts and added `dsh_key_rotation_expired_keys` gauge.
+  - `lib/budget-monitor.js`: excluded revoked and paused keys from healthy count in `warnBelowHealthy` threshold evaluation.
+  - `lib/webhook.js`: hardened Ntfy topic extraction against URLs with trailing slashes and query strings.
+
 ## 0.8.28 - 2026-09-27
 
 ### Security & Key Lifecycle (Block 3)

package/lib/budget-monitor.js

@@ -1,5 +1,5 @@
 // lib/budget-monitor.js — periodic budget, expiry, low health and SLO alerts
-import { expiringSoon, shouldNotifyDaily, costForDay, costForWeek, budgetVerdict } from './pool.js';
+import { expiringSoon, shouldNotifyDaily, costForDay, costForWeek, budgetVerdict, isKeyPaused, isKeyRevoked } from './pool.js';
 
 const DAY_MS = 86400000;
 
@@ -72,6 +72,7 @@ export function checkBudgetAndHealthAlerts({
       if (warnBelow > 0) {
         let healthy = 0;
         for (const ref of pool.refs) {
+          if (isKeyRevoked(pool, ref) || isKeyPaused(pool, ref)) continue;
           const fu = pool.state.failedUntil.get(ref);
           if (fu !== undefined && fu > now) continue;
           const exp = pool.expiresAt?.[ref];

package/lib/cascade.js

@@ -1,15 +1,9 @@
-// cascade.js — cross-provider failover cascade (issues #194, #364).
+// cascade.js — cross-provider failover cascade (issues #194, #364, #376).
 // ponytail: minimal — pick fallback provider with model mapping from config.
+import { isKeyPaused, isKeyRevoked } from './pool.js';
 
 export const CASCADE_MAX_DEPTH = 3;
 
-function isKeyPaused(pool, ref) {
-  if (!pool || !ref) return false;
-  if (pool.pausedRefs instanceof Set) return pool.pausedRefs.has(ref);
-  if (Array.isArray(pool.pausedRefs)) return pool.pausedRefs.includes(ref);
-  return false;
-}
-
 export function pickCascadeFallback(provider, cfg, pools, currentModel) {
   const list = Array.isArray(cfg && cfg.cascade) ? cfg.cascade : [];
   for (const entry of list) {
@@ -20,7 +14,7 @@ export function pickCascadeFallback(provider, cfg, pools, currentModel) {
     const now = Date.now();
     let healthy = 0;
     for (const ref of pool.refs) {
-      if (isKeyPaused(pool, ref) || (pool.revokedRefs && pool.revokedRefs.has(ref)) || (pool.state?.revokedRefs && pool.state.revokedRefs.has(ref))) continue;
+      if (isKeyPaused(pool, ref) || isKeyRevoked(pool, ref)) continue;
       const failedUntil = (pool.state && pool.state.failedUntil && pool.state.failedUntil.get(ref)) || 0;
       if (failedUntil > now) continue;
       const exp = pool.expiresAt ? pool.expiresAt[ref] : undefined;

package/lib/index.js

@@ -1,3 +1,4 @@
+import { decryptSecret } from './crypto-storage.js';
 // dsh-key-rotation — per-provider API key rotation for DeepSeek Harness.
 //
 // Transparent key pool rotation: credentials.resolve patch + llm/stream
@@ -13,7 +14,7 @@ import {
   recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter,
   computeHealthScore, extractRateLimit, isRateLimited, selectPool, isSwitchableError,
   formatExhaustionMessage, expiringSoon, shouldNotifyDaily, costForDay, costForWeek,
-  budgetVerdict, sortAttemptList,
+  budgetVerdict, sortAttemptList, isKeyPaused, isKeyRevoked,
 } from './pool.js';
 import { LatencyHistogram } from './histogram.js';
 import { ConcurrencyTracker } from './concurrency.js';
@@ -115,6 +116,8 @@ const SettingsSchema = Schema.object({
     provider: Schema.string().required(),
     keys: Schema.array(Schema.string()).default([]),
     weights: Schema.array(Schema.number()).default([]),
+    paused: Schema.array(Schema.boolean()).default([]),
+    revoked: Schema.array(Schema.boolean()).default([]),
     expiresAt: Schema.array(Schema.union([Schema.number(), Schema.string()])).default([]),
     tags: Schema.array(Schema.string()).default([]),
     costBudgetDaily: Schema.number(),
@@ -263,13 +266,13 @@ export function apply(ctx, config = {}) {
       return initializePoolState(st);
     };
 
-    const buildPool = (base, keys, weights, paused, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard) =>
-      buildPoolItem({ base, keys, weights, paused, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, rpmLimit, makeState });
+    const buildPool = (base, keys, weights, paused, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, revoked) =>
+      buildPoolItem({ base, keys, weights, paused, revoked, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, rpmLimit, makeState });
 
     for (const p of cfg.providers ?? []) {
       const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
       const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
-      const pool = buildPool(p.provider, p.keys, p.weights, p.paused, poolCooldown, poolMax, p.expiresAt, p.routingStrategy, p.proactiveRateLimitGuard);
+      const pool = buildPool(p.provider, p.keys, p.weights, p.paused, poolCooldown, poolMax, p.expiresAt, p.routingStrategy, p.proactiveRateLimitGuard, p.revoked);
       if (pool) {
         for (const ref of pool.refs) poolByRef.set(ref, pool);
         for (let i = 1; i < pool.refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
@@ -277,7 +280,7 @@ export function apply(ctx, config = {}) {
       const models = p.models ?? {};
       const byModel = new Map();
       for (const [model, mp] of Object.entries(models)) {
-        const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, mp.paused, poolCooldown, poolMax, undefined, p.routingStrategy, p.proactiveRateLimitGuard);
+        const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, mp.paused, poolCooldown, poolMax, undefined, p.routingStrategy, p.proactiveRateLimitGuard, mp.revoked);
         if (mpool) {
           byModel.set(model, mpool);
           for (const ref of mpool.refs) poolByRef.set(ref, mpool);
@@ -353,7 +356,15 @@ export function apply(ctx, config = {}) {
       credentials.resolve = async (ref) => {
         const { poolByRef } = buildRuntime();
         const pool = poolByRef.get(ref);
-        if (!pool) return original(ref);
+        if (!pool) {
+          const res = await original(ref);
+          if (res && typeof res.value === 'string') {
+            return { ...res, value: decryptSecret(res.value) };
+          }
+          const ev = envValue(ref);
+          if (ev !== undefined) return { value: decryptSecret(ev) };
+          return res;
+        }
         const now = Date.now();
         const strat = pool.routingStrategy ?? buildRuntime().routingStrategy ?? 'round-robin';
         let list = pool.weightedRefs ?? pool.refs;
@@ -364,6 +375,8 @@ export function apply(ctx, config = {}) {
         for (let i = 0; i < list.length; i++) {
           const index = (start + i) % list.length;
           const candidate = list[index];
+          if (isKeyRevoked(pool, candidate)) continue;
+          if (isKeyPaused(pool, candidate)) continue;
           const until = pool.state.failedUntil.get(candidate);
           if (until !== undefined && until > now) continue;
           if (pool.expiresAt?.[candidate] !== undefined && now >= pool.expiresAt[candidate]) continue;
@@ -391,6 +404,7 @@ export function apply(ctx, config = {}) {
           pool.state.pointer = (index + 1) % list.length;
           let hit = await original(candidate);
           if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
+            hit = { ...hit, value: decryptSecret(hit.value) };
             pool.state.lastUsed = candidate;
             if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
             pool.state.lastUsedAt.set(candidate, now);
@@ -413,6 +427,7 @@ export function apply(ctx, config = {}) {
           }
           const envVal = envValue(candidate);
           if (envVal !== undefined) {
+            const decEnv = decryptSecret(envVal);
             pool.state.lastUsed = candidate;
             if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
             pool.state.lastUsedAt.set(candidate, now);

package/lib/ops-keys.js

@@ -1,3 +1,4 @@
+import { decryptSecret } from './crypto-storage.js';
 // lib/ops-keys.js — key store + cooldown reset + import routes (#312 split from routes-ops.js).
 import {
   json,
@@ -75,6 +76,7 @@ ctx.effect(() => ctx.webServer.register({
           st.failCounts?.delete(ref);
           st.authFailCounts?.delete(ref);
           st.brokenUntil?.delete(ref);
+          st.revokedRefs?.delete(ref);
           if (st.lastUsed === ref) st.lastUsed = undefined;
         }
         bestEffort('lastTestCache.delete', () => { lastTestCache?.delete?.(ref); }, ctx.logger);
@@ -127,6 +129,7 @@ ctx.effect(() => ctx.webServer.register({
       st.failCounts?.clear();
       st.authFailCounts?.clear();
       st.brokenUntil?.clear();
+      st.revokedRefs?.clear();
       st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
       let circuitReset = false;
       const br = circuitBreaker ?? buildRuntime().breaker;
@@ -145,6 +148,7 @@ ctx.effect(() => ctx.webServer.register({
           st.failCounts?.delete(ref);
           st.authFailCounts?.delete(ref);
           st.brokenUntil?.delete(ref);
+          st.revokedRefs?.delete(ref);
           if (st.lastUsed === ref) st.lastUsed = undefined;
           found = true; break;
         }

package/lib/ops-metrics.js

@@ -47,6 +47,9 @@ export function registerMetricsRoutes(ctx, deps) {
       lines.push('# HELP dsh_key_rotation_revoked_keys Number of permanently revoked keys');
       lines.push('# TYPE dsh_key_rotation_revoked_keys gauge');
 
+      lines.push('# HELP dsh_key_rotation_expired_keys Number of expired keys');
+      lines.push('# TYPE dsh_key_rotation_expired_keys gauge');
+
       lines.push('# HELP dsh_key_rotation_proactive_switches_total Number of proactive rate-limit switches');
       lines.push('# TYPE dsh_key_rotation_proactive_switches_total counter');
 
@@ -56,10 +59,11 @@ export function registerMetricsRoutes(ctx, deps) {
       for (const pool of pools) {
         const p = pool.base;
         const st = poolState.get(p);
-        let active = 0, cooling = 0, paused = 0, revoked = 0;
+        let active = 0, cooling = 0, paused = 0, revoked = 0, expired = 0;
         for (const r of (pool.refs ?? [])) {
           if (isKeyRevoked(pool, r)) { revoked++; continue; }
           if (isKeyPaused(pool, r)) { paused++; continue; }
+          if (pool.expiresAt?.[r] !== undefined && now >= pool.expiresAt[r]) { expired++; continue; }
           const fu = st?.failedUntil?.get(r) ?? 0;
           if (fu > now) cooling++;
           else active++;
@@ -68,6 +72,7 @@ export function registerMetricsRoutes(ctx, deps) {
         lines.push(`dsh_key_rotation_cooldown_keys{provider="${p}"} ${cooling}`);
         lines.push(`dsh_key_rotation_paused_keys{provider="${p}"} ${paused}`);
         lines.push(`dsh_key_rotation_revoked_keys{provider="${p}"} ${revoked}`);
+        lines.push(`dsh_key_rotation_expired_keys{provider="${p}"} ${expired}`);
         lines.push(`dsh_key_rotation_proactive_switches_total{provider="${p}"} ${st?.proactiveSwitches ?? 0}`);
         lines.push(`dsh_key_rotation_cascade_switches_total{provider="${p}"} ${st?.cascadeCount ?? 0}`);
       }

package/lib/ops-status.js

@@ -1,7 +1,10 @@
+import { decryptSecret } from './crypto-storage.js';
 // lib/ops-status.js — status + health operational routes (#312 split from routes-ops.js).
 import { json } from './http-bridge.js';
 import {
   isTrustedBridgeRequest,
+  isKeyPaused,
+  isKeyRevoked,
   keyTail,
   envValue,
   computeHealthScore,
@@ -106,6 +109,8 @@ ctx.effect(() => ctx.webServer.register({
           expiresAt: pool.expiresAt?.[ref] ?? null,
           expired: pool.expiresAt?.[ref] !== undefined && now >= pool.expiresAt[ref],
           broken: pool.state.brokenUntil?.has(ref) ?? false,
+          paused: isKeyPaused(pool, ref),
+          revoked: isKeyRevoked(pool, ref),
         });
       }
       providers.push({

package/lib/pool.js

@@ -1,3 +1,4 @@
+import { decryptSecret } from './crypto-storage.js';
 import { bestEffort } from './best-effort.js';
 // lib/pool.js — pure pool arithmetic and selection logic for dsh-key-rotation.
 // Isolated from cordis/dsh runtime so unit tests run in vanilla Node.js.
@@ -11,8 +12,9 @@ export const KEY_TAIL_PLACEHOLDER = '***';
 /** Mask key to show only the last KEY_TAIL_CHARS; returns placeholder for short keys (#355). */
 export function keyTail(key) {
   if (typeof key !== 'string' || key.length === 0) return '';
-  if (key.length <= KEY_TAIL_CHARS) return KEY_TAIL_PLACEHOLDER;
-  return key.slice(-KEY_TAIL_CHARS);
+  const plain = decryptSecret(key);
+  if (plain.length <= KEY_TAIL_CHARS) return KEY_TAIL_PLACEHOLDER;
+  return plain.slice(-KEY_TAIL_CHARS);
 }
 
 /** Check if an IP address is a loopback address. */
@@ -290,9 +292,10 @@ export function parseDurationToSeconds(v) {
 /** Check if a key reference is in maintenance/paused mode. */
 export function isKeyPaused(pool, ref) {
   if (!pool || !ref) return false;
-  if (pool.pausedRefs instanceof Set) return pool.pausedRefs.has(ref);
-  if (Array.isArray(pool.pausedRefs)) return pool.pausedRefs.includes(ref);
-  if (pool.state && pool.state.pausedRefs instanceof Set) return pool.state.pausedRefs.has(ref);
+  if (pool.pausedRefs instanceof Set && pool.pausedRefs.has(ref)) return true;
+  if (Array.isArray(pool.pausedRefs) && pool.pausedRefs.includes(ref)) return true;
+  if (pool.state && pool.state.pausedRefs instanceof Set && pool.state.pausedRefs.has(ref)) return true;
+  if (pool.state && Array.isArray(pool.state.pausedRefs) && pool.state.pausedRefs.includes(ref)) return true;
   return false;
 }
 

package/lib/webhook.js

@@ -81,8 +81,16 @@ export function formatInteractive(url, payload, actionToken) {
     };
   }
   if (platform === 'ntfy') {
+    let topic = 'alerts';
+    try {
+      const u = new URL(url);
+      const parts = u.pathname.split('/').filter(Boolean);
+      if (parts.length > 0) topic = parts[parts.length - 1];
+    } catch (_) {
+      topic = String(url).split('?')[0].split('/').filter(Boolean).pop() || 'alerts';
+    }
     return {
-      topic: url.split('/').pop(),
+      topic,
       title,
       message: text,
       priority: 4,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.28",
+  "version": "0.8.29",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [