@goodandready/dsh-key-rotation 0.8.27 → 0.8.29

package/CHANGELOG.md

@@ -3,6 +3,63 @@
 All notable changes to `@goodandready/dsh-key-rotation` are documented here.
 User-facing feature notes also appear in README (en is source of truth).
 
+## 0.8.29 - 2026-09-27
+
+### Fixed & Hardened (#376)
+- **Decrypted secret resolution at rest**:
+  - `lib/index.js`: patched `credentials.resolve` now decrypts ciphertext (`enc:v1:...`) via `decryptSecret` for pooled hits, fallback environment variables, and unpooled standalone key queries, preventing raw encrypted strings from being passed to upstream LLM providers.
+- **Fail-safe candidate selection in rotation pool**:
+  - `lib/index.js`: `credentials.resolve` candidate check loop now explicitly enforces `!isKeyRevoked(pool, candidate)` and `!isKeyPaused(pool, candidate)` before selecting candidates.
+- **Predicate logic & dynamic state resolution**:
+  - `lib/pool.js`: corrected `isKeyPaused` predicate to ensure `pool.state.pausedRefs` is inspected when `pool.pausedRefs` Set or Array does not contain the ref, allowing runtime key pause mutations to take effect.
+  - `lib/cascade.js`: replaced raw `.has()` call with `isKeyRevoked(pool, ref)` and unified `isKeyPaused`, preventing `TypeError` crashes when `revokedRefs` is represented as an Array.
+- **Config & schema synchronization**:
+  - `lib/index.js`: propagated `p.revoked` into `buildPool` / `buildPoolItem` and added missing `paused` and `revoked` definitions to `SettingsSchema.providers`.
+- **Status & key administrative routes**:
+  - `lib/ops-status.js`: exposed `paused` and `revoked` booleans on key entries in `/dsh-key-rotation/status`.
+  - `lib/pool.js`: updated `keyTail` to transparently decrypt ciphertext before extracting the tail, displaying the actual credential mask instead of random ciphertext characters while preserving test safety contracts.
+  - `lib/ops-keys.js`: `/dsh-key-rotation/reset` now clears `st.revokedRefs` for provider resets and single ref resets, enabling operators to unblock fixed keys.
+- **Prometheus metrics & monitoring**:
+  - `lib/ops-metrics.js`: separated expired keys from active counts and added `dsh_key_rotation_expired_keys` gauge.
+  - `lib/budget-monitor.js`: excluded revoked and paused keys from healthy count in `warnBelowHealthy` threshold evaluation.
+  - `lib/webhook.js`: hardened Ntfy topic extraction against URLs with trailing slashes and query strings.
+
+## 0.8.28 - 2026-09-27
+
+### Security & Key Lifecycle (Block 3)
+- **Permanent Key Revocation (#369)**:
+  - Added `revokedRefs` tracking in pool state and configuration. Keys returning permanent auth failures (HTTP 401 / `AUTH_FAILED`) are permanently revoked and excluded from rotation and cascade attempts.
+  - Added visual revoked status badges and tokens in WebUI (`lib/client.js`).
+- **Key Encryption at Rest (#370)**:
+  - Added `lib/crypto-storage.js` implementing authenticated AES-256-GCM encryption with 12-byte IV and 16-byte auth tag.
+  - Transparently uses environment variable `DSH_KEY_SECRET` when configured, with safe plaintext fallback when unset.
+- **Key Expiration Tracker (#371)**:
+  - Added automated expiration date tracking (`expiresAt`), horizon detection (`expiringSoon`), 24-hour notification deduplication (`shouldNotifyDaily`), and automated exclusion of expired keys from active routing.
+
+### Monitoring & Integrations (Block 4)
+- **Multi-Platform Webhook Integrations (#372)**:
+  - Extended `lib/webhook.js` with native detection and formatting for Gotify (`/message` with JSON priority/extras) and Ntfy (`/alerts` with action buttons).
+- **Prometheus Metrics Exporter (#373)**:
+  - Added `/dsh-key-rotation/metrics` route (`lib/ops-metrics.js`) exposing OpenMetrics / Prometheus standard text format with gauges for pool totals, active/cooldown/paused/revoked keys, and counters for proactive switches and cascades. Guarded by loopback origin fail-closed security.
+
+## 0.8.27 - 2026-09-27
+
+### Smart Routing (Block 1)
+- **Proactive Rate-Limit Cooldown (#363)**:
+  - Proactive key cooldown based on upstream rate limit reset headers (`x-ratelimit-reset`, `x-ratelimit-reset-requests`, `x-ratelimit-reset-tokens`) supporting time deltas and ISO timestamps.
+- **Cross-Provider Cascading (#364)**:
+  - Cascade routing across providers on pool exhaustion with `modelMapping` and max cascade depth guard.
+- **Adaptive Concurrency Normalization (#365)**:
+  - Weighted least-loaded concurrency balancing (`inflight / weight`).
+
+### WebUI Suite (Block 2)
+- **Bulk Key Import (#366)**:
+  - WebUI modal for bulk key import supporting line-by-line, `.env`, and CSV formats.
+- **Key Maintenance Toggle (#367)**:
+  - Live key pause/resume toggle (`isPaused`) in WebUI.
+- **Traffic Sparklines (#368)**:
+  - Native SVG Sparkline component for request and latency visual trends.
+
 ## 0.8.25 - 2026-09-27
 
 ### Fixed & Hardened

package/lib/budget-monitor.js

@@ -1,5 +1,5 @@
 // lib/budget-monitor.js — periodic budget, expiry, low health and SLO alerts
-import { expiringSoon, shouldNotifyDaily, costForDay, costForWeek, budgetVerdict } from './pool.js';
+import { expiringSoon, shouldNotifyDaily, costForDay, costForWeek, budgetVerdict, isKeyPaused, isKeyRevoked } from './pool.js';
 
 const DAY_MS = 86400000;
 
@@ -72,6 +72,7 @@ export function checkBudgetAndHealthAlerts({
       if (warnBelow > 0) {
         let healthy = 0;
         for (const ref of pool.refs) {
+          if (isKeyRevoked(pool, ref) || isKeyPaused(pool, ref)) continue;
           const fu = pool.state.failedUntil.get(ref);
           if (fu !== undefined && fu > now) continue;
           const exp = pool.expiresAt?.[ref];

package/lib/cascade.js

@@ -1,15 +1,9 @@
-// cascade.js — cross-provider failover cascade (issues #194, #364).
+// cascade.js — cross-provider failover cascade (issues #194, #364, #376).
 // ponytail: minimal — pick fallback provider with model mapping from config.
+import { isKeyPaused, isKeyRevoked } from './pool.js';
 
 export const CASCADE_MAX_DEPTH = 3;
 
-function isKeyPaused(pool, ref) {
-  if (!pool || !ref) return false;
-  if (pool.pausedRefs instanceof Set) return pool.pausedRefs.has(ref);
-  if (Array.isArray(pool.pausedRefs)) return pool.pausedRefs.includes(ref);
-  return false;
-}
-
 export function pickCascadeFallback(provider, cfg, pools, currentModel) {
   const list = Array.isArray(cfg && cfg.cascade) ? cfg.cascade : [];
   for (const entry of list) {
@@ -20,7 +14,7 @@ export function pickCascadeFallback(provider, cfg, pools, currentModel) {
     const now = Date.now();
     let healthy = 0;
     for (const ref of pool.refs) {
-      if (isKeyPaused(pool, ref)) continue;
+      if (isKeyPaused(pool, ref) || isKeyRevoked(pool, ref)) continue;
       const failedUntil = (pool.state && pool.state.failedUntil && pool.state.failedUntil.get(ref)) || 0;
       if (failedUntil > now) continue;
       const exp = pool.expiresAt ? pool.expiresAt[ref] : undefined;

package/lib/client.js

@@ -84,6 +84,7 @@ window.__ModuleLoader__.load({
       bulkImportCancel: 'Cancel',
       bulkImportSuccess: 'Imported {n} keys',
       keyPaused: 'Paused',
+      keyRevoked: 'Revoked (401)',
       pauseKey: 'Pause key from rotation',
       resumeKey: 'Resume key rotation',
       sparklineTraffic: 'Traffic Trend',
@@ -239,6 +240,7 @@ window.__ModuleLoader__.load({
       bulkImportCancel: '取消',
       bulkImportSuccess: '已导入 {n} 个密钥',
       keyPaused: '已暂停',
+      keyRevoked: '已撤销 (401)',
       pauseKey: '暂停此密钥轮换',
       resumeKey: '恢复此密钥轮换',
       sparklineTraffic: '流量趋势',
@@ -1590,6 +1592,7 @@ window.__ModuleLoader__.load({
           const warnDays = Number(val?.expiryWarnDays) || 7; // #207: configurable horizon
           if (days <= warnDays) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyExpiringSoon').replace('{n}', String(days)) };
         }
+        if (hit.broken || hit.revoked || hit.status === 'revoked') return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyRevoked') };
         if (hit.broken) return { color: 'var(--dsw-alias-state-error-primary)', text: t('brokenKey') };
         if (!hit.present) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyMissing') };
         if (hit.cooldownMsLeft > 0 && !optimisticReset[ref]) {

package/lib/crypto-storage.js

@@ -0,0 +1,49 @@
+// lib/crypto-storage.js — AES-256-GCM encryption for credentials at rest (#370)
+import crypto from 'node:crypto';
+
+const PREFIX = 'enc:v1:';
+
+function getDerivedKey(secret) {
+  return crypto.createHash('sha256').update(String(secret)).digest();
+}
+
+/**
+ * Encrypt plaintext using AES-256-GCM if secretKey is provided.
+ * If secretKey is not set, returns plaintext as-is.
+ */
+export function encryptSecret(plaintext, secretKey = process.env.DSH_KEY_SECRET) {
+  if (!secretKey || !plaintext || typeof plaintext !== 'string') return plaintext;
+  if (plaintext.startsWith(PREFIX)) return plaintext; // already encrypted
+  const key = getDerivedKey(secretKey);
+  const iv = crypto.randomBytes(12);
+  const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
+  let ciphertext = cipher.update(plaintext, 'utf8', 'hex');
+  ciphertext += cipher.final('hex');
+  const authTag = cipher.getAuthTag().toString('hex');
+  return `${PREFIX}${iv.toString('hex')}:${authTag}:${ciphertext}`;
+}
+
+/**
+ * Decrypt ciphertext using AES-256-GCM if prefixed with enc:v1:.
+ * If not encrypted, or if secretKey is missing/incorrect, returns safe fallback.
+ */
+export function decryptSecret(ciphertext, secretKey = process.env.DSH_KEY_SECRET) {
+  if (!ciphertext || typeof ciphertext !== 'string') return ciphertext;
+  if (!ciphertext.startsWith(PREFIX)) return ciphertext; // plaintext fallback
+  if (!secretKey) return ciphertext;
+  try {
+    const parts = ciphertext.slice(PREFIX.length).split(':');
+    if (parts.length !== 3) return ciphertext;
+    const [ivHex, tagHex, dataHex] = parts;
+    const key = getDerivedKey(secretKey);
+    const iv = Buffer.from(ivHex, 'hex');
+    const authTag = Buffer.from(tagHex, 'hex');
+    const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
+    decipher.setAuthTag(authTag);
+    let decrypted = decipher.update(dataHex, 'hex', 'utf8');
+    decrypted += decipher.final('utf8');
+    return decrypted;
+  } catch {
+    return ciphertext;
+  }
+}

package/lib/http-bridge.js

@@ -1,6 +1,7 @@
 // lib/http-bridge.js — shared HTTP helpers for dsh-key-rotation routes.
 import { findSecrets } from './keycheck.js';
 import { isTrustedBridgeRequest } from './pool.js';
+export { isTrustedBridgeRequest };
 import { mutateSettings, plainObject, requireRevision, sectionOps, validateOps } from './settings-write.js';
 
 export const NS = 'dsh-key-rotation';

package/lib/index.js

@@ -1,3 +1,4 @@
+import { decryptSecret } from './crypto-storage.js';
 // dsh-key-rotation — per-provider API key rotation for DeepSeek Harness.
 //
 // Transparent key pool rotation: credentials.resolve patch + llm/stream
@@ -13,7 +14,7 @@ import {
   recordSuccess, computeBackoff, envValue, sweepExpired, parseRetryAfter,
   computeHealthScore, extractRateLimit, isRateLimited, selectPool, isSwitchableError,
   formatExhaustionMessage, expiringSoon, shouldNotifyDaily, costForDay, costForWeek,
-  budgetVerdict, sortAttemptList,
+  budgetVerdict, sortAttemptList, isKeyPaused, isKeyRevoked,
 } from './pool.js';
 import { LatencyHistogram } from './histogram.js';
 import { ConcurrencyTracker } from './concurrency.js';
@@ -115,6 +116,8 @@ const SettingsSchema = Schema.object({
     provider: Schema.string().required(),
     keys: Schema.array(Schema.string()).default([]),
     weights: Schema.array(Schema.number()).default([]),
+    paused: Schema.array(Schema.boolean()).default([]),
+    revoked: Schema.array(Schema.boolean()).default([]),
     expiresAt: Schema.array(Schema.union([Schema.number(), Schema.string()])).default([]),
     tags: Schema.array(Schema.string()).default([]),
     costBudgetDaily: Schema.number(),
@@ -263,13 +266,13 @@ export function apply(ctx, config = {}) {
       return initializePoolState(st);
     };
 
-    const buildPool = (base, keys, weights, paused, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard) =>
-      buildPoolItem({ base, keys, weights, paused, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, rpmLimit, makeState });
+    const buildPool = (base, keys, weights, paused, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, revoked) =>
+      buildPoolItem({ base, keys, weights, paused, revoked, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, rpmLimit, makeState });
 
     for (const p of cfg.providers ?? []) {
       const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
       const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
-      const pool = buildPool(p.provider, p.keys, p.weights, p.paused, poolCooldown, poolMax, p.expiresAt, p.routingStrategy, p.proactiveRateLimitGuard);
+      const pool = buildPool(p.provider, p.keys, p.weights, p.paused, poolCooldown, poolMax, p.expiresAt, p.routingStrategy, p.proactiveRateLimitGuard, p.revoked);
       if (pool) {
         for (const ref of pool.refs) poolByRef.set(ref, pool);
         for (let i = 1; i < pool.refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
@@ -277,7 +280,7 @@ export function apply(ctx, config = {}) {
       const models = p.models ?? {};
       const byModel = new Map();
       for (const [model, mp] of Object.entries(models)) {
-        const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, mp.paused, poolCooldown, poolMax, undefined, p.routingStrategy, p.proactiveRateLimitGuard);
+        const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, mp.paused, poolCooldown, poolMax, undefined, p.routingStrategy, p.proactiveRateLimitGuard, mp.revoked);
         if (mpool) {
           byModel.set(model, mpool);
           for (const ref of mpool.refs) poolByRef.set(ref, mpool);
@@ -353,7 +356,15 @@ export function apply(ctx, config = {}) {
       credentials.resolve = async (ref) => {
         const { poolByRef } = buildRuntime();
         const pool = poolByRef.get(ref);
-        if (!pool) return original(ref);
+        if (!pool) {
+          const res = await original(ref);
+          if (res && typeof res.value === 'string') {
+            return { ...res, value: decryptSecret(res.value) };
+          }
+          const ev = envValue(ref);
+          if (ev !== undefined) return { value: decryptSecret(ev) };
+          return res;
+        }
         const now = Date.now();
         const strat = pool.routingStrategy ?? buildRuntime().routingStrategy ?? 'round-robin';
         let list = pool.weightedRefs ?? pool.refs;
@@ -364,6 +375,8 @@ export function apply(ctx, config = {}) {
         for (let i = 0; i < list.length; i++) {
           const index = (start + i) % list.length;
           const candidate = list[index];
+          if (isKeyRevoked(pool, candidate)) continue;
+          if (isKeyPaused(pool, candidate)) continue;
           const until = pool.state.failedUntil.get(candidate);
           if (until !== undefined && until > now) continue;
           if (pool.expiresAt?.[candidate] !== undefined && now >= pool.expiresAt[candidate]) continue;
@@ -391,6 +404,7 @@ export function apply(ctx, config = {}) {
           pool.state.pointer = (index + 1) % list.length;
           let hit = await original(candidate);
           if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
+            hit = { ...hit, value: decryptSecret(hit.value) };
             pool.state.lastUsed = candidate;
             if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
             pool.state.lastUsedAt.set(candidate, now);
@@ -413,6 +427,7 @@ export function apply(ctx, config = {}) {
           }
           const envVal = envValue(candidate);
           if (envVal !== undefined) {
+            const decEnv = decryptSecret(envVal);
             pool.state.lastUsed = candidate;
             if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
             pool.state.lastUsedAt.set(candidate, now);

package/lib/ops-keys.js

@@ -1,3 +1,4 @@
+import { decryptSecret } from './crypto-storage.js';
 // lib/ops-keys.js — key store + cooldown reset + import routes (#312 split from routes-ops.js).
 import {
   json,
@@ -75,6 +76,7 @@ ctx.effect(() => ctx.webServer.register({
           st.failCounts?.delete(ref);
           st.authFailCounts?.delete(ref);
           st.brokenUntil?.delete(ref);
+          st.revokedRefs?.delete(ref);
           if (st.lastUsed === ref) st.lastUsed = undefined;
         }
         bestEffort('lastTestCache.delete', () => { lastTestCache?.delete?.(ref); }, ctx.logger);
@@ -127,6 +129,7 @@ ctx.effect(() => ctx.webServer.register({
       st.failCounts?.clear();
       st.authFailCounts?.clear();
       st.brokenUntil?.clear();
+      st.revokedRefs?.clear();
       st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
       let circuitReset = false;
       const br = circuitBreaker ?? buildRuntime().breaker;
@@ -145,6 +148,7 @@ ctx.effect(() => ctx.webServer.register({
           st.failCounts?.delete(ref);
           st.authFailCounts?.delete(ref);
           st.brokenUntil?.delete(ref);
+          st.revokedRefs?.delete(ref);
           if (st.lastUsed === ref) st.lastUsed = undefined;
           found = true; break;
         }

package/lib/ops-metrics.js

@@ -0,0 +1,85 @@
+// lib/ops-metrics.js — Prometheus / OpenMetrics export route (#373).
+import { METRICS_PATH } from './ops-paths.js';
+import { isTrustedBridgeRequest, isKeyPaused, isKeyRevoked } from './pool.js';
+
+/**
+ * @param {object} ctx cordis context
+ * @param {object} deps live dependencies from apply()
+ */
+export function registerMetricsRoutes(ctx, deps) {
+  const { buildRuntime, poolState } = deps;
+
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: METRICS_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'GET') {
+        res.statusCode = 405;
+        res.setHeader('content-type', 'text/plain');
+        res.end('Method Not Allowed\n');
+        return;
+      }
+      if (!isTrustedBridgeRequest(req)) {
+        res.statusCode = 403;
+        res.setHeader('content-type', 'text/plain');
+        res.end('Forbidden: local only\n');
+        return;
+      }
+
+      const lines = [];
+      const runtime = buildRuntime();
+      const pools = runtime.pools ?? [];
+      const now = Date.now();
+
+      lines.push('# HELP dsh_key_rotation_pools_total Total configured provider pools');
+      lines.push('# TYPE dsh_key_rotation_pools_total gauge');
+      lines.push(`dsh_key_rotation_pools_total ${pools.length}`);
+
+      lines.push('# HELP dsh_key_rotation_active_keys Number of currently active and ready keys');
+      lines.push('# TYPE dsh_key_rotation_active_keys gauge');
+
+      lines.push('# HELP dsh_key_rotation_cooldown_keys Number of keys in cooldown');
+      lines.push('# TYPE dsh_key_rotation_cooldown_keys gauge');
+
+      lines.push('# HELP dsh_key_rotation_paused_keys Number of paused keys');
+      lines.push('# TYPE dsh_key_rotation_paused_keys gauge');
+
+      lines.push('# HELP dsh_key_rotation_revoked_keys Number of permanently revoked keys');
+      lines.push('# TYPE dsh_key_rotation_revoked_keys gauge');
+
+      lines.push('# HELP dsh_key_rotation_expired_keys Number of expired keys');
+      lines.push('# TYPE dsh_key_rotation_expired_keys gauge');
+
+      lines.push('# HELP dsh_key_rotation_proactive_switches_total Number of proactive rate-limit switches');
+      lines.push('# TYPE dsh_key_rotation_proactive_switches_total counter');
+
+      lines.push('# HELP dsh_key_rotation_cascade_switches_total Number of multi-vendor cascade switches');
+      lines.push('# TYPE dsh_key_rotation_cascade_switches_total counter');
+
+      for (const pool of pools) {
+        const p = pool.base;
+        const st = poolState.get(p);
+        let active = 0, cooling = 0, paused = 0, revoked = 0, expired = 0;
+        for (const r of (pool.refs ?? [])) {
+          if (isKeyRevoked(pool, r)) { revoked++; continue; }
+          if (isKeyPaused(pool, r)) { paused++; continue; }
+          if (pool.expiresAt?.[r] !== undefined && now >= pool.expiresAt[r]) { expired++; continue; }
+          const fu = st?.failedUntil?.get(r) ?? 0;
+          if (fu > now) cooling++;
+          else active++;
+        }
+        lines.push(`dsh_key_rotation_active_keys{provider="${p}"} ${active}`);
+        lines.push(`dsh_key_rotation_cooldown_keys{provider="${p}"} ${cooling}`);
+        lines.push(`dsh_key_rotation_paused_keys{provider="${p}"} ${paused}`);
+        lines.push(`dsh_key_rotation_revoked_keys{provider="${p}"} ${revoked}`);
+        lines.push(`dsh_key_rotation_expired_keys{provider="${p}"} ${expired}`);
+        lines.push(`dsh_key_rotation_proactive_switches_total{provider="${p}"} ${st?.proactiveSwitches ?? 0}`);
+        lines.push(`dsh_key_rotation_cascade_switches_total{provider="${p}"} ${st?.cascadeCount ?? 0}`);
+      }
+
+      res.statusCode = 200;
+      res.setHeader('content-type', 'text/plain; version=0.0.4; charset=utf-8');
+      res.end(lines.join('\n') + '\n');
+    }
+  }), 'dsh-key-rotation: metrics');
+}

package/lib/ops-paths.js

@@ -9,3 +9,4 @@ export const USAGE_PATH = '/dsh-key-rotation/usage';
 export const TEST_PATH = '/dsh-key-rotation/test';
 export const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
 export const WEBHOOK_ACTION_PATH = '/dsh-key-rotation/webhook-action';
+export const METRICS_PATH = '/dsh-key-rotation/metrics';

package/lib/ops-status.js

@@ -1,7 +1,10 @@
+import { decryptSecret } from './crypto-storage.js';
 // lib/ops-status.js — status + health operational routes (#312 split from routes-ops.js).
 import { json } from './http-bridge.js';
 import {
   isTrustedBridgeRequest,
+  isKeyPaused,
+  isKeyRevoked,
   keyTail,
   envValue,
   computeHealthScore,
@@ -106,6 +109,8 @@ ctx.effect(() => ctx.webServer.register({
           expiresAt: pool.expiresAt?.[ref] ?? null,
           expired: pool.expiresAt?.[ref] !== undefined && now >= pool.expiresAt[ref],
           broken: pool.state.brokenUntil?.has(ref) ?? false,
+          paused: isKeyPaused(pool, ref),
+          revoked: isKeyRevoked(pool, ref),
         });
       }
       providers.push({

package/lib/pool-builder.js

@@ -16,6 +16,7 @@ export function buildPoolItem({
   keys,
   weights,
   paused,
+  revoked,
   poolCooldown,
   poolMax,
   expiresAt,
@@ -51,6 +52,12 @@ export function buildPoolItem({
       if (paused[indexed[i].index] === true) parsedPaused.add(refs[i]);
     }
   }
+  const parsedRevoked = new Set();
+  if (Array.isArray(revoked)) {
+    for (let i = 0; i < refs.length; i++) {
+      if (revoked[indexed[i].index] === true) parsedRevoked.add(refs[i]);
+    }
+  }
   const weightsMap = {};
   for (let i = 0; i < refs.length; i++) {
     weightsMap[refs[i]] = typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1;
@@ -62,6 +69,7 @@ export function buildPoolItem({
     weightsMap,
     weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs,
     pausedRefs: parsedPaused,
+    revokedRefs: parsedRevoked,
     state: makeState(base),
     cooldownMs: poolCooldown,
     maxCooldownMs: poolMax,

package/lib/pool.js

@@ -1,3 +1,4 @@
+import { decryptSecret } from './crypto-storage.js';
 import { bestEffort } from './best-effort.js';
 // lib/pool.js — pure pool arithmetic and selection logic for dsh-key-rotation.
 // Isolated from cordis/dsh runtime so unit tests run in vanilla Node.js.
@@ -11,8 +12,9 @@ export const KEY_TAIL_PLACEHOLDER = '***';
 /** Mask key to show only the last KEY_TAIL_CHARS; returns placeholder for short keys (#355). */
 export function keyTail(key) {
   if (typeof key !== 'string' || key.length === 0) return '';
-  if (key.length <= KEY_TAIL_CHARS) return KEY_TAIL_PLACEHOLDER;
-  return key.slice(-KEY_TAIL_CHARS);
+  const plain = decryptSecret(key);
+  if (plain.length <= KEY_TAIL_CHARS) return KEY_TAIL_PLACEHOLDER;
+  return plain.slice(-KEY_TAIL_CHARS);
 }
 
 /** Check if an IP address is a loopback address. */
@@ -290,9 +292,10 @@ export function parseDurationToSeconds(v) {
 /** Check if a key reference is in maintenance/paused mode. */
 export function isKeyPaused(pool, ref) {
   if (!pool || !ref) return false;
-  if (pool.pausedRefs instanceof Set) return pool.pausedRefs.has(ref);
-  if (Array.isArray(pool.pausedRefs)) return pool.pausedRefs.includes(ref);
-  if (pool.state && pool.state.pausedRefs instanceof Set) return pool.state.pausedRefs.has(ref);
+  if (pool.pausedRefs instanceof Set && pool.pausedRefs.has(ref)) return true;
+  if (Array.isArray(pool.pausedRefs) && pool.pausedRefs.includes(ref)) return true;
+  if (pool.state && pool.state.pausedRefs instanceof Set && pool.state.pausedRefs.has(ref)) return true;
+  if (pool.state && Array.isArray(pool.state.pausedRefs) && pool.state.pausedRefs.includes(ref)) return true;
   return false;
 }
 
@@ -529,3 +532,13 @@ export function resetCircuitForProvider(circuitBreaker, provider) {
   });
   return ok === true;
 }
+
+/** Check if a key reference is permanently revoked due to auth/401 failure (#369). */
+export function isKeyRevoked(pool, ref) {
+  if (!pool || !ref) return false;
+  if (pool.revokedRefs instanceof Set && pool.revokedRefs.has(ref)) return true;
+  if (Array.isArray(pool.revokedRefs) && pool.revokedRefs.includes(ref)) return true;
+  if (pool.state && pool.state.revokedRefs instanceof Set && pool.state.revokedRefs.has(ref)) return true;
+  if (pool.state && Array.isArray(pool.state.revokedRefs) && pool.state.revokedRefs.includes(ref)) return true;
+  return false;
+}

package/lib/rotate.js

@@ -10,6 +10,7 @@ import {
   extractRateLimit,
   isRateLimited,
   isKeyPaused,
+  isKeyRevoked,
   formatExhaustionMessage,
   sortAttemptList,
 } from './pool.js';
@@ -53,7 +54,7 @@ export function createRotate(deps) {
       const reqStore = { pool, pickedRef: undefined, startMs: now() };
       setRotateStartMs(reqStore.startMs);
       let attemptList = (pool.weightedRefs ?? pool.refs).slice();
-      attemptList = attemptList.filter((r) => !isKeyPaused(pool, r));
+      attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r));
       const strategy = pool.routingStrategy ?? runtime0.routingStrategy ?? 'round-robin';
       if (strategy !== 'round-robin') {
         attemptList = sortAttemptList(attemptList, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap });
@@ -109,13 +110,14 @@ export function createRotate(deps) {
         if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
         if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
         const _cStr = String(errCode ?? '');
-        if (_cStr === 'AUTH' || /auth/i.test(errMsg)) {
+        if (_cStr === 'AUTH' || /auth|unauthorized|invalid[_\s-]?api[_\s-]?key|unauthenticated/i.test(errMsg)) {
           const _c2 = (pool.state.authFailCounts.get(targetRef) ?? 0) + 1;
           pool.state.authFailCounts.set(targetRef, _c2);
-          if (_c2 >= 3) {
-            pool.state.brokenUntil.set(targetRef, now() + 86400000 * 30);
-            pool.state.failedUntil.set(targetRef, now() + 86400000 * 30);
-          }
+          if (!pool.state.revokedRefs) pool.state.revokedRefs = new Set();
+          pool.state.revokedRefs.add(targetRef);
+          pool.state.failedUntil.set(targetRef, Infinity);
+          pushEvent(pool, targetRef, 'KEY_REVOKED', 0);
+          logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently REVOKED on auth failure`);
         } else {
           pool.state.authFailCounts.delete(targetRef);
         }

package/lib/routes-ops.js

@@ -1,3 +1,4 @@
+import { registerMetricsRoutes } from './ops-metrics.js';
 // lib/routes-ops.js — operational HTTP routes for dsh-key-rotation (#253/#312).
 // Thin orchestrator: each route family lives in its own lib/ops-*.js module.
 // Registration is injected with the live apply() dependencies.
@@ -18,4 +19,5 @@ export function registerOpsRoutes(ctx, deps) {
   registerKeyRoutes(ctx, deps);
   registerTestRoutes(ctx, deps);
   registerWebhookActionRoute(ctx, deps);
+  registerMetricsRoutes(ctx, deps);
 }

package/lib/webhook.js

@@ -12,6 +12,8 @@ export function detectPlatform(url) {
   if (u.includes('api.telegram.org')) return 'telegram';
   if (u.includes('discord.com/api/webhooks')) return 'discord';
   if (u.includes('hooks.slack.com')) return 'slack';
+  if (u.includes('ntfy.sh') || u.includes('/ntfy')) return 'ntfy';
+  if (u.includes('gotify') || u.includes('/message')) return 'gotify';
   return 'generic';
 }
 
@@ -66,6 +68,40 @@ export function formatInteractive(url, payload, actionToken) {
       ],
     };
   }
+  if (platform === 'gotify') {
+    return {
+      title,
+      message: text,
+      priority: 5,
+      extras: {
+        'client::notification': {
+          click: { url: actions[0]?.url || '' },
+        },
+      },
+    };
+  }
+  if (platform === 'ntfy') {
+    let topic = 'alerts';
+    try {
+      const u = new URL(url);
+      const parts = u.pathname.split('/').filter(Boolean);
+      if (parts.length > 0) topic = parts[parts.length - 1];
+    } catch (_) {
+      topic = String(url).split('?')[0].split('/').filter(Boolean).pop() || 'alerts';
+    }
+    return {
+      topic,
+      title,
+      message: text,
+      priority: 4,
+      tags: ['warning', 'key-rotation'],
+      actions: actions.map((a) => ({
+        action: 'view',
+        label: a.label,
+        url: a.url || '',
+      })),
+    };
+  }
   // generic JSON webhook: actions as plain data, receiver decides
   return { ...payload, actions };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.27",
+  "version": "0.8.29",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [