@goodandready/dsh-key-rotation 0.8.25 → 0.8.28

package/CHANGELOG.md

@@ -3,6 +3,42 @@
 All notable changes to `@goodandready/dsh-key-rotation` are documented here.
 User-facing feature notes also appear in README (en is source of truth).
 
+## 0.8.28 - 2026-09-27
+
+### Security & Key Lifecycle (Block 3)
+- **Permanent Key Revocation (#369)**:
+  - Added `revokedRefs` tracking in pool state and configuration. Keys returning permanent auth failures (HTTP 401 / `AUTH_FAILED`) are permanently revoked and excluded from rotation and cascade attempts.
+  - Added visual revoked status badges and tokens in WebUI (`lib/client.js`).
+- **Key Encryption at Rest (#370)**:
+  - Added `lib/crypto-storage.js` implementing authenticated AES-256-GCM encryption with 12-byte IV and 16-byte auth tag.
+  - Transparently uses environment variable `DSH_KEY_SECRET` when configured, with safe plaintext fallback when unset.
+- **Key Expiration Tracker (#371)**:
+  - Added automated expiration date tracking (`expiresAt`), horizon detection (`expiringSoon`), 24-hour notification deduplication (`shouldNotifyDaily`), and automated exclusion of expired keys from active routing.
+
+### Monitoring & Integrations (Block 4)
+- **Multi-Platform Webhook Integrations (#372)**:
+  - Extended `lib/webhook.js` with native detection and formatting for Gotify (`/message` with JSON priority/extras) and Ntfy (`/alerts` with action buttons).
+- **Prometheus Metrics Exporter (#373)**:
+  - Added `/dsh-key-rotation/metrics` route (`lib/ops-metrics.js`) exposing OpenMetrics / Prometheus standard text format with gauges for pool totals, active/cooldown/paused/revoked keys, and counters for proactive switches and cascades. Guarded by loopback origin fail-closed security.
+
+## 0.8.27 - 2026-09-27
+
+### Smart Routing (Block 1)
+- **Proactive Rate-Limit Cooldown (#363)**:
+  - Proactive key cooldown based on upstream rate limit reset headers (`x-ratelimit-reset`, `x-ratelimit-reset-requests`, `x-ratelimit-reset-tokens`) supporting time deltas and ISO timestamps.
+- **Cross-Provider Cascading (#364)**:
+  - Cascade routing across providers on pool exhaustion with `modelMapping` and max cascade depth guard.
+- **Adaptive Concurrency Normalization (#365)**:
+  - Weighted least-loaded concurrency balancing (`inflight / weight`).
+
+### WebUI Suite (Block 2)
+- **Bulk Key Import (#366)**:
+  - WebUI modal for bulk key import supporting line-by-line, `.env`, and CSV formats.
+- **Key Maintenance Toggle (#367)**:
+  - Live key pause/resume toggle (`isPaused`) in WebUI.
+- **Traffic Sparklines (#368)**:
+  - Native SVG Sparkline component for request and latency visual trends.
+
 ## 0.8.25 - 2026-09-27
 
 ### Fixed & Hardened

package/lib/cascade.js

@@ -1,9 +1,16 @@
-// cascade.js — cross-provider failover cascade (issue #194).
-// ponytail: minimal — pick fallback provider from a RegionMap-like config.
+// cascade.js — cross-provider failover cascade (issues #194, #364).
+// ponytail: minimal — pick fallback provider with model mapping from config.
 
-export const CASCADE_MAX_DEPTH = 1;
+export const CASCADE_MAX_DEPTH = 3;
 
-export function pickCascadeFallback(provider, cfg, pools) {
+function isKeyPaused(pool, ref) {
+  if (!pool || !ref) return false;
+  if (pool.pausedRefs instanceof Set) return pool.pausedRefs.has(ref);
+  if (Array.isArray(pool.pausedRefs)) return pool.pausedRefs.includes(ref);
+  return false;
+}
+
+export function pickCascadeFallback(provider, cfg, pools, currentModel) {
   const list = Array.isArray(cfg && cfg.cascade) ? cfg.cascade : [];
   for (const entry of list) {
     const fb = typeof entry === 'string' ? { provider: entry } : entry;
@@ -13,6 +20,7 @@ export function pickCascadeFallback(provider, cfg, pools) {
     const now = Date.now();
     let healthy = 0;
     for (const ref of pool.refs) {
+      if (isKeyPaused(pool, ref) || (pool.revokedRefs && pool.revokedRefs.has(ref)) || (pool.state?.revokedRefs && pool.state.revokedRefs.has(ref))) continue;
       const failedUntil = (pool.state && pool.state.failedUntil && pool.state.failedUntil.get(ref)) || 0;
       if (failedUntil > now) continue;
       const exp = pool.expiresAt ? pool.expiresAt[ref] : undefined;
@@ -20,8 +28,8 @@ export function pickCascadeFallback(provider, cfg, pools) {
       healthy += 1;
     }
     if (healthy === 0) continue;
-    return { provider: fb.provider, pool, model: fb.model || null };
+    const mappedModel = (fb.modelMapping && currentModel && fb.modelMapping[currentModel]) || fb.model || currentModel || null;
+    return { provider: fb.provider, pool, model: mappedModel };
   }
   return null;
 }
-

package/lib/client.js

@@ -78,6 +78,17 @@ window.__ModuleLoader__.load({
       addKey: '+ Add key',
       addKeyTitle: 'Add API key',
       noProviders: 'No providers registered with DSH — nothing to pick from yet.',
+      bulkImport: 'Bulk Import',
+      bulkImportDesc: 'Paste API keys line-by-line, as .env (KEY=value), or CSV (KEY, weight):',
+      bulkImportBtn: 'Import Keys',
+      bulkImportCancel: 'Cancel',
+      bulkImportSuccess: 'Imported {n} keys',
+      keyPaused: 'Paused',
+      keyRevoked: 'Revoked (401)',
+      pauseKey: 'Pause key from rotation',
+      resumeKey: 'Resume key rotation',
+      sparklineTraffic: 'Traffic Trend',
+      sparklineLatency: 'Latency Trend',
       addProvider: '+ Add provider',
       desc: 'Per-provider API key rotation. For each provider, list its API keys (env names, stored in DSH credentials). The plugin routes a model through that provider\u2019s keys in order and switches to the next on a quota/rate-limit failure.',
       cooldown: 'Cooldown after failure (ms)',
@@ -223,6 +234,17 @@ window.__ModuleLoader__.load({
       addKey: '+ 添加密钥',
       addKeyTitle: '添加 API 密钥',
       noProviders: 'DSH 尚未注册任何提供商 — 暂无可选项目。',
+      bulkImport: '批量导入',
+      bulkImportDesc: '按行粘贴 API 密钥，支持 .env (KEY=value) 或 CSV (KEY, 权重) 格式：',
+      bulkImportBtn: '导入密钥',
+      bulkImportCancel: '取消',
+      bulkImportSuccess: '已导入 {n} 个密钥',
+      keyPaused: '已暂停',
+      keyRevoked: '已撤销 (401)',
+      pauseKey: '暂停此密钥轮换',
+      resumeKey: '恢复此密钥轮换',
+      sparklineTraffic: '流量趋势',
+      sparklineLatency: '延迟趋势',
       addProvider: '+ 添加提供商',
       desc: '按提供商的 API 密钥轮换。为每个提供商列出其 API 密钥（环境变量名，存储在 DSH 凭据中）。插件按顺序通过提供商的密钥路由模型，并在配额/速率限制失败时切换到下一个。',
       cooldown: '故障后冷却时间 (毫秒)',
@@ -590,6 +612,74 @@ window.__ModuleLoader__.load({
       };
     }
 
+
+    function Sparkline({ points = [], width = 120, height = 22, stroke = 'var(--dsw-alias-state-brand-primary, #3b82f6)' }) {
+      if (!Array.isArray(points) || points.length < 2) {
+        return h('svg', { width, height, viewBox: `0 0 ${width} ${height}`, style: { opacity: 0.3 } },
+          h('line', { x1: 0, y1: height / 2, x2: width, y2: height / 2, stroke, strokeWidth: 1, strokeDasharray: '3 3' })
+        );
+      }
+      const min = Math.min(...points);
+      const max = Math.max(...points);
+      const range = max - min || 1;
+      const step = width / (points.length - 1);
+      const coords = points.map((p, i) => {
+        const x = Math.round(i * step * 10) / 10;
+        const y = Math.round((height - ((p - min) / range) * (height - 4) - 2) * 10) / 10;
+        return `${x},${y}`;
+      }).join(' ');
+
+      return h('svg', {
+        width,
+        height,
+        viewBox: `0 0 ${width} ${height}`,
+        style: { display: 'inline-block', verticalAlign: 'middle', overflow: 'visible' },
+      }, h('polyline', {
+        points: coords,
+        fill: 'none',
+        stroke,
+        strokeWidth: 2,
+        strokeLinecap: 'round',
+        strokeLinejoin: 'round',
+      }));
+    }
+
+    function parseBulkKeys(text) {
+      if (!text || typeof text !== 'string') return [];
+      const lines = text.split(/\r?\n/);
+      const result = [];
+      const seen = new Set();
+      for (let raw of lines) {
+        const line = raw.trim();
+        if (!line || line.startsWith('#') || line.startsWith('//')) continue;
+        const envMatch = line.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);
+        if (envMatch) {
+          const ref = envMatch[1];
+          if (!seen.has(ref)) {
+            seen.add(ref);
+            let weight = 1;
+            const rest = envMatch[2].trim().replace(/^['"]|['"]$/g, '');
+            const wMatch = rest.match(/,\s*(?:weight\s*=\s*)?([0-9]+)$/i);
+            if (wMatch) weight = Math.max(1, parseInt(wMatch[1], 10) || 1);
+            result.push({ ref, weight });
+          }
+          continue;
+        }
+        const parts = line.split(',');
+        const ref = parts[0].trim();
+        if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(ref) && !seen.has(ref)) {
+          seen.add(ref);
+          let weight = 1;
+          if (parts[1]) {
+            const wm = parts[1].trim().match(/(?:weight\s*=\s*)?([0-9]+)/i);
+            if (wm) weight = Math.max(1, parseInt(wm[1], 10) || 1);
+          }
+          result.push({ ref, weight });
+        }
+      }
+      return result;
+    }
+
     function UpdaterSection({ t }) {
       const [state, setState] = React.useState({ phase: 'idle', current: '', latest: '', message: '' });
       const load = React.useCallback(async () => {
@@ -883,6 +973,47 @@ window.__ModuleLoader__.load({
       const [selected, setSelected] = React.useState(new Set());
       const [bulkCooldown, setBulkCooldown] = React.useState('');
       const [undo, setUndo] = React.useState(null);
+      const [bulkOpen, setBulkOpen] = React.useState({});
+      const [bulkText, setBulkText] = React.useState({});
+
+      const togglePauseKey = (pIndex, kIndex) => setField((cur) => {
+        const providers = [...(cur.providers ?? [])];
+        const ent = { ...providers[pIndex] };
+        const paused = [...(ent.paused ?? [])];
+        paused[kIndex] = !paused[kIndex];
+        ent.paused = paused;
+        providers[pIndex] = ent;
+        return { ...cur, providers };
+      });
+
+      const applyBulkImport = (pIndex) => {
+        const text = bulkText[pIndex] || '';
+        const parsed = parseBulkKeys(text);
+        if (!parsed.length) return;
+        setField((cur) => {
+          const providers = [...(cur.providers ?? [])];
+          const ent = { ...providers[pIndex] };
+          const existingKeys = new Set(ent.keys ?? []);
+          const nextKeys = [...(ent.keys ?? [])];
+          const nextWeights = [...(ent.weights ?? [])];
+          const nextPaused = [...(ent.paused ?? [])];
+          for (const item of parsed) {
+            if (!existingKeys.has(item.ref)) {
+              existingKeys.add(item.ref);
+              nextKeys.push(item.ref);
+              nextWeights.push(item.weight);
+              nextPaused.push(false);
+            }
+          }
+          ent.keys = nextKeys;
+          ent.weights = nextWeights;
+          ent.paused = nextPaused;
+          providers[pIndex] = ent;
+          return { ...cur, providers };
+        });
+        setBulkText((cur) => ({ ...cur, [pIndex]: '' }));
+        setBulkOpen((cur) => ({ ...cur, [pIndex]: false }));
+      };
       const undoTimer = React.useRef(null);
       React.useEffect(() => () => { if (undoTimer.current) clearTimeout(undoTimer.current); }, []);
       // #289 a11y: confirm dialog focus trap / Escape / restore (hooks above returns)
@@ -1451,7 +1582,8 @@ window.__ModuleLoader__.load({
 
       // Key state dot: color and label read at a glance; "key not found"
       // catches an env-name typo that would otherwise stay silent.
-      const keyStatus = (providerId, ref) => {
+      const keyStatus = (providerId, ref, isPaused = false) => {
+        if (isPaused) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyPaused') };
         const hit = keyInfo(providerId, ref);
         if (!hit) return null;
         if (hit.expired) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyExpired') };
@@ -1460,6 +1592,7 @@ window.__ModuleLoader__.load({
           const warnDays = Number(val?.expiryWarnDays) || 7; // #207: configurable horizon
           if (days <= warnDays) return { color: 'var(--dsw-alias-state-warning-primary)', text: t('keyExpiringSoon').replace('{n}', String(days)) };
         }
+        if (hit.broken || hit.revoked || hit.status === 'revoked') return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyRevoked') };
         if (hit.broken) return { color: 'var(--dsw-alias-state-error-primary)', text: t('brokenKey') };
         if (!hit.present) return { color: 'var(--dsw-alias-state-error-primary)', text: t('keyMissing') };
         if (hit.cooldownMsLeft > 0 && !optimisticReset[ref]) {
@@ -1484,10 +1617,12 @@ window.__ModuleLoader__.load({
 
         const keys = entry.keys ?? [];
         const entryWeights = entry.weights ?? [];
+        const entryPaused = entry.paused ?? [];
 
         let readyCount = 0, cooldownCount = 0, errorCount = 0;
-        for (const k of keys) {
-          const st = keyStatus(entry.provider, k);
+        for (let idx = 0; idx < keys.length; idx++) {
+          const k = keys[idx];
+          const st = keyStatus(entry.provider, k, Boolean(entryPaused[idx]));
           const isCool = (st && (st.text === t('brokenKey') || (st.text && st.text.includes(t('keyCooling').slice(0, 4))))) && !optimisticReset[k];
           const tr = testResult[k];
           if (tr && !tr.ok) errorCount++;
@@ -1495,9 +1630,9 @@ window.__ModuleLoader__.load({
           else readyCount++;
         }
 
-        const filteredIndices = keys.map((k, idx) => ({ key: k, kIndex: idx })).filter(({ key: k }) => {
+        const filteredIndices = keys.map((k, idx) => ({ key: k, kIndex: idx })).filter(({ key: k, kIndex: idx }) => {
           if (statusFilter === 'all') return true;
-          const st = keyStatus(entry.provider, k);
+          const st = keyStatus(entry.provider, k, Boolean(entryPaused[idx]));
           const isCool = (st && (st.text === t('brokenKey') || (st.text && st.text.includes(t('keyCooling').slice(0, 4))))) && !optimisticReset[k];
           const tr = testResult[k];
           if (statusFilter === 'ready') return !isCool;
@@ -1514,7 +1649,8 @@ window.__ModuleLoader__.load({
         ) : null;
 
         const keyRows = filteredIndices.map(({ key, kIndex }) => {
-          const st = keyStatus(entry.provider, key);
+          const isP = Boolean(entryPaused[kIndex]);
+          const st = keyStatus(entry.provider, key, isP);
           const info = keyInfo(entry.provider, key);
           const rowKey = entry.provider + '/' + key;
           const typed = secretDraft[rowKey];
@@ -1570,6 +1706,14 @@ window.__ModuleLoader__.load({
             title: t('weightHint'),
             onChange: (e) => setKeyWeight(pIndex, kIndex, e.target.value),
             style: { width: '52px', padding: '2px 6px', fontSize: '12px' } }));
+          meta.push(h('button', {
+            key: 'pause-btn',
+            type: 'button',
+            className: 'krot-btn' + (isP ? ' krot-pill-warn' : ''),
+            title: isP ? t('resumeKey') : t('pauseKey'),
+            style: { padding: '2px 6px', fontSize: '11px' },
+            onClick: () => togglePauseKey(pIndex, kIndex),
+          }, isP ? '⏸ ' + t('keyPaused') : '▶'));
           // #210: RPM capacity indicator (only when rpmLimit is active)
           if (info && info.rpm) meta.push(h('span', { key: 'rpm', className: 'krot-tail',
             title: t('rpmTitle').replace('{u}', String(info.rpm.used)).replace('{r}', String(info.rpm.remaining)),
@@ -1701,6 +1845,10 @@ window.__ModuleLoader__.load({
                   h('span', { style: { color: 'var(--dsw-alias-label-secondary)' } }, t('keyLabel').replace('{n}', String(x.index + 1)) + ': ' + x.usage + ' (' + pct + '%)')
                 );
               })
+            ) : null,
+            total > 0 && keyUsages.length > 1 ? h('div', { style: { marginTop: '8px', display: 'flex', alignItems: 'center', gap: '8px' } },
+              h('span', { style: { fontSize: '11px', color: 'var(--dsw-alias-label-secondary)' } }, '📈 ' + t('sparklineTraffic') + ':'),
+              h(Sparkline, { points: keyUsages.map(x => x.usage), width: 140, height: 20 })
             ) : null
           );
         })();
@@ -1773,6 +1921,22 @@ window.__ModuleLoader__.load({
           h(UpdaterSection, { t }),
       h('div', { className: 'krot-foot' },
             btn(t('addKey'), () => addKey(pIndex), { title: t('addKeyTitle'), disabled: !entry.provider.trim() }),
+            btn('📋 ' + t('bulkImport'), () => setBulkOpen(cur => ({ ...cur, [pIndex]: !cur[pIndex] })), { disabled: !entry.provider.trim(), title: t('bulkImport') }),
+            bulkOpen[pIndex] ? h('div', { className: 'krot-bulk-import-box', style: { width: '100%', marginTop: '8px', padding: '8px', background: 'var(--dsw-alias-bg-layer-2)', borderRadius: '6px' } },
+              h('p', { style: { margin: '0 0 6px 0', fontSize: '12px', color: 'var(--dsw-alias-label-secondary)' } }, t('bulkImportDesc')),
+              h('textarea', {
+                className: 'krot-in',
+                rows: 4,
+                style: { width: '100%', fontFamily: 'monospace', fontSize: '11px', resize: 'vertical' },
+                placeholder: 'KEY_1\nKEY_2, weight=5\nOPENAI_KEY=sk-...',
+                value: bulkText[pIndex] || '',
+                onChange: (e) => setBulkText(cur => ({ ...cur, [pIndex]: e.target.value })),
+              }),
+              h('div', { style: { display: 'flex', gap: '8px', marginTop: '6px' } },
+                btn(t('bulkImportBtn'), () => applyBulkImport(pIndex), { primary: true }),
+                btn(t('bulkImportCancel'), () => setBulkOpen(cur => ({ ...cur, [pIndex]: false })))
+              )
+            ) : null,
             switchesLine,
             budgetLine,
             sloLine,

package/lib/concurrency.js

@@ -55,19 +55,21 @@ export class ConcurrencyTracker {
     return out;
   }
 
-  pickLeastLoaded(candidates, now) {
+  pickLeastLoaded(candidates, now, weights) {
     now = now || Date.now();
     if (!Array.isArray(candidates) || candidates.length === 0) return null;
     let best = null;
-    let bestCount = Infinity;
+    let bestScore = Infinity;
     for (const ref of candidates) {
       const e = this._inFlight.get(ref);
       let count = e ? e.count : 0;
       if (e && now - e.lastAcquired > this._staleMs) count = 0;
       if (this.isEnabled() && count >= this._limit) continue;
-      if (count < bestCount) {
+      const w = (weights && Number(weights[ref]) > 0) ? Number(weights[ref]) : 1;
+      const score = count / w;
+      if (score < bestScore) {
         best = ref;
-        bestCount = count;
+        bestScore = score;
       }
     }
     return best;

package/lib/crypto-storage.js

@@ -0,0 +1,49 @@
+// lib/crypto-storage.js — AES-256-GCM encryption for credentials at rest (#370)
+import crypto from 'node:crypto';
+
+const PREFIX = 'enc:v1:';
+
+function getDerivedKey(secret) {
+  return crypto.createHash('sha256').update(String(secret)).digest();
+}
+
+/**
+ * Encrypt plaintext using AES-256-GCM if secretKey is provided.
+ * If secretKey is not set, returns plaintext as-is.
+ */
+export function encryptSecret(plaintext, secretKey = process.env.DSH_KEY_SECRET) {
+  if (!secretKey || !plaintext || typeof plaintext !== 'string') return plaintext;
+  if (plaintext.startsWith(PREFIX)) return plaintext; // already encrypted
+  const key = getDerivedKey(secretKey);
+  const iv = crypto.randomBytes(12);
+  const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
+  let ciphertext = cipher.update(plaintext, 'utf8', 'hex');
+  ciphertext += cipher.final('hex');
+  const authTag = cipher.getAuthTag().toString('hex');
+  return `${PREFIX}${iv.toString('hex')}:${authTag}:${ciphertext}`;
+}
+
+/**
+ * Decrypt ciphertext using AES-256-GCM if prefixed with enc:v1:.
+ * If not encrypted, or if secretKey is missing/incorrect, returns safe fallback.
+ */
+export function decryptSecret(ciphertext, secretKey = process.env.DSH_KEY_SECRET) {
+  if (!ciphertext || typeof ciphertext !== 'string') return ciphertext;
+  if (!ciphertext.startsWith(PREFIX)) return ciphertext; // plaintext fallback
+  if (!secretKey) return ciphertext;
+  try {
+    const parts = ciphertext.slice(PREFIX.length).split(':');
+    if (parts.length !== 3) return ciphertext;
+    const [ivHex, tagHex, dataHex] = parts;
+    const key = getDerivedKey(secretKey);
+    const iv = Buffer.from(ivHex, 'hex');
+    const authTag = Buffer.from(tagHex, 'hex');
+    const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
+    decipher.setAuthTag(authTag);
+    let decrypted = decipher.update(dataHex, 'hex', 'utf8');
+    decrypted += decipher.final('utf8');
+    return decrypted;
+  } catch {
+    return ciphertext;
+  }
+}

package/lib/http-bridge.js

@@ -1,6 +1,7 @@
 // lib/http-bridge.js — shared HTTP helpers for dsh-key-rotation routes.
 import { findSecrets } from './keycheck.js';
 import { isTrustedBridgeRequest } from './pool.js';
+export { isTrustedBridgeRequest };
 import { mutateSettings, plainObject, requireRevision, sectionOps, validateOps } from './settings-write.js';
 
 export const NS = 'dsh-key-rotation';

package/lib/index.js

@@ -263,13 +263,13 @@ export function apply(ctx, config = {}) {
       return initializePoolState(st);
     };
 
-    const buildPool = (base, keys, weights, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard) =>
-      buildPoolItem({ base, keys, weights, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, rpmLimit, makeState });
+    const buildPool = (base, keys, weights, paused, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard) =>
+      buildPoolItem({ base, keys, weights, paused, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, rpmLimit, makeState });
 
     for (const p of cfg.providers ?? []) {
       const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
       const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
-      const pool = buildPool(p.provider, p.keys, p.weights, poolCooldown, poolMax, p.expiresAt, p.routingStrategy, p.proactiveRateLimitGuard);
+      const pool = buildPool(p.provider, p.keys, p.weights, p.paused, poolCooldown, poolMax, p.expiresAt, p.routingStrategy, p.proactiveRateLimitGuard);
       if (pool) {
         for (const ref of pool.refs) poolByRef.set(ref, pool);
         for (let i = 1; i < pool.refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
@@ -277,7 +277,7 @@ export function apply(ctx, config = {}) {
       const models = p.models ?? {};
       const byModel = new Map();
       for (const [model, mp] of Object.entries(models)) {
-        const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, poolCooldown, poolMax, undefined, p.routingStrategy, p.proactiveRateLimitGuard);
+        const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, mp.paused, poolCooldown, poolMax, undefined, p.routingStrategy, p.proactiveRateLimitGuard);
         if (mpool) {
           byModel.set(model, mpool);
           for (const ref of mpool.refs) poolByRef.set(ref, mpool);

package/lib/ops-metrics.js

@@ -0,0 +1,80 @@
+// lib/ops-metrics.js — Prometheus / OpenMetrics export route (#373).
+import { METRICS_PATH } from './ops-paths.js';
+import { isTrustedBridgeRequest, isKeyPaused, isKeyRevoked } from './pool.js';
+
+/**
+ * @param {object} ctx cordis context
+ * @param {object} deps live dependencies from apply()
+ */
+export function registerMetricsRoutes(ctx, deps) {
+  const { buildRuntime, poolState } = deps;
+
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: METRICS_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'GET') {
+        res.statusCode = 405;
+        res.setHeader('content-type', 'text/plain');
+        res.end('Method Not Allowed\n');
+        return;
+      }
+      if (!isTrustedBridgeRequest(req)) {
+        res.statusCode = 403;
+        res.setHeader('content-type', 'text/plain');
+        res.end('Forbidden: local only\n');
+        return;
+      }
+
+      const lines = [];
+      const runtime = buildRuntime();
+      const pools = runtime.pools ?? [];
+      const now = Date.now();
+
+      lines.push('# HELP dsh_key_rotation_pools_total Total configured provider pools');
+      lines.push('# TYPE dsh_key_rotation_pools_total gauge');
+      lines.push(`dsh_key_rotation_pools_total ${pools.length}`);
+
+      lines.push('# HELP dsh_key_rotation_active_keys Number of currently active and ready keys');
+      lines.push('# TYPE dsh_key_rotation_active_keys gauge');
+
+      lines.push('# HELP dsh_key_rotation_cooldown_keys Number of keys in cooldown');
+      lines.push('# TYPE dsh_key_rotation_cooldown_keys gauge');
+
+      lines.push('# HELP dsh_key_rotation_paused_keys Number of paused keys');
+      lines.push('# TYPE dsh_key_rotation_paused_keys gauge');
+
+      lines.push('# HELP dsh_key_rotation_revoked_keys Number of permanently revoked keys');
+      lines.push('# TYPE dsh_key_rotation_revoked_keys gauge');
+
+      lines.push('# HELP dsh_key_rotation_proactive_switches_total Number of proactive rate-limit switches');
+      lines.push('# TYPE dsh_key_rotation_proactive_switches_total counter');
+
+      lines.push('# HELP dsh_key_rotation_cascade_switches_total Number of multi-vendor cascade switches');
+      lines.push('# TYPE dsh_key_rotation_cascade_switches_total counter');
+
+      for (const pool of pools) {
+        const p = pool.base;
+        const st = poolState.get(p);
+        let active = 0, cooling = 0, paused = 0, revoked = 0;
+        for (const r of (pool.refs ?? [])) {
+          if (isKeyRevoked(pool, r)) { revoked++; continue; }
+          if (isKeyPaused(pool, r)) { paused++; continue; }
+          const fu = st?.failedUntil?.get(r) ?? 0;
+          if (fu > now) cooling++;
+          else active++;
+        }
+        lines.push(`dsh_key_rotation_active_keys{provider="${p}"} ${active}`);
+        lines.push(`dsh_key_rotation_cooldown_keys{provider="${p}"} ${cooling}`);
+        lines.push(`dsh_key_rotation_paused_keys{provider="${p}"} ${paused}`);
+        lines.push(`dsh_key_rotation_revoked_keys{provider="${p}"} ${revoked}`);
+        lines.push(`dsh_key_rotation_proactive_switches_total{provider="${p}"} ${st?.proactiveSwitches ?? 0}`);
+        lines.push(`dsh_key_rotation_cascade_switches_total{provider="${p}"} ${st?.cascadeCount ?? 0}`);
+      }
+
+      res.statusCode = 200;
+      res.setHeader('content-type', 'text/plain; version=0.0.4; charset=utf-8');
+      res.end(lines.join('\n') + '\n');
+    }
+  }), 'dsh-key-rotation: metrics');
+}

package/lib/ops-paths.js

@@ -9,3 +9,4 @@ export const USAGE_PATH = '/dsh-key-rotation/usage';
 export const TEST_PATH = '/dsh-key-rotation/test';
 export const SANDBOX_CACHE_PATH = '/dsh-key-rotation/sandbox-cache';
 export const WEBHOOK_ACTION_PATH = '/dsh-key-rotation/webhook-action';
+export const METRICS_PATH = '/dsh-key-rotation/metrics';

package/lib/pool-builder.js

@@ -15,6 +15,8 @@ export function buildPoolItem({
   base,
   keys,
   weights,
+  paused,
+  revoked,
   poolCooldown,
   poolMax,
   expiresAt,
@@ -44,11 +46,30 @@ export function buildPoolItem({
       if (exp !== undefined) parsedExpiry[refs[i]] = exp;
     }
   }
+  const parsedPaused = new Set();
+  if (Array.isArray(paused)) {
+    for (let i = 0; i < refs.length; i++) {
+      if (paused[indexed[i].index] === true) parsedPaused.add(refs[i]);
+    }
+  }
+  const parsedRevoked = new Set();
+  if (Array.isArray(revoked)) {
+    for (let i = 0; i < refs.length; i++) {
+      if (revoked[indexed[i].index] === true) parsedRevoked.add(refs[i]);
+    }
+  }
+  const weightsMap = {};
+  for (let i = 0; i < refs.length; i++) {
+    weightsMap[refs[i]] = typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1;
+  }
   return {
     base,
     refs,
     weights: refs.map((_, i) => (typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1)),
+    weightsMap,
     weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs,
+    pausedRefs: parsedPaused,
+    revokedRefs: parsedRevoked,
     state: makeState(base),
     cooldownMs: poolCooldown,
     maxCooldownMs: poolMax,

package/lib/pool.js

@@ -261,24 +261,52 @@ export function computeHealthScore(state) {
   return Math.max(0, Math.min(100, 100 - (switches * 5) - (exhaustions * 10) - (broken * 15)));
 }
 
+
+/** Parse duration strings (e.g. '1.5s', '500ms', '2m', '1h'), numbers, or ISO dates into seconds. */
+export function parseDurationToSeconds(v) {
+  if (typeof v === 'number') {
+    if (!Number.isFinite(v) || v < 0) return null;
+    return v;
+  }
+  if (!v || typeof v !== 'string') return null;
+  const s = v.trim();
+  const m = s.match(/^([0-9]+(?:\.[0-9]+)?)\s*(ms|s|m|h|d)?$/i);
+  if (m) {
+    const val = parseFloat(m[1]);
+    const unit = (m[2] || 's').toLowerCase();
+    if (unit === 'ms') return val / 1000;
+    if (unit === 's') return val;
+    if (unit === 'm') return val * 60;
+    if (unit === 'h') return val * 3600;
+    if (unit === 'd') return val * 86400;
+  }
+  const dt = Date.parse(s);
+  if (!Number.isNaN(dt)) {
+    return Math.max(0, (dt - Date.now()) / 1000);
+  }
+  return null;
+}
+
+/** Check if a key reference is in maintenance/paused mode. */
+export function isKeyPaused(pool, ref) {
+  if (!pool || !ref) return false;
+  if (pool.pausedRefs instanceof Set) return pool.pausedRefs.has(ref);
+  if (Array.isArray(pool.pausedRefs)) return pool.pausedRefs.includes(ref);
+  if (pool.state && pool.state.pausedRefs instanceof Set) return pool.state.pausedRefs.has(ref);
+  return false;
+}
+
 /** Extract rate-limit info from an object that may carry response headers. */
 export function extractRateLimit(headers) {
   if (!headers || typeof headers !== 'object') return null;
-  let remaining, limit, reset, retryAfter;
+  let remaining, limit, reset, resetMs, retryAfter;
   for (const k in headers) {
     const lower = k.toLowerCase();
     const v = headers[k];
     if (v == null) continue;
     if (lower === 'retry-after') {
-      const num = Number(v);
-      if (Number.isFinite(num) && num >= 0) {
-        retryAfter = Math.ceil(num);
-      } else {
-        const parsed = Date.parse(v);
-        if (!Number.isNaN(parsed)) {
-          retryAfter = Math.max(0, Math.ceil((parsed - Date.now()) / 1000));
-        }
-      }
+      const dur = parseDurationToSeconds(v);
+      if (dur !== null) retryAfter = Math.ceil(dur);
     } else if (
       lower === 'x-ratelimit-remaining' ||
       lower === 'x-ratelimit-remaining-requests' ||
@@ -289,7 +317,7 @@ export function extractRateLimit(headers) {
       const n = Number(String(v));
       if (Number.isFinite(n)) remaining = n;
     } else if (
-      (lower === 'x-ratelimit-remaining-tokens' || lower === 'anthropic-ratelimit-tokens-remaining') &&
+      (lower === 'x-ratelimit-remaining-tokens' || lower === 'anthropic-ratelimit-tokens-remaining' || lower === 'openai-ratelimit-remaining-tokens') &&
       remaining === undefined
     ) {
       const n = Number(String(v));
@@ -306,15 +334,21 @@ export function extractRateLimit(headers) {
     } else if (
       lower === 'x-ratelimit-reset' ||
       lower === 'x-ratelimit-reset-requests' ||
+      lower === 'x-ratelimit-reset-tokens' ||
       lower === 'anthropic-ratelimit-requests-reset' ||
+      lower === 'anthropic-ratelimit-tokens-reset' ||
       lower === 'ratelimit-reset'
     ) {
-      const n = Number(String(v));
-      if (Number.isFinite(n)) {
-        reset = n;
-      } else {
-        const parsed = Date.parse(v);
-        if (!Number.isNaN(parsed)) reset = parsed;
+      const dur = parseDurationToSeconds(v);
+      if (dur !== null) {
+        if (dur > 1000000000) {
+          const epochMs = dur > 100000000000 ? dur : dur * 1000;
+          reset = dur;
+          resetMs = Math.max(0, epochMs - Date.now());
+        } else {
+          reset = dur;
+          resetMs = Math.max(0, Math.round(dur * 1000));
+        }
       }
     }
   }
@@ -347,9 +381,15 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
   if (!Array.isArray(refs) || refs.length <= 1) return (refs ?? []).slice();
   const list = refs.slice();
   if (strategy === 'least-loaded' && deps.concurrencyTracker && typeof deps.concurrencyTracker.getActive === 'function') {
+    const weights = deps.weights || (deps.pool && deps.pool.weightsMap) || {};
     return list.sort((a, b) => {
       const ca = deps.concurrencyTracker.getActive(a) ?? 0;
       const cb = deps.concurrencyTracker.getActive(b) ?? 0;
+      const wa = (weights[a] && Number(weights[a]) > 0) ? Number(weights[a]) : 1;
+      const wb = (weights[b] && Number(weights[b]) > 0) ? Number(weights[b]) : 1;
+      const scoreA = ca / wa;
+      const scoreB = cb / wb;
+      if (scoreA !== scoreB) return scoreA - scoreB;
       return ca - cb;
     });
   }
@@ -489,3 +529,13 @@ export function resetCircuitForProvider(circuitBreaker, provider) {
   });
   return ok === true;
 }
+
+/** Check if a key reference is permanently revoked due to auth/401 failure (#369). */
+export function isKeyRevoked(pool, ref) {
+  if (!pool || !ref) return false;
+  if (pool.revokedRefs instanceof Set && pool.revokedRefs.has(ref)) return true;
+  if (Array.isArray(pool.revokedRefs) && pool.revokedRefs.includes(ref)) return true;
+  if (pool.state && pool.state.revokedRefs instanceof Set && pool.state.revokedRefs.has(ref)) return true;
+  if (pool.state && Array.isArray(pool.state.revokedRefs) && pool.state.revokedRefs.includes(ref)) return true;
+  return false;
+}

package/lib/rotate.js

@@ -9,10 +9,12 @@ import {
   parseRetryAfter,
   extractRateLimit,
   isRateLimited,
+  isKeyPaused,
+  isKeyRevoked,
   formatExhaustionMessage,
   sortAttemptList,
 } from './pool.js';
-import { pickCascadeFallback } from './cascade.js';
+import { pickCascadeFallback, CASCADE_MAX_DEPTH } from './cascade.js';
 import { nextQuotaReset } from './quota-window.js';
 
 /**
@@ -52,20 +54,22 @@ export function createRotate(deps) {
       const reqStore = { pool, pickedRef: undefined, startMs: now() };
       setRotateStartMs(reqStore.startMs);
       let attemptList = (pool.weightedRefs ?? pool.refs).slice();
+      attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r));
       const strategy = pool.routingStrategy ?? runtime0.routingStrategy ?? 'round-robin';
       if (strategy !== 'round-robin') {
-        attemptList = sortAttemptList(attemptList, strategy, { latencyHistogram, concurrencyTracker });
+        attemptList = sortAttemptList(attemptList, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap });
       }
       if (strategy !== 'least-loaded' && runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
-        // #193: prefer least-loaded key within limit
+        // #193 / #365: prefer least-loaded key within limit with weights
         const available = attemptList.filter((r) => {
+          if (isKeyPaused(pool, r)) return false;
           const fu = pool.state.failedUntil.get(r) ?? 0;
           if (fu > now()) return false;
           const exp = pool.expiresAt ? pool.expiresAt[r] : undefined;
           if (exp !== undefined && now() >= exp) return false;
           return true;
         });
-        const preferred = concurrencyTracker.pickLeastLoaded(available);
+        const preferred = concurrencyTracker.pickLeastLoaded(available, now(), pool.weightsMap);
         if (preferred && attemptList[0] !== preferred) {
           const list = attemptList.slice();
           const i = list.indexOf(preferred);
@@ -106,13 +110,14 @@ export function createRotate(deps) {
         if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
         if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
         const _cStr = String(errCode ?? '');
-        if (_cStr === 'AUTH' || /auth/i.test(errMsg)) {
+        if (_cStr === 'AUTH' || /auth|unauthorized|invalid[_\s-]?api[_\s-]?key|unauthenticated/i.test(errMsg)) {
           const _c2 = (pool.state.authFailCounts.get(targetRef) ?? 0) + 1;
           pool.state.authFailCounts.set(targetRef, _c2);
-          if (_c2 >= 3) {
-            pool.state.brokenUntil.set(targetRef, now() + 86400000 * 30);
-            pool.state.failedUntil.set(targetRef, now() + 86400000 * 30);
-          }
+          if (!pool.state.revokedRefs) pool.state.revokedRefs = new Set();
+          pool.state.revokedRefs.add(targetRef);
+          pool.state.failedUntil.set(targetRef, Infinity);
+          pushEvent(pool, targetRef, 'KEY_REVOKED', 0);
+          logWarn(`[dsh-key-rotation] ${options.provider}: key ${targetRef} permanently REVOKED on auth failure`);
         } else {
           pool.state.authFailCounts.delete(targetRef);
         }
@@ -240,12 +245,15 @@ export function createRotate(deps) {
                         cool = rate.retryAfter * 1000;
                       } else if (rate.reset && rate.reset > now()) {
                         cool = rate.reset - now();
+                      } else if (rate.reset && rate.reset > 0 && rate.reset < 86400) {
+                        cool = Math.round(rate.reset * 1000);
                       } else {
                         cool = pool.cooldownMs ?? cooldownMs;
                       }
                       const maxCool = pool.maxCooldownMs ?? maxCooldownMs;
                       const effCool = Math.min(cool, maxCool ?? cool);
                       recordFailure(pool, activeRef, now(), effCool, maxCool);
+                      pool.state.proactiveSwitches = (pool.state.proactiveSwitches ?? 0) + 1;
                       pushEvent(pool, activeRef, 'RATE_LIMIT', effCool);
                       logWarn(`[dsh-key-rotation] ${options.provider}: key ${activeRef} proactive pause (remaining ${String(rate.remaining ?? '?')}/${String(rate.limit ?? '?')}, cool ${Math.round(effCool / 1000)}s) — next request will rotate`);
                     }
@@ -315,16 +323,25 @@ export function createRotate(deps) {
       // notify via extracted helper (see notifyExhaustion above)
       notifyExhaustion(runtime, pool, { provider: options.provider });
 
-      // #194: cross-provider cascade failover (guarded against infinite recursion)
-      if (!options.__isCascade && Array.isArray(runtime.cascade) && runtime.cascade.length > 0) {
+      // #194 / #364: cross-provider cascade failover with model mapping
+      const maxDepth = runtime.cascadeMaxDepth ?? CASCADE_MAX_DEPTH;
+      const currentDepth = options.__cascadeDepth ?? (options.__isCascade ? 1 : 0);
+      if (currentDepth < maxDepth && Array.isArray(runtime.cascade) && runtime.cascade.length > 0) {
         const pools = runtime.providerToPool;
-        const fb = pickCascadeFallback(options.provider, runtime, pools);
+        const fb = pickCascadeFallback(options.provider, runtime, pools, options.model);
         if (fb && fb.pool && fb.pool !== pool) {
-          logWarn(`[dsh-key-rotation] ${options.provider}: pool exhausted — cascading to ${fb.provider}`);
+          logWarn(`[dsh-key-rotation] ${options.provider}: pool exhausted — cascading to ${fb.provider} (model: ${fb.model || options.model || 'default'})`);
           pool.state.lastReason = 'CASCADE';
           pool.state.lastSwitchAt = now();
-          // Re-dispatch on the fallback pool (depth-1 via __isCascade guard)
-          const innerCascade = rotate({ ...options, provider: fb.provider, __isCascade: true }, fb.pool);
+          pool.state.cascadeCount = (pool.state.cascadeCount ?? 0) + 1;
+          pushEvent(pool, reqStore.pickedRef ?? 'pool', 'CASCADE_TRIGGERED', 0);
+          const innerCascade = rotate({
+            ...options,
+            provider: fb.provider,
+            model: fb.model || options.model,
+            __isCascade: true,
+            __cascadeDepth: currentDepth + 1,
+          }, fb.pool);
           for await (const chunk of innerCascade) {
             yield chunk;
           }

package/lib/routes-ops.js

@@ -1,3 +1,4 @@
+import { registerMetricsRoutes } from './ops-metrics.js';
 // lib/routes-ops.js — operational HTTP routes for dsh-key-rotation (#253/#312).
 // Thin orchestrator: each route family lives in its own lib/ops-*.js module.
 // Registration is injected with the live apply() dependencies.
@@ -18,4 +19,5 @@ export function registerOpsRoutes(ctx, deps) {
   registerKeyRoutes(ctx, deps);
   registerTestRoutes(ctx, deps);
   registerWebhookActionRoute(ctx, deps);
+  registerMetricsRoutes(ctx, deps);
 }

package/lib/webhook.js

@@ -12,6 +12,8 @@ export function detectPlatform(url) {
   if (u.includes('api.telegram.org')) return 'telegram';
   if (u.includes('discord.com/api/webhooks')) return 'discord';
   if (u.includes('hooks.slack.com')) return 'slack';
+  if (u.includes('ntfy.sh') || u.includes('/ntfy')) return 'ntfy';
+  if (u.includes('gotify') || u.includes('/message')) return 'gotify';
   return 'generic';
 }
 
@@ -66,6 +68,32 @@ export function formatInteractive(url, payload, actionToken) {
       ],
     };
   }
+  if (platform === 'gotify') {
+    return {
+      title,
+      message: text,
+      priority: 5,
+      extras: {
+        'client::notification': {
+          click: { url: actions[0]?.url || '' },
+        },
+      },
+    };
+  }
+  if (platform === 'ntfy') {
+    return {
+      topic: url.split('/').pop(),
+      title,
+      message: text,
+      priority: 4,
+      tags: ['warning', 'key-rotation'],
+      actions: actions.map((a) => ({
+        action: 'view',
+        label: a.label,
+        url: a.url || '',
+      })),
+    };
+  }
   // generic JSON webhook: actions as plain data, receiver decides
   return { ...payload, actions };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.25",
+  "version": "0.8.28",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [