@goodandready/dsh-key-rotation 0.8.73 → 0.8.75

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## 0.8.75 (2026-10-30)
+
+### Dependencies, Cleanup & Audit Finalization (Block 3 Release)
+- **Zero Runtime Dependencies & Security Hardening (Issues #520, #535)**: Completely removed `undici` dependency and eliminated 5 high/moderate vulnerabilities (`pnpm audit --prod` reports 0 vulnerabilities); `safe-fetch.js` rewritten using standard `globalThis.fetch` and pure Node.js DNS/IP validation.
+- **Atomic I/O Cleanup (Issue #521)**: Crash-safe file writes now safely clean up `.tmp` files on write/sync/rename errors, and `cleanupStaleTmpFiles` purges abandoned temporary files on startup.
+- **Request Budget Inheritance & Signal Management (Issue #464)**: `maxAttempts` and `maxDurationMs` properly propagated from settings to `buildRuntime()` and `pool-builder`; `createRequestBudget` removes caller `AbortSignal` listeners upon `dispose()`; guaranteed cleanup in `rotate.js` `finally` block; traffic smoother now receives `budget.signal`.
+- **Selection Trace Diagnosability (Issue #461)**: Candidate evaluation now logs detailed `SKIPPED` steps (`PAUSED`, `REVOKED`, `BROKEN`, `DRAINING`, `COOLDOWN`, `CAPABILITY_MISMATCH`, `QUOTA_EXHAUSTED`); added interactive Selection Trace UI inspector in settings card.
+- **Managed Key Drain UI & Concurrency Robustness (Issue #463)**: Added interactive Managed Key Drain section in settings card; enhanced `ConcurrencyTracker` with `touch()` and prolonged 30m stale threshold for long-running streaming generation.
+- **Offline Simulator Parity (Issue #462)**: Audit verification finalized following Block 2 quarantine cloning, Map lookup, and per-Context isolation fixes.
+
+## 0.8.74 (2026-10-30)
+
+### Rotation & Routing Reliability (Block 2 Release)
+- **Error Status Normalization (Issue #545)**: `isSwitchableError` now normalizes string and numeric `code: "429"` / `429` to HTTP 429 when `status` is absent, ensuring default `switchCodes` trigger key rotation.
+- **Quick Action Skip (Issue #540)**: Quick action skip now atomically synchronizes both `targetPool.state.pointer` and `cursor`, ensuring resolver round-robin picks the newly advanced key immediately.
+- **Tiers & Tier-Priority Routing (Issue #541)**: Propagate `p.tiers` to `pool-builder` options, fix `typeof tiers === 'object'` syntax error, and add `tier-priority` strategy to Schemastery and resolver.
+- **Traffic Smoothing Leaky Bucket (Issue #538)**: Implemented Generic Cell Rate Algorithm (GCRA) in `TrafficSmoother` to atomically reserve micro-queue slots ahead of sleep, preventing two concurrent requests from claiming the same time slot.
+- **Simulator Quota & Broken Key Parity (Issue #534)**: `simulateRouting` now clones `brokenUntil`, `tokenUsage`, and `quotaWindows` into simulator pool state, properly filtering out unavailable keys.
+- **Simulator Provider Map Lookup (Issue #536)**: Corrected `runtime.providerToPool` Map resolution in `ops-telemetry` simulator route, returning 404 for unrecognized providers.
+- **Config Import Concurrency (Issue #553)**: Added optimistic revision check via `expectedRevision` in `handleConfigImport`, returning 409 Conflict upon concurrent modification.
+- **Config Import Fail-Closed (Issue #533)**: `handleConfigImport` fails closed with 503 Service Unavailable when `settings.replace` is unavailable, preventing false-positive `{ applied: true }`.
+- **Instance & Context Isolation (Issue #544)**: Introduced `KeyDrainManager`, `SelectionTraceStore`, and `DiagBuffer` classes with per-context lifecycle binding; disposal of one plugin instance cleanly purges only its own draining keys and telemetry without corrupting peer instances.
+
 ## 0.8.73 (2026-10-30)
 
 ### Security & Hardening (Block 1 Security Release)

package/lib/atomic-io.js

@@ -1,5 +1,6 @@
-// lib/atomic-io.js — crash-safe file writes and safe JSON loads (#264).
+// lib/atomic-io.js — crash-safe file writes and safe JSON loads (#264, #521).
 // Never overwrite a non-empty destination with empty/corrupt parse results.
+// Clean up orphaned temporary files on failure or during startup.
 
 import { promises as fs } from 'node:fs';
 import path from 'node:path';
@@ -8,6 +9,7 @@ import crypto from 'node:crypto';
 
 /**
  * Atomically write data to filePath (temp + rename in same directory).
+ * Cleans up temporary file if open, write, sync, or rename fails (#521).
  * @param {string} filePath
  * @param {string|Buffer} data
  */
@@ -15,14 +17,55 @@ export async function atomicWriteFile(filePath, data) {
   const dir = path.dirname(filePath);
   await fs.mkdir(dir, { recursive: true });
   const tmp = path.join(dir, `.${path.basename(filePath)}.${process.pid}.${crypto.randomBytes(4).toString('hex')}.tmp`);
-  const fh = await fs.open(tmp, 'w');
+  let fh = null;
   try {
+    fh = await fs.open(tmp, 'w');
     await fh.writeFile(data);
     await fh.sync();
-  } finally {
     await fh.close();
+    fh = null;
+    await fs.rename(tmp, filePath);
+  } catch (err) {
+    if (fh) {
+      try { await fh.close(); } catch {}
+    }
+    try { await fs.unlink(tmp); } catch {}
+    throw err;
+  }
+}
+
+/**
+ * Clean up stale temporary files left behind by crashed or interrupted writes (#521).
+ * @param {string} filePath
+ * @param {number} [maxAgeMs=3600000]
+ * @returns {Promise<number>} number of files unlinked
+ */
+export async function cleanupStaleTmpFiles(filePath, maxAgeMs = 3600000) {
+  if (!filePath || typeof filePath !== 'string') return 0;
+  try {
+    const dir = path.dirname(filePath);
+    const base = path.basename(filePath);
+    const prefix = `.${base}.`;
+    const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
+    const now = Date.now();
+    let cleaned = 0;
+    for (const entry of entries) {
+      if (!entry.isFile()) continue;
+      if (entry.name.startsWith(prefix) && entry.name.endsWith('.tmp')) {
+        const fullPath = path.join(dir, entry.name);
+        try {
+          const st = await fs.stat(fullPath);
+          if (now - st.mtimeMs > maxAgeMs) {
+            await fs.unlink(fullPath);
+            cleaned++;
+          }
+        } catch {}
+      }
+    }
+    return cleaned;
+  } catch {
+    return 0;
   }
-  await fs.rename(tmp, filePath);
 }
 
 /**

package/lib/client.js

@@ -61,6 +61,24 @@ window.__ModuleLoader__.load({
       sectionStatsDesc: 'Real-time telemetry of active key pools, healthy credentials, and failover status.',
       sectionFailover: 'Failover Triggers & Error Codes',
       sectionFailoverDesc: 'Select error conditions that trigger automatic switching to the next available key.',
+            maxAttemptsLabel: 'Max Attempts (Budget)',
+      maxDurationLabel: 'Max Duration ms (Budget)',
+      traceTitle: 'Key Selection Traces',
+      traceDesc: 'Candidate evaluation history, skips and selection decisions for requests.',
+      traceRefresh: 'Refresh Traces',
+      traceClear: 'Clear History',
+      traceEmpty: 'No selection traces recorded yet.',
+      traceSelected: 'Selected Key',
+      traceOutcome: 'Outcome',
+      drainTitle: 'Managed Key Drain',
+      drainDesc: 'Safely transition a key out of rotation waiting for active requests to finish.',
+      drainKeyPlaceholder: 'Key ref (e.g. DEEPSEEK_API_KEY_2)',
+      drainStart: 'Start Drain',
+      drainCancel: 'Cancel Drain',
+      drainActiveStreams: 'Active requests',
+      drainCanComplete: 'Ready to remove',
+      drainDraining: 'Draining Keys',
+      drainEmpty: 'No keys currently draining.',
       sectionTiming: 'Timing & Rotation Schedule',
       sectionTimingDesc: 'Configure cooldown durations and scheduled periodic rotation windows.',
       sectionBackup: 'Backup & Snapshots',
@@ -317,6 +335,24 @@ window.__ModuleLoader__.load({
       sectionStatsDesc: '活跃密钥池、健康凭证和故障转移状态的实时遥测。',
       sectionFailover: '故障转移触发器与错误代码',
       sectionFailoverDesc: '选择触发自动切换至下一个可用密钥的错误条件。',
+            maxAttemptsLabel: '最大重试次数 (预算)',
+      maxDurationLabel: '最大超时毫秒 (预算)',
+      traceTitle: '密钥选择追踪 (Trace)',
+      traceDesc: '查看请求的候选密钥评估、跳过原因和最终选择决策。',
+      traceRefresh: '刷新追踪',
+      traceClear: '清空历史',
+      traceEmpty: '暂无选择追踪记录。',
+      traceSelected: '所选密钥',
+      traceOutcome: '结果',
+      drainTitle: '安全下线密钥 (Drain)',
+      drainDesc: '平滑下线密钥并等待当前在途请求完成，避免中途断流。',
+      drainKeyPlaceholder: '密钥标识 (例如 DEEPSEEK_API_KEY_2)',
+      drainStart: '开始下线',
+      drainCancel: '取消下线',
+      drainActiveStreams: '在途请求数',
+      drainCanComplete: '可安全移除',
+      drainDraining: '下线中的密钥',
+      drainEmpty: '当前没有正在下线的密钥。',
       sectionTiming: '时间与轮换计划',
       sectionTimingDesc: '配置冷却持续时间和定期轮换计划窗口。',
       sectionBackup: '备份与快照',
@@ -1257,6 +1293,16 @@ window.__ModuleLoader__.load({
       // Auto-Discovery State (#491)
       const [discoveredKeys, setDiscoveredKeys] = React.useState([]);
       const [autoImporting, setAutoImporting] = React.useState(false);
+
+      // Selection Trace State (#461)
+      const [selectionTraces, setSelectionTraces] = React.useState([]);
+      const [loadingTraces, setLoadingTraces] = React.useState(false);
+
+      // Managed Key Drain State (#463)
+      const [drainingKeys, setDrainingKeys] = React.useState([]);
+      const [drainRefInput, setDrainRefInput] = React.useState('');
+      const [loadingDrain, setLoadingDrain] = React.useState(false);
+      const [drainActionError, setDrainActionError] = React.useState('');
       const [autoImportMsg, setAutoImportMsg] = React.useState('');
 
       // Test Matrix State (#490)
@@ -1472,6 +1518,70 @@ window.__ModuleLoader__.load({
         }
       };
 
+      const loadTraces = async () => {
+        setLoadingTraces(true);
+        try {
+          const res = await fetch('/dsh-key-rotation/trace?limit=20');
+          const data = await res.json();
+          if (data && data.ok) {
+            setSelectionTraces(data.traces || []);
+          }
+        } catch (_) {}
+        finally {
+          setLoadingTraces(false);
+        }
+      };
+
+      const clearTraces = async () => {
+        try {
+          await fetch('/dsh-key-rotation/trace', { method: 'DELETE' });
+          setSelectionTraces([]);
+        } catch (_) {}
+      };
+
+      const loadDrainStatus = async () => {
+        setLoadingDrain(true);
+        try {
+          const res = await fetch('/dsh-key-rotation/drain');
+          const data = await res.json();
+          if (data && data.ok) {
+            setDrainingKeys(data.draining || []);
+          }
+        } catch (_) {}
+        finally {
+          setLoadingDrain(false);
+        }
+      };
+
+      const startKeyDrain = async (ref) => {
+        if (!ref || !ref.trim()) return;
+        setDrainActionError('');
+        try {
+          const res = await fetch('/dsh-key-rotation/drain', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ ref: ref.trim() }),
+          });
+          const data = await res.json();
+          if (data && data.ok) {
+            setDrainRefInput('');
+            await loadDrainStatus();
+          } else {
+            setDrainActionError(data?.error?.message || 'Drain request failed');
+          }
+        } catch (e) {
+          setDrainActionError(String(e?.message || e));
+        }
+      };
+
+      const cancelKeyDrain = async (ref) => {
+        if (!ref) return;
+        try {
+          await fetch(`/dsh-key-rotation/drain?ref=${encodeURIComponent(ref)}`, { method: 'DELETE' });
+          await loadDrainStatus();
+        } catch (_) {}
+      };
+
       const [optimisticReset, setOptimisticReset] = React.useState({});
       const [selected, setSelected] = React.useState(new Set());
       const [bulkCooldown, setBulkCooldown] = React.useState('');
@@ -2928,6 +3038,10 @@ window.__ModuleLoader__.load({
           field(t('rpmLimitLabel'), textInput(String(val.rpmLimit ?? 0), (v) => setField((cur) => ({ ...cur, rpmLimit: Number(v) || 0 })))),
           field(t('tpmLimitLabel'), textInput(String(val.tpmLimit ?? 0), (v) => setField((cur) => ({ ...cur, tpmLimit: Number(v) || 0 }))))
         ),
+        h('div', { className: 'krot-grid-2', style: { marginTop: '10px' } },
+          field(t('maxAttemptsLabel'), textInput(String(val.maxAttempts ?? 0), (v) => setField((cur) => ({ ...cur, maxAttempts: Number(v) || 0 })))),
+          field(t('maxDurationLabel'), textInput(String(val.maxDurationMs ?? 0), (v) => setField((cur) => ({ ...cur, maxDurationMs: Number(v) || 0 }))))
+        ),
         h('div', { className: 'krot-grid-3', style: { marginTop: '10px' } },
           field(t('routingStrategyLabel'), h('select', {
             className: 'krot-in',
@@ -3017,6 +3131,68 @@ window.__ModuleLoader__.load({
         ) : null
       );
 
+      const drainSection = h('div', { className: 'krot-section-card' },
+        h('div', { className: 'krot-section-title' },
+          h('span', null, '⏳ ' + t('drainTitle')),
+          h('span', { className: 'krot-badge ' + (drainingKeys.length > 0 ? 'krot-badge-warn' : 'krot-badge-ok') },
+            drainingKeys.length > 0 ? (drainingKeys.length + ' ' + t('drainDraining')) : t('drainEmpty')
+          )
+        ),
+        h('div', { className: 'krot-section-desc' }, t('drainDesc')),
+        h('div', { style: { display: 'flex', gap: '8px', flexWrap: 'wrap', alignItems: 'center', marginTop: '10px' } },
+          h('input', {
+            className: 'krot-in',
+            placeholder: t('drainKeyPlaceholder'),
+            value: drainRefInput,
+            onChange: (e) => setDrainRefInput(e.target.value),
+            style: { maxWidth: '240px' },
+          }),
+          btn(t('drainStart'), () => startKeyDrain(drainRefInput), { primary: true, disabled: !drainRefInput.trim() }),
+          btn(t('traceRefresh'), loadDrainStatus, { disabled: loadingDrain })
+        ),
+        drainActionError ? h('div', { className: 'krot-warn', style: { marginTop: '8px' } }, '⚠ ' + drainActionError) : null,
+        drainingKeys.length > 0 ? h('div', { style: { marginTop: '10px', display: 'flex', flexDirection: 'column', gap: '6px' } },
+          drainingKeys.map((dk) => h('div', { key: dk.ref, className: 'krot-stat-box', style: { display: 'flex', justifyContent: 'space-between', alignItems: 'center', padding: '6px 12px' } },
+            h('span', { style: { fontWeight: 600 } }, dk.ref),
+            h('span', { className: 'krot-badge ' + (dk.canComplete ? 'krot-badge-ok' : 'krot-badge-warn') },
+              dk.canComplete ? t('drainCanComplete') : (t('drainActiveStreams') + ': ' + dk.inFlight)
+            ),
+            btn(t('drainCancel'), () => cancelKeyDrain(dk.ref), { danger: true })
+          ))
+        ) : null
+      );
+
+      const traceSection = h('div', { className: 'krot-section-card' },
+        h('div', { className: 'krot-section-title' },
+          h('span', null, '🧭 ' + t('traceTitle')),
+          h('span', { className: 'krot-badge krot-badge-ok' }, selectionTraces.length + ' traces')
+        ),
+        h('div', { className: 'krot-section-desc' }, t('traceDesc')),
+        h('div', { style: { display: 'flex', gap: '8px', alignItems: 'center', marginTop: '10px' } },
+          btn(loadingTraces ? '...' : t('traceRefresh'), loadTraces, { disabled: loadingTraces }),
+          btn(t('traceClear'), clearTraces, { disabled: selectionTraces.length === 0 })
+        ),
+        selectionTraces.length > 0 ? h('div', { style: { marginTop: '10px', display: 'flex', flexDirection: 'column', gap: '8px', maxHeight: '280px', overflowY: 'auto' } },
+          selectionTraces.slice().reverse().map((tr) => h('div', { key: tr.requestId, style: { border: '1px solid var(--dsw-alias-border-l2)', borderRadius: '6px', padding: '8px', fontSize: '11px' } },
+            h('div', { style: { display: 'flex', justifyContent: 'space-between', marginBottom: '4px' } },
+              h('span', { style: { fontWeight: 600 } }, (tr.provider || 'default') + (tr.model ? ' / ' + tr.model : '')),
+              h('span', { className: 'krot-badge ' + (tr.outcome === 'SUCCESS' ? 'krot-badge-ok' : 'krot-badge-bad') }, tr.outcome)
+            ),
+            h('div', { style: { color: 'var(--dsw-alias-label-secondary)', marginBottom: '4px' } },
+              tr.selectedRef ? (t('traceSelected') + ': ' + tr.selectedRef) : ''
+            ),
+            h('div', { style: { display: 'flex', flexDirection: 'column', gap: '2px', paddingLeft: '8px', borderLeft: '2px solid var(--dsw-alias-border-l2)' } },
+              (tr.steps || []).map((st, i) => h('div', { key: i, style: { display: 'flex', gap: '6px' } },
+                h('span', { style: { color: st.decision === 'SELECTED' ? 'var(--dsw-alias-state-success-primary)' : st.decision === 'SKIPPED' ? 'var(--dsw-alias-label-tertiary)' : 'var(--dsw-alias-state-danger-primary)', fontWeight: 600 } }, st.decision),
+                h('span', { style: { fontWeight: 600 } }, st.ref || '-'),
+                h('span', { style: { color: 'var(--dsw-alias-label-secondary)' } }, st.reasonCode ? ('[' + st.reasonCode + ']') : ''),
+                h('span', null, st.details || '')
+              ))
+            )
+          ))
+        ) : h('div', { className: 'krot-hint', style: { marginTop: '8px' } }, t('traceEmpty'))
+      );
+
       const backupSection = h('div', { className: 'krot-section-card' },
         h('div', { className: 'krot-section-title' },
           h('span', null, '💾 ' + t('sectionBackup')),
@@ -3305,6 +3481,8 @@ window.__ModuleLoader__.load({
           poolsSection,
           failoverSection,
           timingSection,
+          drainSection,
+          traceSection,
           backupSection,
           simulatorSection,
         ),

package/lib/clone-lifecycle.js

@@ -148,7 +148,7 @@ export class CloneLifecycleManager {
   startDrain(cloneId, provider, onDeregister) {
     if (this.drainingClones.has(cloneId)) return;
 
-    markKeyDraining(cloneId, provider);
+    markKeyDraining(cloneId, provider, this.keyDrainManager);
     const entry = {
       cloneId,
       provider,
@@ -176,7 +176,7 @@ export class CloneLifecycleManager {
 
     if (inFlight === 0) {
       this.drainingClones.delete(cloneId);
-      cancelKeyDrain(cloneId);
+      cancelKeyDrain(cloneId, this.keyDrainManager);
       if (this.logger?.info) {
         this.logger.info(`[dsh-key-rotation] clone ${cloneId} (${entry.provider}) drained cleanly; safe to deregister`);
       }
@@ -191,11 +191,11 @@ export class CloneLifecycleManager {
 
   cancelDrain(cloneId) {
     this.drainingClones.delete(cloneId);
-    cancelKeyDrain(cloneId);
+    cancelKeyDrain(cloneId, this.keyDrainManager);
   }
 
   isDraining(cloneId) {
-    return this.drainingClones.has(cloneId) || isKeyDraining(cloneId);
+    return this.drainingClones.has(cloneId) || isKeyDraining(cloneId, this.keyDrainManager);
   }
 
   getDrainingList() {
@@ -209,7 +209,7 @@ export class CloneLifecycleManager {
 
   destroy() {
     for (const cloneId of this.drainingClones.keys()) {
-      cancelKeyDrain(cloneId);
+      cancelKeyDrain(cloneId, this.keyDrainManager);
     }
     this.drainingClones.clear();
   }

package/lib/concurrency.js

@@ -1,7 +1,8 @@
-// concurrency.js — per-key in-flight counter + least-connections picking (issues #193, #407).
+// concurrency.js — per-key in-flight counter + least-connections picking (issues #193, #407, #463).
 
 const CONCURRENCY_DEFAULT_LIMIT = 0;
-const CONCURRENCY_STALE_LOCK_MS = 5 * 60 * 1000;
+// Default stale timeout: 30 minutes to support prolonged streaming generation (#463)
+const CONCURRENCY_STALE_LOCK_MS = 30 * 60 * 1000;
 
 export class ConcurrencyTracker {
   constructor(opts) {
@@ -42,6 +43,14 @@ export class ConcurrencyTracker {
     return true;
   }
 
+  touch(ref, now) {
+    now = now || Date.now();
+    const e = this._inFlight.get(ref);
+    if (e && e.count > 0) {
+      e.lastAcquired = now;
+    }
+  }
+
   release(ref, now) {
     now = now || Date.now();
     const e = this._inFlight.get(ref);
@@ -55,13 +64,24 @@ export class ConcurrencyTracker {
     if (!ref) return 0;
     const e = this._inFlight.get(ref);
     if (!e) return 0;
-    if (now - e.lastAcquired > this._staleMs) return 0;
+    if (now - e.lastAcquired > this._staleMs) {
+      e.count = 0;
+      return 0;
+    }
     return e.count;
   }
 
-  snapshot() {
+  snapshot(now) {
+    now = now || Date.now();
     const out = {};
-    for (const [k, v] of this._inFlight) out[k] = { count: v.count };
+    for (const [k, v] of this._inFlight) {
+      let count = v.count;
+      if (now - v.lastAcquired > this._staleMs) {
+        count = 0;
+        v.count = 0;
+      }
+      out[k] = { count };
+    }
     return out;
   }
 

package/lib/index.js

@@ -1,3 +1,6 @@
+import { KeyDrainManager } from './key-drain.js';
+import { SelectionTraceStore } from './selection-trace.js';
+import { DiagBuffer } from './ops-telemetry.js';
 import { createResolver } from './resolver.js';
 import { decryptSecret } from './crypto-storage.js';
 // dsh-key-rotation — per-provider API key rotation for DeepSeek Harness.
@@ -98,7 +101,7 @@ const SettingsSchema = Schema.object({
   rateLimitThreshold: Schema.number().default(0.1),
   proactiveRateLimitGuard: Schema.boolean().default(true),
   selfHealingIntervalMinutes: Schema.number().default(30),
-  routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky']).default('round-robin'),
+  routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky', 'tier-priority']).default('round-robin'),
   rpmLimit: Schema.number().default(0),
   tpmLimit: Schema.number().default(0),
   webhookActionToken: Schema.string().role('secret').default(''),
@@ -148,7 +151,7 @@ const SettingsSchema = Schema.object({
     })).default({}),
     cooldownMs: Schema.number(),
     maxCooldownMs: Schema.number(),
-    routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky']),
+    routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky', 'tier-priority']),
     proactiveRateLimitGuard: Schema.boolean(),
     rpmLimit: Schema.number(),
     tpmLimit: Schema.number(),
@@ -196,13 +199,16 @@ export function apply(ctx, config = {}) {
   const latencyHistogram = new LatencyHistogram();
   const quotaStore = new QuotaStore();
   const concurrencyTracker = new ConcurrencyTracker();
+  const keyDrainManager = new KeyDrainManager();
+  const selectionTraceStore = new SelectionTraceStore();
+  const diagBuffer = new DiagBuffer();
 
   const moduleBreaker = new CircuitBreaker({ threshold: 5, openMs: 30000, halfOpenProbes: 1, now: nowMono });
   const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
   const alertDebouncer = new AlertDebouncer({ sender: webhookSender, debounceMs: 2000 });
   const moduleNotifyQueue = new NotifyQueue({ send: (url, payload) => alertDebouncer.enqueue(url, payload) });
   const trafficSmoother = new TrafficSmoother({ maxQueueWaitMs: 800 });
-  const cloneLifecycle = new CloneLifecycleManager({ concurrencyTracker, logger });
+  const cloneLifecycle = new CloneLifecycleManager({ concurrencyTracker, logger, keyDrainManager });
   ctx.effect(() => () => { cloneLifecycle.destroy(); }, 'dsh-key-rotation: clone lifecycle cleanup');
 
   const sandboxService = createSandboxService({ getRuntime: () => buildRuntime() });
@@ -214,7 +220,11 @@ export function apply(ctx, config = {}) {
 
   setupIdleHealEffect(ctx, getConfig, buildRuntime);
   setupProactiveHealthProbeEffect(ctx, getConfig, buildRuntime, ensureSandboxRunner, logger);
-  setupKeyDrainEffect(ctx);
+  setupKeyDrainEffect(ctx, keyDrainManager);
+  ctx.effect(() => () => {
+    selectionTraceStore.clear();
+    diagBuffer.clear();
+  }, 'dsh-key-rotation: telemetry cleanup');
 
   const poolState = new Map();
   const { schedulePersist, persistenceSnapshot } = setupPersistence(ctx, {
@@ -363,6 +373,8 @@ export function apply(ctx, config = {}) {
       showChatHeaderChip: cfg.showChatHeaderChip !== false,
       switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000,
       warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0,
+      maxAttempts: typeof cfg.maxAttempts === 'number' && cfg.maxAttempts > 0 ? Math.floor(cfg.maxAttempts) : 0,
+      maxDurationMs: typeof cfg.maxDurationMs === 'number' && cfg.maxDurationMs > 0 ? Math.floor(cfg.maxDurationMs) : 0,
       index, pools,
       breaker: moduleBreaker, notifyQueue: moduleNotifyQueue, concurrencyTracker, quotaStore, latencyHistogram, trafficSmoother,
       providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider,
@@ -461,6 +473,9 @@ export function apply(ctx, config = {}) {
     circuitBreaker: moduleBreaker,
     notifyQueue: moduleNotifyQueue,
     quotaStore,
+    keyDrainManager,
+    selectionTraceStore,
+    diagBuffer,
   });
 
   ctx.effect(() => {

package/lib/key-drain.js

@@ -1,56 +1,91 @@
-// lib/key-drain.js — Managed key draining and graceful replacement (#463, #502).
+// lib/key-drain.js — Managed key draining and graceful replacement (#463, #502, #544).
 // Allows draining an old key ref (no new dispatches assigned) while letting
 // in-flight active streams complete cleanly before permanent removal.
+// KeyDrainManager provides instance-level isolation (#544).
 
-const drainingKeys = new Map(); // ref -> { provider, startedAt, inFlightCount }
+export class KeyDrainManager {
+  constructor() {
+    this.drainingKeys = new Map(); // ref -> { provider, startedAt, inFlightCount }
+  }
+
+  markKeyDraining(ref, provider = '') {
+    if (!ref) return false;
+    this.drainingKeys.set(ref, {
+      ref,
+      provider,
+      startedAt: Date.now(),
+    });
+    return true;
+  }
+
+  isKeyDraining(ref) {
+    if (!ref) return false;
+    return this.drainingKeys.has(ref);
+  }
+
+  cancelKeyDrain(ref) {
+    if (!ref) return false;
+    return this.drainingKeys.delete(ref);
+  }
+
+  getDrainingStatus(ref, concurrencyTracker) {
+    if (!ref || !this.drainingKeys.has(ref)) return null;
+    const entry = this.drainingKeys.get(ref);
+    const inFlight = concurrencyTracker && typeof concurrencyTracker.getActive === 'function'
+      ? concurrencyTracker.getActive(ref)
+      : (concurrencyTracker && typeof concurrencyTracker.getInFlight === 'function'
+        ? concurrencyTracker.getInFlight(ref)
+        : 0);
+
+    return {
+      ref,
+      provider: entry.provider,
+      startedAt: entry.startedAt,
+      elapsedMs: Date.now() - entry.startedAt,
+      inFlight,
+      canComplete: inFlight === 0,
+    };
+  }
+
+  getAllDrainingKeys(concurrencyTracker) {
+    const result = [];
+    for (const ref of this.drainingKeys.keys()) {
+      result.push(this.getDrainingStatus(ref, concurrencyTracker));
+    }
+    return result;
+  }
 
-export function markKeyDraining(ref, provider = '') {
-  if (!ref) return false;
-  drainingKeys.set(ref, {
-    ref,
-    provider,
-    startedAt: Date.now(),
-  });
-  return true;
+  clearAllDraining() {
+    this.drainingKeys.clear();
+  }
 }
 
-export function isKeyDraining(ref) {
-  if (!ref) return false;
-  return drainingKeys.has(ref);
+const defaultDrainManager = new KeyDrainManager();
+
+export function markKeyDraining(ref, provider = '', manager = null) {
+  return (manager || defaultDrainManager).markKeyDraining(ref, provider);
 }
 
-export function cancelKeyDrain(ref) {
-  if (!ref) return false;
-  return drainingKeys.delete(ref);
+export function isKeyDraining(ref, manager = null) {
+  return (manager || defaultDrainManager).isKeyDraining(ref);
 }
 
-export function getDrainingStatus(ref, concurrencyTracker) {
-  if (!ref || !drainingKeys.has(ref)) return null;
-  const entry = drainingKeys.get(ref);
-  const inFlight = concurrencyTracker && typeof concurrencyTracker.getActive === 'function'
-    ? concurrencyTracker.getActive(ref)
-    : (concurrencyTracker && typeof concurrencyTracker.getInFlight === 'function'
-      ? concurrencyTracker.getInFlight(ref)
-      : 0);
-
-  return {
-    ref,
-    provider: entry.provider,
-    startedAt: entry.startedAt,
-    elapsedMs: Date.now() - entry.startedAt,
-    inFlight,
-    canComplete: inFlight === 0,
-  };
+export function cancelKeyDrain(ref, manager = null) {
+  return (manager || defaultDrainManager).cancelKeyDrain(ref);
 }
 
-export function getAllDrainingKeys(concurrencyTracker) {
-  const result = [];
-  for (const ref of drainingKeys.keys()) {
-    result.push(getDrainingStatus(ref, concurrencyTracker));
-  }
-  return result;
+export function getDrainingStatus(ref, concurrencyTracker, manager = null) {
+  return (manager || defaultDrainManager).getDrainingStatus(ref, concurrencyTracker);
+}
+
+export function getAllDrainingKeys(concurrencyTracker, manager = null) {
+  return (manager || defaultDrainManager).getAllDrainingKeys(concurrencyTracker);
 }
 
-export function clearAllDraining() {
-  drainingKeys.clear();
+export function clearAllDraining(manager = null) {
+  if (manager) {
+    manager.clearAllDraining();
+  } else {
+    defaultDrainManager.clearAllDraining();
+  }
 }

package/lib/lifecycle.js

@@ -2,6 +2,7 @@
 import { healIdleCooldowns, autoUnbreakBrokenKeys } from './heal.js';
 import { decryptSecret } from './crypto-storage.js';
 import { StatePersistence, resolveStatePath } from './persistence.js';
+import { cleanupStaleTmpFiles } from './atomic-io.js';
 import path from 'node:path';
 import { envValue } from './pool.js';
 import { clearAllDraining } from './key-drain.js';
@@ -97,6 +98,7 @@ export function setupPersistence(ctx, { cfg0, poolState, moduleBreaker, verboseL
       dataDir: hostDirs[0],
     });
     if (cfg0.persistenceEnabled !== false && resolvedPath) {
+      cleanupStaleTmpFiles(resolvedPath).catch(() => {});
       statePersistence = new StatePersistence({ filePath: resolvedPath });
       statePersistence.load().then((snap) => {
         if (!snap) return;
@@ -251,10 +253,10 @@ export function setupProactiveHealthProbeEffect(ctx, getConfig, buildRuntime, en
   }, 'dsh-key-rotation: proactive health probing');
 }
 
-export function setupKeyDrainEffect(ctx) {
+export function setupKeyDrainEffect(ctx, keyDrainManager = null) {
   return ctx.effect(() => {
     return () => {
-      clearAllDraining();
+      clearAllDraining(keyDrainManager);
     };
   }, 'dsh-key-rotation: clear draining keys on teardown');
 }