@goodandready/dsh-key-rotation 0.8.73 → 0.8.74

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.8.74 (2026-10-30)
+
+### Rotation & Routing Reliability (Block 2 Release)
+- **Error Status Normalization (Issue #545)**: `isSwitchableError` now normalizes string and numeric `code: "429"` / `429` to HTTP 429 when `status` is absent, ensuring default `switchCodes` trigger key rotation.
+- **Quick Action Skip (Issue #540)**: Quick action skip now atomically synchronizes both `targetPool.state.pointer` and `cursor`, ensuring resolver round-robin picks the newly advanced key immediately.
+- **Tiers & Tier-Priority Routing (Issue #541)**: Propagate `p.tiers` to `pool-builder` options, fix `typeof tiers === 'object'` syntax error, and add `tier-priority` strategy to Schemastery and resolver.
+- **Traffic Smoothing Leaky Bucket (Issue #538)**: Implemented Generic Cell Rate Algorithm (GCRA) in `TrafficSmoother` to atomically reserve micro-queue slots ahead of sleep, preventing two concurrent requests from claiming the same time slot.
+- **Simulator Quota & Broken Key Parity (Issue #534)**: `simulateRouting` now clones `brokenUntil`, `tokenUsage`, and `quotaWindows` into simulator pool state, properly filtering out unavailable keys.
+- **Simulator Provider Map Lookup (Issue #536)**: Corrected `runtime.providerToPool` Map resolution in `ops-telemetry` simulator route, returning 404 for unrecognized providers.
+- **Config Import Concurrency (Issue #553)**: Added optimistic revision check via `expectedRevision` in `handleConfigImport`, returning 409 Conflict upon concurrent modification.
+- **Config Import Fail-Closed (Issue #533)**: `handleConfigImport` fails closed with 503 Service Unavailable when `settings.replace` is unavailable, preventing false-positive `{ applied: true }`.
+- **Instance & Context Isolation (Issue #544)**: Introduced `KeyDrainManager`, `SelectionTraceStore`, and `DiagBuffer` classes with per-context lifecycle binding; disposal of one plugin instance cleanly purges only its own draining keys and telemetry without corrupting peer instances.
+
 ## 0.8.73 (2026-10-30)
 
 ### Security & Hardening (Block 1 Security Release)

package/lib/clone-lifecycle.js

@@ -148,7 +148,7 @@ export class CloneLifecycleManager {
   startDrain(cloneId, provider, onDeregister) {
     if (this.drainingClones.has(cloneId)) return;
 
-    markKeyDraining(cloneId, provider);
+    markKeyDraining(cloneId, provider, this.keyDrainManager);
     const entry = {
       cloneId,
       provider,
@@ -176,7 +176,7 @@ export class CloneLifecycleManager {
 
     if (inFlight === 0) {
       this.drainingClones.delete(cloneId);
-      cancelKeyDrain(cloneId);
+      cancelKeyDrain(cloneId, this.keyDrainManager);
       if (this.logger?.info) {
         this.logger.info(`[dsh-key-rotation] clone ${cloneId} (${entry.provider}) drained cleanly; safe to deregister`);
       }
@@ -191,11 +191,11 @@ export class CloneLifecycleManager {
 
   cancelDrain(cloneId) {
     this.drainingClones.delete(cloneId);
-    cancelKeyDrain(cloneId);
+    cancelKeyDrain(cloneId, this.keyDrainManager);
   }
 
   isDraining(cloneId) {
-    return this.drainingClones.has(cloneId) || isKeyDraining(cloneId);
+    return this.drainingClones.has(cloneId) || isKeyDraining(cloneId, this.keyDrainManager);
   }
 
   getDrainingList() {
@@ -209,7 +209,7 @@ export class CloneLifecycleManager {
 
   destroy() {
     for (const cloneId of this.drainingClones.keys()) {
-      cancelKeyDrain(cloneId);
+      cancelKeyDrain(cloneId, this.keyDrainManager);
     }
     this.drainingClones.clear();
   }

package/lib/index.js

@@ -1,3 +1,6 @@
+import { KeyDrainManager } from './key-drain.js';
+import { SelectionTraceStore } from './selection-trace.js';
+import { DiagBuffer } from './ops-telemetry.js';
 import { createResolver } from './resolver.js';
 import { decryptSecret } from './crypto-storage.js';
 // dsh-key-rotation — per-provider API key rotation for DeepSeek Harness.
@@ -98,7 +101,7 @@ const SettingsSchema = Schema.object({
   rateLimitThreshold: Schema.number().default(0.1),
   proactiveRateLimitGuard: Schema.boolean().default(true),
   selfHealingIntervalMinutes: Schema.number().default(30),
-  routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky']).default('round-robin'),
+  routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky', 'tier-priority']).default('round-robin'),
   rpmLimit: Schema.number().default(0),
   tpmLimit: Schema.number().default(0),
   webhookActionToken: Schema.string().role('secret').default(''),
@@ -148,7 +151,7 @@ const SettingsSchema = Schema.object({
     })).default({}),
     cooldownMs: Schema.number(),
     maxCooldownMs: Schema.number(),
-    routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky']),
+    routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky', 'tier-priority']),
     proactiveRateLimitGuard: Schema.boolean(),
     rpmLimit: Schema.number(),
     tpmLimit: Schema.number(),
@@ -196,13 +199,16 @@ export function apply(ctx, config = {}) {
   const latencyHistogram = new LatencyHistogram();
   const quotaStore = new QuotaStore();
   const concurrencyTracker = new ConcurrencyTracker();
+  const keyDrainManager = new KeyDrainManager();
+  const selectionTraceStore = new SelectionTraceStore();
+  const diagBuffer = new DiagBuffer();
 
   const moduleBreaker = new CircuitBreaker({ threshold: 5, openMs: 30000, halfOpenProbes: 1, now: nowMono });
   const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
   const alertDebouncer = new AlertDebouncer({ sender: webhookSender, debounceMs: 2000 });
   const moduleNotifyQueue = new NotifyQueue({ send: (url, payload) => alertDebouncer.enqueue(url, payload) });
   const trafficSmoother = new TrafficSmoother({ maxQueueWaitMs: 800 });
-  const cloneLifecycle = new CloneLifecycleManager({ concurrencyTracker, logger });
+  const cloneLifecycle = new CloneLifecycleManager({ concurrencyTracker, logger, keyDrainManager });
   ctx.effect(() => () => { cloneLifecycle.destroy(); }, 'dsh-key-rotation: clone lifecycle cleanup');
 
   const sandboxService = createSandboxService({ getRuntime: () => buildRuntime() });
@@ -214,7 +220,11 @@ export function apply(ctx, config = {}) {
 
   setupIdleHealEffect(ctx, getConfig, buildRuntime);
   setupProactiveHealthProbeEffect(ctx, getConfig, buildRuntime, ensureSandboxRunner, logger);
-  setupKeyDrainEffect(ctx);
+  setupKeyDrainEffect(ctx, keyDrainManager);
+  ctx.effect(() => () => {
+    selectionTraceStore.clear();
+    diagBuffer.clear();
+  }, 'dsh-key-rotation: telemetry cleanup');
 
   const poolState = new Map();
   const { schedulePersist, persistenceSnapshot } = setupPersistence(ctx, {
@@ -461,6 +471,9 @@ export function apply(ctx, config = {}) {
     circuitBreaker: moduleBreaker,
     notifyQueue: moduleNotifyQueue,
     quotaStore,
+    keyDrainManager,
+    selectionTraceStore,
+    diagBuffer,
   });
 
   ctx.effect(() => {

package/lib/key-drain.js

@@ -1,56 +1,91 @@
-// lib/key-drain.js — Managed key draining and graceful replacement (#463, #502).
+// lib/key-drain.js — Managed key draining and graceful replacement (#463, #502, #544).
 // Allows draining an old key ref (no new dispatches assigned) while letting
 // in-flight active streams complete cleanly before permanent removal.
+// KeyDrainManager provides instance-level isolation (#544).
 
-const drainingKeys = new Map(); // ref -> { provider, startedAt, inFlightCount }
+export class KeyDrainManager {
+  constructor() {
+    this.drainingKeys = new Map(); // ref -> { provider, startedAt, inFlightCount }
+  }
+
+  markKeyDraining(ref, provider = '') {
+    if (!ref) return false;
+    this.drainingKeys.set(ref, {
+      ref,
+      provider,
+      startedAt: Date.now(),
+    });
+    return true;
+  }
+
+  isKeyDraining(ref) {
+    if (!ref) return false;
+    return this.drainingKeys.has(ref);
+  }
+
+  cancelKeyDrain(ref) {
+    if (!ref) return false;
+    return this.drainingKeys.delete(ref);
+  }
+
+  getDrainingStatus(ref, concurrencyTracker) {
+    if (!ref || !this.drainingKeys.has(ref)) return null;
+    const entry = this.drainingKeys.get(ref);
+    const inFlight = concurrencyTracker && typeof concurrencyTracker.getActive === 'function'
+      ? concurrencyTracker.getActive(ref)
+      : (concurrencyTracker && typeof concurrencyTracker.getInFlight === 'function'
+        ? concurrencyTracker.getInFlight(ref)
+        : 0);
+
+    return {
+      ref,
+      provider: entry.provider,
+      startedAt: entry.startedAt,
+      elapsedMs: Date.now() - entry.startedAt,
+      inFlight,
+      canComplete: inFlight === 0,
+    };
+  }
+
+  getAllDrainingKeys(concurrencyTracker) {
+    const result = [];
+    for (const ref of this.drainingKeys.keys()) {
+      result.push(this.getDrainingStatus(ref, concurrencyTracker));
+    }
+    return result;
+  }
 
-export function markKeyDraining(ref, provider = '') {
-  if (!ref) return false;
-  drainingKeys.set(ref, {
-    ref,
-    provider,
-    startedAt: Date.now(),
-  });
-  return true;
+  clearAllDraining() {
+    this.drainingKeys.clear();
+  }
 }
 
-export function isKeyDraining(ref) {
-  if (!ref) return false;
-  return drainingKeys.has(ref);
+const defaultDrainManager = new KeyDrainManager();
+
+export function markKeyDraining(ref, provider = '', manager = null) {
+  return (manager || defaultDrainManager).markKeyDraining(ref, provider);
 }
 
-export function cancelKeyDrain(ref) {
-  if (!ref) return false;
-  return drainingKeys.delete(ref);
+export function isKeyDraining(ref, manager = null) {
+  return (manager || defaultDrainManager).isKeyDraining(ref);
 }
 
-export function getDrainingStatus(ref, concurrencyTracker) {
-  if (!ref || !drainingKeys.has(ref)) return null;
-  const entry = drainingKeys.get(ref);
-  const inFlight = concurrencyTracker && typeof concurrencyTracker.getActive === 'function'
-    ? concurrencyTracker.getActive(ref)
-    : (concurrencyTracker && typeof concurrencyTracker.getInFlight === 'function'
-      ? concurrencyTracker.getInFlight(ref)
-      : 0);
-
-  return {
-    ref,
-    provider: entry.provider,
-    startedAt: entry.startedAt,
-    elapsedMs: Date.now() - entry.startedAt,
-    inFlight,
-    canComplete: inFlight === 0,
-  };
+export function cancelKeyDrain(ref, manager = null) {
+  return (manager || defaultDrainManager).cancelKeyDrain(ref);
 }
 
-export function getAllDrainingKeys(concurrencyTracker) {
-  const result = [];
-  for (const ref of drainingKeys.keys()) {
-    result.push(getDrainingStatus(ref, concurrencyTracker));
-  }
-  return result;
+export function getDrainingStatus(ref, concurrencyTracker, manager = null) {
+  return (manager || defaultDrainManager).getDrainingStatus(ref, concurrencyTracker);
+}
+
+export function getAllDrainingKeys(concurrencyTracker, manager = null) {
+  return (manager || defaultDrainManager).getAllDrainingKeys(concurrencyTracker);
 }
 
-export function clearAllDraining() {
-  drainingKeys.clear();
+export function clearAllDraining(manager = null) {
+  if (manager) {
+    manager.clearAllDraining();
+  } else {
+    defaultDrainManager.clearAllDraining();
+  }
 }

package/lib/lifecycle.js

@@ -251,10 +251,10 @@ export function setupProactiveHealthProbeEffect(ctx, getConfig, buildRuntime, en
   }, 'dsh-key-rotation: proactive health probing');
 }
 
-export function setupKeyDrainEffect(ctx) {
+export function setupKeyDrainEffect(ctx, keyDrainManager = null) {
   return ctx.effect(() => {
     return () => {
-      clearAllDraining();
+      clearAllDraining(keyDrainManager);
     };
   }, 'dsh-key-rotation: clear draining keys on teardown');
 }

package/lib/ops-config-io.js

@@ -224,7 +224,7 @@ export function registerConfigIoRoutes(ctx, deps) {
         }
       }
 
-      const settings = ctx.get('settings');
+      const settings = typeof ctx?.get === 'function' ? ctx.get('settings') : (ctx?.settings || null);
       let currentConfig = {};
       try {
         if (settings && typeof settings.get === 'function') {
@@ -322,28 +322,54 @@ export function registerConfigIoRoutes(ctx, deps) {
     const mode = body.mode === 'replace' ? 'replace' : 'merge';
     const dryRun = Boolean(body.dryRun);
 
-    const settings = ctx.get('settings');
+    const settings = typeof ctx?.get === 'function' ? ctx.get('settings') : (ctx?.settings || null);
+    if (!settings || typeof settings.replace !== 'function') {
+      // Fail closed when settings service is missing or lacks replace (#533)
+      json(res, 503, { error: { code: 'settings-unavailable', message: 'Settings service is not available or does not support writing configuration' } });
+      return;
+    }
+
     let currentConfig = {};
+    let currentRevision = null;
     try {
-      if (settings && typeof settings.get === 'function') {
-        currentConfig = settings.get(NS) || {};
-      } else if (settings && typeof settings.describe === 'function') {
+      if (typeof settings.describe === 'function') {
         const desc = settings.describe({ redactSecrets: false })?.find((c) => c.ns === NS);
-        currentConfig = desc?.value || {};
-      } else {
-        currentConfig = deps.buildRuntime().cfg0 || {};
+        if (desc) {
+          currentConfig = desc.value || {};
+          currentRevision = desc.revision ?? null;
+        }
+      }
+      if (!currentConfig || Object.keys(currentConfig).length === 0) {
+        if (typeof settings.get === 'function') {
+          currentConfig = settings.get(NS) || {};
+        }
       }
     } catch (_) {
       currentConfig = deps.buildRuntime().cfg0 || {};
     }
     const rawCurrent = plainConfig(currentConfig);
 
+    // Concurrency control: if client provided expectedRevision, reject if stale (#553)
+    if (typeof body.expectedRevision === 'number' && currentRevision !== null) {
+      if (body.expectedRevision !== currentRevision) {
+        json(res, 409, {
+          error: {
+            code: 'conflict',
+            message: `Configuration has been modified concurrently (expected revision ${body.expectedRevision}, current is ${currentRevision})`,
+            currentRevision,
+          }
+        });
+        return;
+      }
+    }
+
     const preview = diffConfigs(rawCurrent, incomingConfig, mode);
 
     if (dryRun) {
       json(res, 200, {
         ok: true,
         dryRun: true,
+        revision: currentRevision,
         preview,
       });
       return;
@@ -379,19 +405,18 @@ export function registerConfigIoRoutes(ctx, deps) {
       mergedConfig.providers = curProviders;
     }
 
-    if (settings && typeof settings.replace === 'function') {
-      try {
-        await settings.replace(NS, mergedConfig);
-      } catch (err) {
-        json(res, 500, { error: { code: 'settings-write-failed', message: err.message } });
-        return;
-      }
+    try {
+      await settings.replace(NS, mergedConfig, currentRevision ?? undefined);
+    } catch (err) {
+      json(res, 500, { error: { code: 'settings-write-failed', message: err.message } });
+      return;
     }
 
     json(res, 200, {
       ok: true,
       applied: true,
       mode,
+      revision: typeof currentRevision === 'number' ? currentRevision + 1 : undefined,
       preview,
     });
   };

package/lib/ops-keys.js

@@ -165,17 +165,18 @@ ctx.effect(() => ctx.webServer.register({
       return;
     }
     const concurrency = deps.concurrencyTracker;
+    const keyDrainManager = deps.keyDrainManager;
 
     if (req.method === 'GET') {
       const url = new URL(req.url ?? DRAIN_PATH, 'http://localhost');
       const ref = url.searchParams.get('ref');
       if (ref) {
-        const status = getDrainingStatus(ref, concurrency);
+        const status = getDrainingStatus(ref, concurrency, keyDrainManager);
         if (!status) { json(res, 404, { error: { code: 'not-found', message: `Key ${ref} is not currently draining` } }); return; }
         json(res, 200, { ok: true, draining: status });
         return;
       }
-      json(res, 200, { ok: true, draining: getAllDrainingKeys(concurrency) });
+      json(res, 200, { ok: true, draining: getAllDrainingKeys(concurrency, keyDrainManager) });
       return;
     }
 
@@ -187,8 +188,8 @@ ctx.effect(() => ctx.webServer.register({
       }
       const ref = body?.ref;
       if (!ref) { json(res, 400, { error: { code: 'bad-request', message: 'Missing ref' } }); return; }
-      markKeyDraining(ref, body?.provider || '');
-      json(res, 200, { ok: true, status: getDrainingStatus(ref, concurrency) });
+      markKeyDraining(ref, body?.provider || '', keyDrainManager);
+      json(res, 200, { ok: true, status: getDrainingStatus(ref, concurrency, keyDrainManager) });
       return;
     }
 
@@ -196,7 +197,7 @@ ctx.effect(() => ctx.webServer.register({
       const url = new URL(req.url ?? DRAIN_PATH, 'http://localhost');
       const ref = url.searchParams.get('ref');
       if (!ref) { json(res, 400, { error: { code: 'bad-request', message: 'Missing ref query parameter' } }); return; }
-      const cancelled = cancelKeyDrain(ref);
+      const cancelled = cancelKeyDrain(ref, keyDrainManager);
       json(res, 200, { ok: true, cancelled });
       return;
     }

package/lib/ops-quick-action.js

@@ -66,10 +66,11 @@ export function registerQuickActionRoutes(ctx, deps) {
           json(res, 400, { error: { code: 'no-keys', message: 'Pool has no keys to rotate' } });
           return;
         }
-        const currentCursor = targetPool.state.cursor ?? 0;
-        const nextCursor = (currentCursor + 1) % total;
-        targetPool.state.cursor = nextCursor;
-        const nextRef = targetPool.refs[nextCursor];
+        const currentPointer = targetPool.state.pointer ?? targetPool.state.cursor ?? 0;
+        const nextPointer = (currentPointer + 1) % total;
+        targetPool.state.pointer = nextPointer;
+        targetPool.state.cursor = nextPointer;
+        const nextRef = targetPool.refs[nextPointer];
         targetPool.state.lastUsed = nextRef;
 
         if (typeof pushEvent === 'function') {
@@ -80,7 +81,7 @@ export function registerQuickActionRoutes(ctx, deps) {
           ok: true,
           action: 'skip',
           provider,
-          cursor: nextCursor,
+          cursor: nextPointer,
           activeRef: nextRef,
           activeTail: keyTail(nextRef),
         });

package/lib/ops-telemetry.js

@@ -30,20 +30,40 @@ import { getPromptCacheSummary } from './prompt-cache.js';
  */
 
 const DIAG_BUFFER_LIMIT = 200;
-const diagBuffer = [];
 
-export function pushDiag(entry) {
-  if (diagBuffer.length >= DIAG_BUFFER_LIMIT) {
-    diagBuffer.shift();
+export class DiagBuffer {
+  constructor(limit = DIAG_BUFFER_LIMIT) {
+    this.limit = limit;
+    this.buffer = [];
   }
-  diagBuffer.push({
-    timestamp: Date.now(),
-    ...entry,
-  });
+
+  push(entry) {
+    if (this.buffer.length >= this.limit) {
+      this.buffer.shift();
+    }
+    this.buffer.push({
+      timestamp: Date.now(),
+      ...entry,
+    });
+  }
+
+  get() {
+    return this.buffer.slice();
+  }
+
+  clear() {
+    this.buffer.length = 0;
+  }
+}
+
+const defaultDiagBuffer = new DiagBuffer();
+
+export function pushDiag(entry, buffer = null) {
+  (buffer || defaultDiagBuffer).push(entry);
 }
 
-export function getDiagBuffer() {
-  return diagBuffer.slice();
+export function getDiagBuffer(buffer = null) {
+  return (buffer || defaultDiagBuffer).get();
 }
 
 export function registerTelemetryRoutes(ctx, deps) {
@@ -185,11 +205,25 @@ ctx.effect(() => ctx.webServer.register({
     }
 
     const runtime = buildRuntime();
-    const provider = body?.provider || (runtime.pools?.[0]?.base ?? '');
-    const pool = runtime.providerToPool?.[provider] || runtime.pools?.[0];
+    const provider = body?.provider?.trim();
+    if (!provider) {
+      json(res, 400, { error: { code: 'bad-request', message: 'provider is required for simulation' } });
+      return;
+    }
+
+    let pool = null;
+    if (runtime.providerToPool instanceof Map) {
+      pool = runtime.providerToPool.get(provider);
+    } else if (runtime.providerToPool && typeof runtime.providerToPool === 'object') {
+      pool = runtime.providerToPool[provider];
+    }
+    if (!pool) {
+      const pools = runtime.pools ?? [];
+      pool = pools.find(p => p.provider === provider || p.base === provider);
+    }
 
     if (!pool) {
-      json(res, 404, { error: { code: 'not-found', message: `Provider '${provider}' not found` } });
+      json(res, 404, { error: { code: 'not-found', message: `Provider '${provider}' not found in configured pools` } });
       return;
     }
 
@@ -234,7 +268,7 @@ ctx.effect(() => ctx.webServer.register({
       json(res, 200, { ok: true, count: diagBuffer.length });
       return;
     }
-    json(res, 200, { diag: getDiagBuffer() });
+    json(res, 200, { diag: deps.diagBuffer ? deps.diagBuffer.get() : getDiagBuffer() });
   },
 }), 'dsh-key-rotation: diag route');
 

package/lib/pool-builder.js

@@ -89,7 +89,7 @@ export function buildPoolItem({
   
   const tiersMap = {};
   for (let i = 0; i < refs.length; i++) {
-    const val = Array.isArray(tiers) ? tiers[indexed[i].index] : (tiers && typeof tiers ===  object ? tiers[refs[i]] : undefined);
+    const val = Array.isArray(tiers) ? tiers[indexed[i].index] : (tiers && typeof tiers === 'object' ? tiers[refs[i]] : undefined);
     tiersMap[refs[i]] = Number.isFinite(val) && val >= 1 ? Math.floor(val) : 1;
   }
 const weightsMap = {};
@@ -182,6 +182,7 @@ export function buildPools({ cfg = {}, poolState = new Map(), defaultState = ini
       provider: p.provider,
       model: null,
       labels: p.labels,
+      tiers: p.tiers,
       pinnedRef: p.pinnedRef ?? p.useKeyRef,
       trafficSmoothingBurst: p.trafficSmoothingBurst ?? cfg.trafficSmoothingBurst,
       maxQueueWaitMs: p.maxQueueWaitMs ?? cfg.maxQueueWaitMs,

package/lib/pool.js

@@ -468,14 +468,20 @@ export function isSwitchableError(failureOrPayload, switchCodes = new Set(DEFAUL
   // and DEFAULT_SWITCH_CODES is exported as an array); normalize before .has().
   if (!(switchCodes instanceof Set)) switchCodes = new Set(Array.isArray(switchCodes) ? switchCodes : DEFAULT_SWITCH_CODES);
   const failure = failureOrPayload.failure ?? failureOrPayload;
-  const status = Number(failure.status ?? failure.statusCode ?? failure.httpStatus ?? 0);
-  let code = String(failure.code ?? failure.reason ?? '').toUpperCase();
+  let status = Number(failure.status ?? failure.statusCode ?? failure.httpStatus ?? 0);
+  let rawCode = String(failure.code ?? failure.reason ?? '').trim();
+  let code = rawCode.toUpperCase();
   if (code && SOCKET_TRANSPORT_CODES.has(code)) code = 'TRANSPORT';
   const message = String(failure.message ?? failureOrPayload.message ?? '');
 
-  // 1. Direct HTTP status codes (#267: also 408/425 transient)
+  // Treat numeric failure.code (e.g. code: "429") as HTTP status if status was not set (#545)
+  if (!status && /^\d{3}$/.test(rawCode)) {
+    status = Number(rawCode);
+  }
+
+  // 1. Direct HTTP status codes (#267: also 408/425 transient, #545: numeric status/code mapped to semantic switch codes)
   if ((status === 429 || status === 408 || status === 425) && (switchCodes.has('RATE_LIMIT') || switchCodes.has('QUOTA') || switchCodes.has('429') || switchCodes.has('TIMEOUT') || switchCodes.has(String(status)))) return true;
-  if ((status === 401 || status === 403) && (switchCodes.has('AUTH') || switchCodes.has('401'))) return true;
+  if ((status === 401 || status === 403) && (switchCodes.has('AUTH') || switchCodes.has('401') || switchCodes.has('403') || switchCodes.has(String(status)))) return true;
   if ((status >= 500 && status <= 504) && (switchCodes.has('SERVER') || switchCodes.has(String(status)))) return true;
 
   // 2. Standard gRPC / cloud error codes

package/lib/resolver.js

@@ -121,7 +121,7 @@ export function createResolver({
     const strategy = pool.routingStrategy ?? runtime.routingStrategy ?? 'round-robin';
     const sessionId = resolveOpts?.sessionId ?? resolveOpts?.session ?? resolveOpts?.metadata?.sessionId ?? currentSession?.();
     let list = pool.weightedRefs ?? pool.refs;
-    if (strategy === 'lowest-latency' || strategy === 'least-loaded' || strategy === 'sticky') {
+    if (strategy === 'lowest-latency' || strategy === 'least-loaded' || strategy === 'sticky' || strategy === 'tier-priority') {
       list = sortAttemptList(list, strategy, {
         latencyHistogram,
         concurrencyTracker,

package/lib/rotate.js

@@ -80,6 +80,8 @@ export function createRotate(deps) {
     logger,
     schedulePersist,
     trafficSmoother,
+    keyDrainManager,
+    selectionTraceStore,
     now = nowMono,
   } = deps;
 
@@ -102,7 +104,7 @@ export function createRotate(deps) {
       const trace = options.__selectionTrace || createSelectionTrace(options.requestId, {
         provider: options.provider,
         model: options.model,
-      });
+      }, selectionTraceStore);
       const isTraceOwner = !options.__selectionTrace;
       const { switchCodes, cooldownMs, maxCooldownMs, switchNotify, rateLimitThreshold } = runtime0;
       let lastFailure = null;
@@ -114,7 +116,7 @@ export function createRotate(deps) {
       const wallNow = Date.now();
       const quotaManaged = hasModelQuotaConfig(pool);
       let attemptList = (pool.weightedRefs ?? pool.refs).slice();
-      attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r) && !isKeyBroken(pool, r, wallNow) && !isKeyDraining(r));
+      attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r) && !isKeyBroken(pool, r, wallNow) && !isKeyDraining(r, keyDrainManager));
       // Issue #500: filter candidate keys by confirmed model capabilities
       if (options.model && pool.state.modelCapabilities) {
         attemptList = attemptList.filter((ref) => {

package/lib/selection-trace.js

@@ -1,10 +1,33 @@
-// lib/selection-trace.js — Request-scoped key selection explanation and attempt history (#461).
+// lib/selection-trace.js — Request-scoped key selection explanation and attempt history (#461, #544).
 // Provides auditable reasoning for key skips, selection, failovers, and cascade with strict secret redaction.
 
 import { keyTail } from './pool.js';
 
 const TRACE_BUFFER_LIMIT = 100;
-const traces = [];
+
+export class SelectionTraceStore {
+  constructor(limit = TRACE_BUFFER_LIMIT) {
+    this.limit = limit;
+    this.traces = [];
+  }
+
+  push(summary) {
+    if (this.traces.length >= this.limit) {
+      this.traces.shift();
+    }
+    this.traces.push(summary);
+  }
+
+  get(limit = 50) {
+    return this.traces.slice(-Math.max(1, Math.min(this.limit, limit)));
+  }
+
+  clear() {
+    this.traces.length = 0;
+  }
+}
+
+const defaultSelectionTraceStore = new SelectionTraceStore();
 
 /**
  * Redacts any sensitive tokens from string / object representation (#537).
@@ -30,10 +53,11 @@ export function sanitizeTraceDetails(str) {
     .slice(0, 300);
 }
 
-export function createSelectionTrace(requestId, options = {}) {
+export function createSelectionTrace(requestId, options = {}, store = null) {
   const reqId = requestId || `req-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`;
   const startTime = Date.now();
   const events = [];
+  const targetStore = store || defaultSelectionTraceStore;
 
   return {
     requestId: reqId,
@@ -73,19 +97,16 @@ export function createSelectionTrace(requestId, options = {}) {
         steps: events,
       };
 
-      if (traces.length >= TRACE_BUFFER_LIMIT) {
-        traces.shift();
-      }
-      traces.push(summary);
+      targetStore.push(summary);
       return summary;
     }
   };
 }
 
-export function getSelectionTraces(limit = 50) {
-  return traces.slice(-Math.max(1, Math.min(100, limit)));
+export function getSelectionTraces(limit = 50, store = null) {
+  return (store || defaultSelectionTraceStore).get(limit);
 }
 
-export function clearSelectionTraces() {
-  traces.length = 0;
+export function clearSelectionTraces(store = null) {
+  (store || defaultSelectionTraceStore).clear();
 }

package/lib/simulator.js

@@ -37,18 +37,24 @@ export function simulateRouting(params = {}) {
     throw new Error('simulateRouting: valid pool with refs is required');
   }
 
-  // Clone pool state into isolated mock state so real runtime is never touched
+  // Clone pool state into isolated mock state so real runtime is never touched (#534)
   const mockFailedUntil = new Map(pool.state?.failedUntil || []);
   const mockFailCounts = new Map(pool.state?.failCounts || []);
   const mockRevokedRefs = new Set(pool.state?.revokedRefs || []);
+  const mockBrokenUntil = new Map(pool.state?.brokenUntil || []);
+  const mockTokenUsage = pool.state?.tokenUsage ? new Map(pool.state.tokenUsage) : new Map();
+  const mockQuotaWindows = pool.state?.quotaWindows ? new Map(pool.state.quotaWindows) : new Map();
 
   const simPool = {
     ...pool,
     state: {
+      ...pool.state,
       failedUntil: mockFailedUntil,
       failCounts: mockFailCounts,
       revokedRefs: mockRevokedRefs,
-      brokenUntil: new Map(),
+      brokenUntil: mockBrokenUntil,
+      tokenUsage: mockTokenUsage,
+      quotaWindows: mockQuotaWindows,
     },
   };
 

package/lib/traffic-smoother.js

@@ -1,21 +1,26 @@
-// lib/traffic-smoother.js — leaky bucket micro-queue traffic smoother (#496).
+// lib/traffic-smoother.js — GCRA leaky bucket micro-queue traffic smoother (#496, #538).
 // Absorbs momentary request bursts without tripping upstream per-second/per-minute rate limits.
+// Uses Generic Cell Rate Algorithm to ensure atomic slot reservation and burst compliance.
 
 export class TrafficSmoother {
   constructor({ maxQueueWaitMs = 800, now = Date.now, sleep = null } = {}) {
     this.maxQueueWaitMs = maxQueueWaitMs;
     this.now = now;
     this.sleep = sleep || ((ms, signal) => new Promise((resolve, reject) => {
-      if (signal?.aborted) return reject(signal.reason ?? new Error('aborted'));
+      if (signal?.aborted) {
+        const err = (signal.reason instanceof Error) ? signal.reason : Object.assign(new Error('This operation was aborted'), { name: 'AbortError' });
+        return reject(err);
+      }
       const timer = setTimeout(resolve, ms);
       if (signal) {
         signal.addEventListener('abort', () => {
           clearTimeout(timer);
-          reject(signal.reason ?? new Error('aborted'));
+          const err = (signal.reason instanceof Error) ? signal.reason : Object.assign(new Error('This operation was aborted'), { name: 'AbortError' });
+          reject(err);
         }, { once: true });
       }
     }));
-    // Map<ref, { lastTime: number, tokens: number, lastUpdate: number }>
+    // Map<ref, { virtualTime: number }>
     this._buckets = new Map();
   }
 
@@ -45,35 +50,41 @@ export class TrafficSmoother {
 
     let bucket = this._buckets.get(ref);
     if (!bucket) {
-      bucket = { tokens: capacity, lastUpdate: curNow, nextSlot: curNow };
+      bucket = { virtualTime: curNow - capacity * intervalMs };
       this._buckets.set(ref, bucket);
     }
 
-    // Replenish tokens based on elapsed time
-    const elapsed = Math.max(0, curNow - bucket.lastUpdate);
-    const addedTokens = elapsed * (ratePerSec / 1000);
-    bucket.tokens = Math.min(capacity, bucket.tokens + addedTokens);
-    bucket.lastUpdate = curNow;
+    // GCRA: virtualTime cannot fall behind (curNow - capacity * intervalMs)
+    const minVirtualTime = curNow - capacity * intervalMs;
+    const vTime = Math.max(minVirtualTime, bucket.virtualTime);
+    const targetSlot = vTime + intervalMs;
 
-    if (bucket.tokens >= 1) {
-      bucket.tokens -= 1;
-      bucket.nextSlot = Math.max(curNow, bucket.nextSlot);
+    if (targetSlot <= curNow) {
+      // Immediate acquisition within available burst capacity
+      bucket.virtualTime = targetSlot;
       return { allowed: true, delayed: false, delayMs: 0, overflow: false };
     }
 
-    // Burst is exhausted: calculate delay required for next available slot
-    const targetSlot = Math.max(curNow, bucket.nextSlot) + intervalMs;
-    const delayMs = Math.max(0, targetSlot - curNow);
+    // Queue delay required
+    const delayMs = targetSlot - curNow;
 
     if (delayMs > maxWait) {
       // Exceeds max micro-queue wait: signal overflow so caller rotates to next candidate
       return { allowed: false, delayed: false, delayMs, overflow: true };
     }
 
-    bucket.nextSlot = targetSlot;
+    // Atomically reserve the slot ahead of sleeping (#538)
+    bucket.virtualTime = targetSlot;
 
-    // Wait for the scheduled slot
-    await this.sleep(delayMs, signal);
+    try {
+      await this.sleep(delayMs, signal);
+    } catch (err) {
+      // Rollback reservation if this was the last queued request
+      if (bucket.virtualTime === targetSlot) {
+        bucket.virtualTime = Math.max(minVirtualTime, targetSlot - intervalMs);
+      }
+      throw err;
+    }
 
     return { allowed: true, delayed: true, delayMs, overflow: false };
   }
@@ -83,7 +94,7 @@ export class TrafficSmoother {
     const b = this._buckets.get(ref);
     if (!b) return 0;
     const curNow = this.now();
-    const wait = Math.max(0, b.nextSlot - curNow);
+    const wait = Math.max(0, b.virtualTime - curNow);
     return wait > 0 ? Math.ceil(wait / 100) : 0;
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.73",
+  "version": "0.8.74",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [