@goodandready/dsh-key-rotation 0.8.56 → 0.8.57

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## 0.8.57 - 2026-10-12
+
+### Added
+- **Tier-based / Cost-aware Routing (`tier-priority`) (#475)**:
+  - Added `tier-priority` routing strategy and optional per-key `tiers` configuration.
+  - Automatically sorts and routes requests through tier 1 keys first, gracefully falling back to tier 2 only when primary tier keys are cooling down or exhausted.
+- **Proactive Health Probing (#475)**:
+  - Added opt-in `proactiveHealthProbing` and `proactiveProbeIntervalMs` background lifecycle effect.
+  - Automatically sends lightweight probe pings for keys in cooldown or quarantine to restore healthy status proactively without waiting for user failover.
+- **RPM / TPM Micro-Throttling (#475)**:
+  - Added opt-in `microThrottling` and `microThrottleMaxMs` options.
+  - If a key hits rate limits but the sliding window resets within `microThrottleMaxMs` (default 1000ms), resolves inline micro-sleep rather than failing over prematurely.
+- **Early-Warning Alerts (#475)**:
+  - Added `earlyWarningNotify` and `earlyWarningKeyThreshold` options.
+  - Automatically dispatches webhook alerts when available healthy keys drop below threshold (e.g. 25%) before pool-wide exhaustion.
+- **Interactive Routing Simulator UI (#475)**:
+  - Integrated an interactive simulator section into Web Settings UI backed by `/dsh-key-rotation/simulate`.
+  - Enables playground testing of rotation, synthetic rate limits, and failure fallback traces without sending live upstream requests.
+
 ## 0.8.56 - 2026-10-12
 
 ### Fixed

package/lib/client.js

@@ -44,6 +44,19 @@ window.__ModuleLoader__.load({
       'header.sub': 'Automated multi-key round-robin rotation, 429/5xx exponential backoff cooldowns, circuit breaker failover protection, and credential security (~/.dsh/.credentials.yaml).',
       'header.pools_badge': 'pools active',
       'header.keys_badge': 'keys configured',
+            simulatorTitle: 'Interactive Routing Simulator',
+      simulatorDesc: 'Test routing strategies and visualize failover trace across key pools without sending real upstream requests.',
+      simulatorRun: 'Run Simulation',
+      simulatorRunning: 'Simulating...',
+      simulatorSelectProvider: 'Provider',
+      simulatorSelectModel: 'Model (optional)',
+      simulatorMaxAttempts: 'Max Attempts',
+      simulatorEvent: 'Simulated Outcome',
+      simulatorEventSuccess: 'All keys succeed',
+      simulatorEventRateLimit: 'Key 1 Rate Limited (429)',
+      simulatorEventServer500: 'Key 1 Internal Error (500)',
+      simulatorResults: 'Simulation Trace',
+      simulatorNoResults: 'Run a simulation to view the candidate execution trace.',
       sectionStats: 'Pool Health & Telemetry',
       sectionStatsDesc: 'Real-time telemetry of active key pools, healthy credentials, and failover status.',
       sectionFailover: 'Failover Triggers & Error Codes',
@@ -1103,6 +1116,50 @@ window.__ModuleLoader__.load({
       const [search, setSearch] = React.useState('');
       const [statusFilter, setStatusFilter] = React.useState('all');
       const [confirmModal, setConfirmModal] = React.useState(null);
+      // Simulator State (#475)
+      const [simProvider, setSimProvider] = React.useState('');
+      const [simModel, setSimModel] = React.useState('');
+      const [simAttempts, setSimAttempts] = React.useState(3);
+      const [simScenario, setSimScenario] = React.useState('success');
+      const [simRunning, setSimRunning] = React.useState(false);
+      const [simResult, setSimResult] = React.useState(null);
+      const [simError, setSimError] = React.useState('');
+
+      const runSimulator = async () => {
+        const activeProv = simProvider || (providersList[0]?.provider || '');
+        if (!activeProv) return;
+        setSimRunning(true);
+        setSimError('');
+        try {
+          const events = [];
+          if (simScenario === 'rate_limit') {
+            events.push({ code: '429', message: 'Rate limit reached' });
+          } else if (simScenario === 'error_500') {
+            events.push({ code: '500', message: 'Internal Server Error' });
+          }
+          const res = await fetch('/dsh-key-rotation/simulate', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+              provider: activeProv,
+              model: simModel,
+              maxAttempts: Number(simAttempts) || 3,
+              events,
+            }),
+          });
+          const data = await res.json();
+          if (data && data.ok) {
+            setSimResult(data.simulation);
+          } else {
+            setSimError(data?.error?.message || 'Simulation error');
+          }
+        } catch (err) {
+          setSimError(String(err?.message || err));
+        } finally {
+          setSimRunning(false);
+        }
+      };
+
       const [optimisticReset, setOptimisticReset] = React.useState({});
       const [selected, setSelected] = React.useState(new Set());
       const [bulkCooldown, setBulkCooldown] = React.useState('');
@@ -2445,6 +2502,62 @@ window.__ModuleLoader__.load({
         )
       );
 
+      
+      
+      const configuredProviders = (draft?.providers ?? state.value?.providers ?? []);
+      const simulatorSection = h('div', { className: 'krot-section', style: { marginTop: '14px', border: '1px solid var(--dsw-alias-border-l2)', borderRadius: '8px', padding: '14px' } },
+        h('div', { style: { display: 'flex', alignItems: 'center', justifyContent: 'space-between', marginBottom: '8px' } },
+          h('div', null,
+            h('span', { style: { fontWeight: 600, fontSize: '13px' } }, '🧪 ' + t('simulatorTitle')),
+            h('div', { className: 'krot-hint', style: { fontSize: '11px', marginTop: '2px' } }, t('simulatorDesc'))
+          )
+        ),
+        h('div', { style: { display: 'flex', gap: '8px', flexWrap: 'wrap', alignItems: 'center', marginTop: '10px' } },
+          h('select', {
+            className: 'krot-input',
+            style: { width: '140px' },
+            value: simProvider || (configuredProviders[0]?.provider || ''),
+            onChange: (e) => setSimProvider(e.target.value),
+          }, configuredProviders.map((p) => h('option', { key: p.provider, value: p.provider }, p.provider))),
+          h('input', {
+            type: 'text',
+            className: 'krot-input',
+            style: { width: '120px' },
+            placeholder: t('simulatorSelectModel'),
+            value: simModel,
+            onChange: (e) => setSimModel(e.target.value),
+          }),
+          h('select', {
+            className: 'krot-input',
+            style: { width: '180px' },
+            value: simScenario,
+            onChange: (e) => setSimScenario(e.target.value),
+          },
+            h('option', { value: 'success' }, t('simulatorEventSuccess')),
+            h('option', { value: 'rate_limit' }, t('simulatorEventRateLimit')),
+            h('option', { value: 'error_500' }, t('simulatorEventServer500')),
+          ),
+          btn(simRunning ? t('simulatorRunning') : t('simulatorRun'), runSimulator, {
+            primary: true,
+            disabled: simRunning,
+          })
+        ),
+        simError ? h('div', { className: 'krot-warn', style: { marginTop: '8px', color: 'var(--dsw-alias-error)' } }, simError) : null,
+        simResult ? h('div', { style: { marginTop: '12px', background: 'var(--dsw-alias-bg-hover)', padding: '10px', borderRadius: '6px' } },
+          h('div', { style: { fontWeight: 600, fontSize: '12px', marginBottom: '6px' } }, t('simulatorResults') + ': ' + (simResult.status || 'DONE')),
+          h('div', { style: { fontSize: '11px', display: 'flex', flexDirection: 'column', gap: '4px' } },
+            Array.isArray(simResult.steps) ? simResult.steps.map((st, idx) =>
+              h('div', { key: idx, style: { display: 'flex', gap: '8px', fontFamily: 'monospace' } },
+                h('span', { style: { color: 'var(--dsw-alias-text-sub)' } }, '[' + (idx + 1) + ']'),
+                h('span', { style: { fontWeight: 500 } }, st.ref || '-'),
+                h('span', { style: { color: st.status === 'OK' ? 'var(--dsw-alias-label-primary)' : 'var(--dsw-alias-label-secondary)' } }, st.status),
+                h('span', { style: { opacity: 0.8 } }, st.reason || st.error || '')
+              )
+            ) : null
+          )
+        ) : null
+      );
+
       const backupSection = h('div', { className: 'krot-section-card' },
         h('div', { className: 'krot-section-title' },
           h('span', null, '💾 ' + t('sectionBackup')),
@@ -2555,6 +2668,7 @@ window.__ModuleLoader__.load({
           failoverSection,
           timingSection,
           backupSection,
+          simulatorSection,
         ),
         alerts.length > 0 ? h('div', { style: { display: 'flex', flexDirection: 'column', gap: '8px' } }, alerts) : null,
         h('div', { className: 'krot-foot', style: { marginTop: '6px', paddingTop: '14px', borderTop: '1px solid var(--dsw-alias-border-l2)' } },

package/lib/index.js

@@ -44,7 +44,7 @@ import {
   formatModelQuotaExhaustion,
 } from './model-quota.js';
 import { LOCAL_QUOTA_CODE } from './rotate.js';
-import { setupIdleHealEffect, setupAutoUnbreakEffect, setupPersistence } from './lifecycle.js';
+import { setupIdleHealEffect, setupAutoUnbreakEffect, setupPersistence, setupProactiveHealthProbeEffect } from './lifecycle.js';
 
 export const name = '@goodandready/dsh-key-rotation';
 export const inject = ['llm', 'webServer', 'settings', 'credentials'];
@@ -110,12 +110,19 @@ const SettingsSchema = Schema.object({
   persistenceEnabled: Schema.boolean().default(true),
   persistencePath: Schema.string().default(''),
   switchNotifyThrottleMs: Schema.number().default(60000),
+  proactiveHealthProbing: Schema.boolean().default(false),
+  proactiveProbeIntervalMs: Schema.number().default(60000),
+  microThrottling: Schema.boolean().default(false),
+  microThrottleMaxMs: Schema.number().default(1000),
+  earlyWarningNotify: Schema.boolean().default(false),
+  earlyWarningKeyThreshold: Schema.number().default(0.25),
   warnBelowHealthy: Schema.number().default(0),
   latencySloMs: Schema.number().default(0),
   providers: Schema.array(Schema.object({
     provider: Schema.string().required(),
     keys: Schema.array(Schema.string()).default([]),
     weights: Schema.array(Schema.number()).default([]),
+    tiers: Schema.array(Schema.number()).default([]),
     paused: Schema.array(Schema.boolean()).default([]),
     revoked: Schema.array(Schema.boolean()).default([]),
     expiresAt: Schema.array(Schema.union([Schema.number(), Schema.string()])).default([]),
@@ -198,6 +205,7 @@ export function apply(ctx, config = {}) {
   ensureSandboxRunner(ctx);
 
   setupIdleHealEffect(ctx, getConfig, buildRuntime);
+  setupProactiveHealthProbeEffect(ctx, getConfig, buildRuntime, ensureSandboxRunner, logger);
 
   const poolState = new Map();
   const { schedulePersist, persistenceSnapshot } = setupPersistence(ctx, {

package/lib/lifecycle.js

@@ -156,3 +156,91 @@ export function setupPersistence(ctx, { cfg0, poolState, moduleBreaker, verboseL
 
   return { schedulePersist, persistenceSnapshot };
 }
+
+
+
+export function setupProactiveHealthProbeEffect(ctx, getConfig, buildRuntime, ensureSandboxRunner, logger, timers = globalThis) {
+  return ctx.effect(() => {
+    let timer;
+    let running = false;
+    let disposed = false;
+
+    const run = async () => {
+      if (running || disposed) return;
+      running = true;
+      try {
+        const c = getConfig();
+        if (!c || c.proactiveHealthProbing !== true) return;
+        const { pools } = buildRuntime();
+        const runner = ensureSandboxRunner(ctx);
+        const now = Date.now();
+        for (const pool of (pools || [])) {
+          if (!pool || !pool.state) continue;
+          const fu = pool.state.failedUntil;
+          const bu = pool.state.brokenUntil;
+          const candidates = new Set();
+          if (fu) {
+            for (const [ref, until] of fu.entries()) {
+              if (until > now && until !== Infinity) candidates.add(ref);
+            }
+          }
+          if (bu) {
+            for (const [ref, until] of bu.entries()) {
+              if (until > now) candidates.add(ref);
+            }
+          }
+          for (const ref of candidates) {
+            if (disposed) break;
+            try {
+              const credentials = ctx.get?.('credentials') ?? ctx.credentials;
+              const resolve = credentials?.__dshKeyRotationOriginalResolve ?? credentials?.resolve;
+              const hit = await resolve?.call(credentials, ref);
+              const raw = typeof hit?.value === 'string' && hit.value.length > 0 ? hit.value : envValue(ref);
+              const val = raw ? decryptSecret(raw) : raw;
+              if (!val || typeof val !== 'string' || val.startsWith('enc:v1:')) continue;
+              const res = await runner.probeModels(ref, val);
+              if (res && res.ok) {
+                pool.state.failedUntil?.delete(ref);
+                pool.state.brokenUntil?.delete(ref);
+                pool.state.failCounts?.delete(ref);
+                if (Array.isArray(pool.state.events)) {
+                  pool.state.events.push({ at: Date.now(), ref, reason: 'proactive-probe-heal', cooldownMs: 0, type: 'heal' });
+                  if (pool.state.events.length > 50) pool.state.events.shift();
+                }
+                logger?.warn?.(`[dsh-key-rotation] proactive-probe: restored key ${ref} in pool ${pool.base}`);
+              }
+            } catch (_) {}
+          }
+        }
+      } catch (e) {
+        logger?.warn?.('[dsh-key-rotation] proactive probe failed', e);
+      } finally {
+        running = false;
+      }
+    };
+
+    const configure = () => {
+      if (disposed) return;
+      const c = getConfig();
+      const intervalMs = (c && Number.isFinite(c.proactiveProbeIntervalMs) && c.proactiveProbeIntervalMs >= 1000)
+        ? c.proactiveProbeIntervalMs : 60000;
+      if (timer !== undefined) timers.clearInterval(timer);
+      if (c?.proactiveHealthProbing === true) {
+        timer = timers.setInterval(run, intervalMs);
+        timer?.unref?.();
+      }
+    };
+
+    const changed = () => queueMicrotask(configure);
+    const off = [
+      typeof ctx?.on === 'function' ? ctx.on('settings/document-updated', changed) : () => {},
+      typeof ctx?.on === 'function' ? ctx.on('loader/volatile-update', changed) : () => {},
+    ];
+    configure();
+    return () => {
+      disposed = true;
+      if (timer !== undefined) timers.clearInterval(timer);
+      for (const stop of off) stop();
+    };
+  }, 'dsh-key-rotation: proactive health probing');
+}

package/lib/notify-events.js

@@ -64,3 +64,33 @@ export function notifyExhaustion(runtime, pool, options, hooks = { webhookSender
   } catch (_) { /* ponytail: never crash rotate() */ }
 }
 
+
+const earlyWarningNotifiedAt = new Map();
+
+export function notifyEarlyWarning(runtime, pool, info = {}, hooks = { webhookSender: defaultWebhookSender, now: () => Date.now() }) {
+  if (!runtime?.notifyWebhook || runtime.earlyWarningNotify !== true) return;
+  const provider = info.provider || pool?.provider || pool?.base || 'unknown';
+  const throttle = Math.max(0, runtime.switchNotifyThrottleMs ?? 60000);
+  const last = earlyWarningNotifiedAt.get(provider) ?? 0;
+  const now = hooks.now();
+  if (now - last < throttle) return;
+  earlyWarningNotifiedAt.set(provider, now);
+
+  const availableCount = info.availableCount ?? 0;
+  const totalCount = info.totalCount ?? (pool?.refs?.length || 1);
+  const ratio = totalCount > 0 ? (availableCount / totalCount) : 0;
+  const send = hooks.notifyQueue
+    ? (url, payload) => { hooks.notifyQueue.enqueue(url, payload); return { sent: true, queued: true }; }
+    : (url, payload) => hooks.webhookSender.send(url, payload);
+
+  send(runtime.notifyWebhook, {
+    title: `Early warning: Key pool degraded (${provider})`,
+    text: `Pool ${provider} has only ${availableCount}/${totalCount} (${Math.round(ratio * 100)}%) active keys remaining.`,
+    provider,
+    kind: 'early_warning',
+    availableCount,
+    totalCount,
+    ratio,
+    at: now,
+  });
+}

package/lib/pool-builder.js

@@ -18,6 +18,7 @@ export function buildPoolItem({
   base,
   keys,
   weights,
+  tiers,
   paused,
   revoked,
   poolCooldown,
@@ -83,7 +84,13 @@ export function buildPoolItem({
       if (revoked.has(refs[i])) parsedRevoked.add(refs[i]);
     }
   }
-  const weightsMap = {};
+  
+  const tiersMap = {};
+  for (let i = 0; i < refs.length; i++) {
+    const val = Array.isArray(tiers) ? tiers[indexed[i].index] : (tiers && typeof tiers ===  object ? tiers[refs[i]] : undefined);
+    tiersMap[refs[i]] = Number.isFinite(val) && val >= 1 ? Math.floor(val) : 1;
+  }
+const weightsMap = {};
   for (let i = 0; i < refs.length; i++) {
     weightsMap[refs[i]] = typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1;
   }
@@ -113,6 +120,7 @@ export function buildPoolItem({
     refs,
     weights: refs.map((_, i) => (typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1)),
     weightsMap,
+    tiers: tiersMap,
     weightedRefs: weightedRefs.length > 0 ? weightedRefs : refs,
     pausedRefs: parsedPaused,
     revokedRefs: parsedRevoked,

package/lib/pool.js

@@ -409,6 +409,14 @@ export function isRateLimited(rate, threshold = 0.1) {
 export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
   if (!Array.isArray(refs) || refs.length <= 1) return (refs ?? []).slice();
   const list = refs.slice();
+  if (strategy === 'tier-priority') {
+    const tiers = deps.tiers || (deps.pool && deps.pool.tiers) || {};
+    return list.sort((a, b) => {
+      const ta = (tiers[a] && Number(tiers[a]) >= 1) ? Number(tiers[a]) : 1;
+      const tb = (tiers[b] && Number(tiers[b]) >= 1) ? Number(tiers[b]) : 1;
+      return ta - tb;
+    });
+  }
   if (strategy === 'least-loaded' && deps.concurrencyTracker && typeof deps.concurrencyTracker.getActive === 'function') {
     const weights = deps.weights || (deps.pool && deps.pool.weightsMap) || {};
     return list.sort((a, b) => {

package/lib/resolver.js

@@ -118,7 +118,7 @@ export function createResolver({
     const strategy = pool.routingStrategy ?? runtime.routingStrategy ?? 'round-robin';
     let list = pool.weightedRefs ?? pool.refs;
     if (strategy === 'lowest-latency' || strategy === 'least-loaded') {
-      list = sortAttemptList(list, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap });
+      list = sortAttemptList(list, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap, tiers: pool.tiers });
     }
     // Fail closed: when this request is managed by a model pool that has local
     // token budgets, an exhausted candidate may not be dispatched and the
@@ -148,15 +148,26 @@ export function createResolver({
       if (candidateExp !== undefined && at >= candidateExp) continue;
       // Local model token budget: a spent credential is simply not a candidate.
       if (quotaManaged && !isModelQuotaAvailable(pool, candidate, at)) continue;
+      const microThrottling = pool.microThrottling ?? runtime.microThrottling ?? false;
+      const microThrottleMaxMs = pool.microThrottleMaxMs ?? runtime.microThrottleMaxMs ?? 1000;
+
       const rpmLimit = pool.rpmLimit ?? 0;
       if (rpmLimit > 0) {
         if (!pool.state.rpmWindows) pool.state.rpmWindows = new Map();
         if (!bucketAllow(pool.state.rpmWindows, candidate, rpmLimit, at)) {
           const waitMs = bucketRetryMs(pool.state.rpmWindows, candidate, rpmLimit, at);
-          if ((pool.state.failedUntil.get(candidate) ?? 0) < at + waitMs) {
-            pool.state.failedUntil.set(candidate, at + waitMs);
+          if (microThrottling && waitMs > 0 && waitMs <= microThrottleMaxMs) {
+            await new Promise((r) => setTimeout(r, waitMs));
+            // re-check after micro-sleep
+            if (!bucketAllow(pool.state.rpmWindows, candidate, rpmLimit, now())) {
+              continue;
+            }
+          } else {
+            if ((pool.state.failedUntil.get(candidate) ?? 0) < at + waitMs) {
+              pool.state.failedUntil.set(candidate, at + waitMs);
+            }
+            continue;
           }
-          continue;
         }
       }
       const tpmLimit = pool.tpmLimit ?? 0;
@@ -164,10 +175,17 @@ export function createResolver({
         if (!pool.state.tpmWindows) pool.state.tpmWindows = new Map();
         if (!tpmAllow(pool.state.tpmWindows, candidate, tpmLimit, at)) {
           const waitMs = tpmRetryMs(pool.state.tpmWindows, candidate, tpmLimit, at);
-          if ((pool.state.failedUntil.get(candidate) ?? 0) < at + waitMs) {
-            pool.state.failedUntil.set(candidate, at + waitMs);
+          if (microThrottling && waitMs > 0 && waitMs <= microThrottleMaxMs) {
+            await new Promise((r) => setTimeout(r, waitMs));
+            if (!tpmAllow(pool.state.tpmWindows, candidate, tpmLimit, now())) {
+              continue;
+            }
+          } else {
+            if ((pool.state.failedUntil.get(candidate) ?? 0) < at + waitMs) {
+              pool.state.failedUntil.set(candidate, at + waitMs);
+            }
+            continue;
           }
-          continue;
         }
       }
       if (pool.perHour) {

package/lib/rotate.js

@@ -152,7 +152,7 @@ export function createRotate(deps) {
       }
       const strategy = pool.routingStrategy ?? runtime0.routingStrategy ?? 'round-robin';
       if (strategy !== 'round-robin') {
-        attemptList = sortAttemptList(attemptList, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap });
+        attemptList = sortAttemptList(attemptList, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap, tiers: pool.tiers });
       }
       if (strategy !== 'least-loaded' && runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
         // #193 / #365: prefer least-loaded key within limit with weights

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.56",
+  "version": "0.8.57",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [