@goodandready/dsh-key-rotation 0.8.60 → 0.8.61

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## 0.8.61 - 2026-10-15
+
+### Fixed
+- **Prevent Transient Rate Limit Deadlocks from Calendar Quota Locks (#486 / GitHub #24)**:
+  - Upstream errors returning transient 429, RPM/TPM exhaustion, or concurrency limits (such as SenseNova `{"message":"rpm exhausted","type":"quota_exceeded_error"}` or `{"message":"inference exceeds tpm/rpm limit","code":"insufficient_quota"}`) are now classified as `isRateThrottling` and excluded from `isQuota`.
+  - Transient rate limits receive standard short backoff (`cooldownMs` with jitter) rather than being locked out until next UTC midnight.
+  - Cleared `pool.state.quotaWindows` in `recordSuccess` upon first successful request.
+  - Extended background `autoUnbreakBrokenKeys` and proactive health probing to probe keys with calendar quota locks, restoring healthy keys immediately upon receiving 200 OK from upstream.
+  - Added `INSUFFICIENT_QUOTA` and present-tense regex matches to switchable error classification.
+
+### Added
+- **Sticky Key Routing Strategy to Preserve Prompt Caching (#487 / GitHub #23)**:
+  - Added `'sticky'` routing strategy option in settings and provider configurations.
+  - In `'sticky'` mode, the resolver retains the same active key across consecutive requests as long as it remains healthy, maximizing prefix prompt cache hit rate on providers that cache per API key (Anthropic, DeepSeek, OpenAI).
+  - Automatically fails over to the next healthy key in the pool if the sticky key experiences an error or cooldown, then sticks to the new key.
+  - Added UI dropdown selection in the Key Rotation Settings Card with English and Chinese localizations.
+
 ## 0.8.60 - 2026-10-14
 
 ### Changed

package/lib/client.js

@@ -221,6 +221,7 @@ window.__ModuleLoader__.load({
       routingStrategyRoundRobin: 'Round-Robin (default)',
       routingStrategyLeastLoaded: 'Least-Loaded (concurrency)',
       routingStrategyLowestLatency: 'Lowest-Latency (p95)',
+      routingStrategySticky: 'Sticky (preserves prompt cache)',
       proactiveGuardLabel: 'Proactive Rate-Limit Guard',
       proactiveGuardHint: 'Pre-emptively pause keys when remaining quota or Retry-After header indicates limit before hitting 429',
       selfHealingLabel: 'Auto-Unbreak Interval (min)',
@@ -405,6 +406,7 @@ window.__ModuleLoader__.load({
       routingStrategyRoundRobin: '轮询 (默认)',
       routingStrategyLeastLoaded: '最低负载 (并发)',
       routingStrategyLowestLatency: '最低延迟 (p95)',
+      routingStrategySticky: '粘性模式 (优先保持提示词缓存)',
       proactiveGuardLabel: '前瞻性限流防护',
       proactiveGuardHint: '当响应中的剩余配额或 Retry-After 头提示即将超限时，预先冷却密钥以防触发 429',
       selfHealingLabel: '自愈检查间隔 (分钟)',
@@ -2489,6 +2491,7 @@ window.__ModuleLoader__.load({
             h('option', { key: 'round-robin', value: 'round-robin' }, t('routingStrategyRoundRobin')),
             h('option', { key: 'least-loaded', value: 'least-loaded' }, t('routingStrategyLeastLoaded')),
             h('option', { key: 'lowest-latency', value: 'lowest-latency' }, t('routingStrategyLowestLatency')),
+            h('option', { key: 'sticky', value: 'sticky' }, t('routingStrategySticky')),
           ])),
           field(t('selfHealingLabel'), textInput(String(val.selfHealingIntervalMinutes ?? 30), (v) => setField((cur) => ({ ...cur, selfHealingIntervalMinutes: Number(v) || 0 })))),
           field(t('proactiveGuardLabel'), h('label', { style: { display: 'flex', alignItems: 'center', gap: '8px', height: '36px', cursor: 'pointer' } }, [

package/lib/heal.js

@@ -52,21 +52,31 @@ export async function autoUnbreakBrokenKeys(pools, probeFn, now = Date.now(), {
   for (const pool of pools) {
     if (!pool || !pool.state || !pool.base) continue;
     const bu = pool.state.brokenUntil;
-    if (!bu || bu.size === 0) continue;
-    const brokenRefs = [];
-    for (const [ref, until] of bu.entries()) {
-      if (isKeyRevoked(pool, ref) || isKeyPaused(pool, ref)) continue;
-      if (Number.isFinite(until) && until > now) {
-        brokenRefs.push(ref);
+    const qw = pool.state.quotaWindows;
+    const candidates = new Set();
+    if (bu && bu.size > 0) {
+      for (const [ref, until] of bu.entries()) {
+        if (!isKeyRevoked(pool, ref) && !isKeyPaused(pool, ref) && Number.isFinite(until) && until > now) {
+          candidates.add(ref);
+        }
       }
     }
-    for (const ref of brokenRefs) {
+    if (qw && qw.size > 0) {
+      for (const [ref, until] of qw.entries()) {
+        if (!isKeyRevoked(pool, ref) && !isKeyPaused(pool, ref) && Number.isFinite(until) && until > now) {
+          candidates.add(ref);
+        }
+      }
+    }
+    if (candidates.size === 0) continue;
+
+    for (const ref of candidates) {
       if (!isActive()) return results;
       const state = pool.state;
-      const fields = ['brokenUntil', 'failedUntil', 'failCounts', 'authFailCounts'];
+      const fields = ['brokenUntil', 'failedUntil', 'quotaWindows', 'failCounts', 'authFailCounts'];
       const observed = fields.map(field => [field, state[field], state[field]?.get(ref)]);
       // An earlier await may have removed this key from the captured worklist.
-      if (state.brokenUntil !== bu || !bu.has(ref)) continue;
+      if (!((bu && bu.has(ref)) || (qw && qw.has(ref)))) continue;
       try {
         const res = await probeFn(ref);
         if (!isActive()) return results;
@@ -74,7 +84,9 @@ export async function autoUnbreakBrokenKeys(pools, probeFn, now = Date.now(), {
         if (pool.state !== state || observed.some(([field, map, value]) =>
           state[field] !== map || map?.get(ref) !== value)) continue;
         if (res && res.ok) {
-          bu.delete(ref);
+          bu?.delete(ref);
+          qw?.delete(ref);
+          pool.state.quotaWindows?.delete(ref);
           pool.state.failedUntil?.delete(ref);
           pool.state.failCounts?.delete(ref);
           pool.state.authFailCounts?.delete(ref);

package/lib/index.js

@@ -95,7 +95,7 @@ const SettingsSchema = Schema.object({
   rateLimitThreshold: Schema.number().default(0.1),
   proactiveRateLimitGuard: Schema.boolean().default(true),
   selfHealingIntervalMinutes: Schema.number().default(30),
-  routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency']).default('round-robin'),
+  routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky']).default('round-robin'),
   rpmLimit: Schema.number().default(0),
   tpmLimit: Schema.number().default(0),
   webhookActionToken: Schema.string().role('secret').default(''),
@@ -144,7 +144,7 @@ const SettingsSchema = Schema.object({
     })).default({}),
     cooldownMs: Schema.number(),
     maxCooldownMs: Schema.number(),
-    routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency']),
+    routingStrategy: Schema.union(['round-robin', 'least-loaded', 'lowest-latency', 'sticky']),
     proactiveRateLimitGuard: Schema.boolean(),
     rpmLimit: Schema.number(),
     tpmLimit: Schema.number(),

package/lib/lifecycle.js

@@ -171,8 +171,10 @@ export function setupProactiveHealthProbeEffect(ctx, getConfig, buildRuntime, en
       running = true;
       try {
         const c = getConfig();
-        if (!c || c.proactiveHealthProbing !== true) return;
+        if (!c) return;
         const { pools } = buildRuntime();
+        const hasCalendarLocks = (pools || []).some(p => p?.state?.quotaWindows?.size > 0);
+        if (c.proactiveHealthProbing !== true && !hasCalendarLocks) return;
         const runner = ensureSandboxRunner(ctx);
         const now = Date.now();
         for (const pool of (pools || [])) {
@@ -203,6 +205,7 @@ export function setupProactiveHealthProbeEffect(ctx, getConfig, buildRuntime, en
               if (res && res.ok) {
                 pool.state.failedUntil?.delete(ref);
                 pool.state.brokenUntil?.delete(ref);
+                pool.state.quotaWindows?.delete(ref);
                 pool.state.failCounts?.delete(ref);
                 if (Array.isArray(pool.state.events)) {
                   pool.state.events.push({ at: Date.now(), ref, reason: 'proactive-probe-heal', cooldownMs: 0, type: 'heal' });

package/lib/pool.js

@@ -68,7 +68,8 @@ export const SWITCHABLE_MESSAGE_PATTERN = new RegExp([
   /\b(?:quota|usage[\s_-]+limit|rate[\s_-]?limit)\b/i,
   /\binsufficient[\s_-]+(?:quota|balance|credits?)\b/i,
   /\bout[\s_-]+of[\s_-]+(?:credits?|budget)\b/i,
-  /\b(?:exceeded|exhausted)[\s_-]+(?:quota|limit|budget)\b/i,
+  /\b(?:exceeded|exceeds|exhausted|exhausts)[\s_-]+(?:quota|limit|budget|rpm|tpm|inference)\b/i,
+  /\b(?:rpm|tpm)\b/i,
   /\bbilling\b/i,
   /\bresource[\s_-]+exhausted\b/i,
   /\bcapacity[\s_-]+(?:limit|exceeded|reached)\b/i,
@@ -171,7 +172,8 @@ export function recordFailure(pool, ref, now, baseMs, maxMs, isSoft = false, jit
 /** Record a success for `ref`. */
 export function recordSuccess(pool, ref, now = Date.now()) {
   if (pool.state.failCounts) pool.state.failCounts.delete(ref);
-  pool.state.failedUntil.delete(ref);
+  pool.state.failedUntil?.delete(ref);
+  pool.state.quotaWindows?.delete(ref);
   if (!pool.state.lastSuccessAt) pool.state.lastSuccessAt = new Map();
   pool.state.lastSuccessAt.set(ref, now);
 }
@@ -409,6 +411,17 @@ export function isRateLimited(rate, threshold = 0.1) {
 export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
   if (!Array.isArray(refs) || refs.length <= 1) return (refs ?? []).slice();
   const list = refs.slice();
+  if (strategy === 'sticky') {
+    const last = deps.lastUsed || (deps.pool && deps.pool.state && deps.pool.state.lastUsed);
+    if (last && list.includes(last)) {
+      const idx = list.indexOf(last);
+      if (idx > 0) {
+        list.splice(idx, 1);
+        list.unshift(last);
+      }
+    }
+    return list;
+  }
   if (strategy === 'tier-priority') {
     const tiers = deps.tiers || (deps.pool && deps.pool.tiers) || {};
     return list.sort((a, b) => {
@@ -482,7 +495,7 @@ export function isSwitchableError(failureOrPayload, switchCodes = new Set(DEFAUL
   if ((status >= 500 && status <= 504) && (switchCodes.has('SERVER') || switchCodes.has(String(status)))) return true;
 
   // 2. Standard gRPC / cloud error codes
-  if (code === 'RESOURCE_EXHAUSTED' && (switchCodes.has('QUOTA') || switchCodes.has('RATE_LIMIT'))) return true;
+  if ((code === 'RESOURCE_EXHAUSTED' || code === 'INSUFFICIENT_QUOTA') && (switchCodes.has('QUOTA') || switchCodes.has('RATE_LIMIT'))) return true;
   if ((code === 'UNAVAILABLE' || code === 'INTERNAL') && switchCodes.has('SERVER')) return true;
   if (code === 'DEADLINE_EXCEEDED' && switchCodes.has('TIMEOUT')) return true;
   if ((code === 'UNAUTHENTICATED' || code === 'PERMISSION_DENIED') && switchCodes.has('AUTH')) return true;

package/lib/resolver.js

@@ -117,8 +117,8 @@ export function createResolver({
 
     const strategy = pool.routingStrategy ?? runtime.routingStrategy ?? 'round-robin';
     let list = pool.weightedRefs ?? pool.refs;
-    if (strategy === 'lowest-latency' || strategy === 'least-loaded') {
-      list = sortAttemptList(list, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap, tiers: pool.tiers });
+    if (strategy === 'lowest-latency' || strategy === 'least-loaded' || strategy === 'sticky') {
+      list = sortAttemptList(list, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap, tiers: pool.tiers, lastUsed: pool.state.lastUsed });
     }
     // Fail closed: when this request is managed by a model pool that has local
     // token budgets, an exhausted candidate may not be dispatched and the
@@ -126,6 +126,10 @@ export function createResolver({
     // the loop, so it cannot be re-evaluated mid-iteration.
     const quotaManaged = hasModelQuotaConfig(pool) && isQuotaManagedRequest(runtime.index, ref, requested);
     let start = strategy === 'round-robin' ? (pool.state.pointer ?? 0) : 0;
+    if (strategy === 'sticky') {
+      const lastIdx = pool.state.lastUsed ? list.indexOf(pool.state.lastUsed) : -1;
+      start = lastIdx >= 0 ? lastIdx : (pool.state.pointer ?? 0);
+    }
     if (pool.pinnedRef && list.includes(pool.pinnedRef)) {
       const pIdx = list.indexOf(pool.pinnedRef);
       if (pIdx > 0) {
@@ -217,7 +221,11 @@ export function createResolver({
 
       // Advance the shared cursor at selection time: concurrent requests must
       // not all pick the same slot.
-      pool.state.pointer = (index + 1) % list.length;
+      if (strategy === 'sticky') {
+        pool.state.pointer = index;
+      } else {
+        pool.state.pointer = (index + 1) % list.length;
+      }
       try {
         const hit = await original(candidate);
         if (hit && typeof hit.value === 'string' && hit.value.length > 0) {

package/lib/rotate.js

@@ -155,7 +155,7 @@ export function createRotate(deps) {
       }
       const strategy = pool.routingStrategy ?? runtime0.routingStrategy ?? 'round-robin';
       if (strategy !== 'round-robin') {
-        attemptList = sortAttemptList(attemptList, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap, tiers: pool.tiers });
+        attemptList = sortAttemptList(attemptList, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap, tiers: pool.tiers, lastUsed: pool.state.lastUsed });
       }
       if (strategy !== 'least-loaded' && runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
         // #193 / #365: prefer least-loaded key within limit with weights
@@ -186,11 +186,16 @@ export function createRotate(deps) {
         const cls = classifyFailure({ code: errCode, message: errMsg });
         const _b = recordFailure(pool, targetRef, now(), _effBase, _max, cls.soft, true);
 
-        // #337: connect quota-window calendar reset for quota exhaustion
-        // "rate limit exceeded" and "deadline exceeded" are not evidence
-        // of an exhausted billing quota. Keep those on ordinary cooldowns.
-        const isQuota = errCode === 'QUOTA' || cls.code === 'QUOTA' ||
-          /\b(?:insufficient[_\s-]+quota|quota(?:\s+is)?\s+(?:exhausted|exceeded)|exceeded\s+(?:your\s+)?(?:current\s+)?quota)\b/i.test(errMsg);
+        // #337 & #486 (GitHub #24): connect quota-window calendar reset for quota exhaustion
+        // Transient rate limiting (RPM/TPM, requests/tokens per minute, concurrency)
+        // must NOT be locked until midnight. Keep those on ordinary cooldowns.
+        const isRateThrottling = /\b(?:rpm|tpm|requests?\s+per\s+minute|tokens?\s+per\s+minute|per[-_\s]*minute|per[-_\s]*second|concurrency|requests?\s+limit|tokens?\s+limit|traffic)\b/i.test(errMsg) ||
+          (/\b(?:rate[-_\s]*limit)\b/i.test(errMsg) && !/\b(?:insufficient[_\s-]+quota|exceeded\s+(?:your\s+)?(?:current\s+)?quota|credit\s+balance|billing)\b/i.test(errMsg));
+
+        const isQuota = !isRateThrottling && (
+          (errCode === 'QUOTA' && !/\b(?:rpm|tpm|rate|traffic|throttle)\b/i.test(errMsg)) ||
+          /\b(?:quota(?:\s+is)?\s+(?:exhausted|exceeded)|exceeded\s+(?:your\s+)?(?:current\s+)?quota|insufficient[_\s-]+quota|credit\s+balance|billing\s+quota|out\s+of\s+credits?)\b/i.test(errMsg)
+        );
         const qWindow = pool.quotaResetWindow ?? runtime0.quotaResetWindow;
         if (isQuota && qWindow) {
           const quotaResetAt = poolResetAt(pool, qWindow, now());

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.60",
+  "version": "0.8.61",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [