@goodandready/dsh-key-rotation 0.7.24 → 0.7.26

package/lib/canary.js

@@ -0,0 +1,56 @@
+// canary.js — canary probing before releasing a key from cooldown (issue #196).
+
+export const CANARY_PROBE_TIMEOUT_MS = 5000;
+export const CANARY_DEFAULT_INTERVAL_MS = 30 * 1000;
+
+export class CanaryProber {
+  constructor(opts) {
+    opts = opts || {};
+    if (!opts.sandboxRunner) throw new Error('canary: sandboxRunner required');
+    this._runner = opts.sandboxRunner;
+    this._intervalMs = opts.intervalMs || CANARY_DEFAULT_INTERVAL_MS;
+    this._results = new Map();
+    this._inProgress = new Set();
+  }
+
+  get intervalMs() { return this._intervalMs; }
+
+  async probe(ref, key) {
+    if (!ref || this._inProgress.has(ref)) return null;
+    this._inProgress.add(ref);
+    try {
+      const result = await this._runner.probeModels(ref, key);
+      this._results.set(ref, Object.assign({}, result, { at: Date.now() }));
+      return result;
+    } catch (e) {
+      return { ok: false, code: 'error', at: Date.now() };
+    } finally {
+      this._inProgress.delete(ref);
+    }
+  }
+
+  lastResult(ref) {
+    return this._results.get(ref) || null;
+  }
+
+  isHealthy(ref) {
+    const r = this._results.get(ref);
+    return Boolean(r && r.ok);
+  }
+
+  clear(ref) {
+    if (ref) {
+      this._results.delete(ref);
+      this._inProgress.delete(ref);
+    } else {
+      this._results.clear();
+      this._inProgress.clear();
+    }
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._results) out[k] = v;
+    return out;
+  }
+}

package/lib/cascade.js

@@ -0,0 +1,39 @@
+// cascade.js — cross-provider failover cascade (issue #194).
+// ponytail: minimal — pick fallback provider from a RegionMap-like config.
+
+export const CASCADE_MAX_DEPTH = 1;
+
+export function pickCascadeFallback(provider, cfg, pools) {
+  const list = Array.isArray(cfg && cfg.cascade) ? cfg.cascade : [];
+  for (const entry of list) {
+    const fb = typeof entry === 'string' ? { provider: entry } : entry;
+    if (!fb || !fb.provider || fb.provider === provider) continue;
+    const pool = pools instanceof Map ? pools.get(fb.provider) : (pools ? pools[fb.provider] : null);
+    if (!pool) continue;
+    const now = Date.now();
+    let healthy = 0;
+    for (const ref of pool.refs) {
+      const failedUntil = (pool.state && pool.state.failedUntil && pool.state.failedUntil.get(ref)) || 0;
+      if (failedUntil > now) continue;
+      const exp = pool.expiresAt ? pool.expiresAt[ref] : undefined;
+      if (exp !== undefined && now >= exp) continue;
+      healthy += 1;
+    }
+    if (healthy === 0) continue;
+    return { provider: fb.provider, pool, model: fb.model || null };
+  }
+  return null;
+}
+
+export function hasHealthyKey(pool, now) {
+  now = now || Date.now();
+  if (!pool || !Array.isArray(pool.refs)) return false;
+  for (const ref of pool.refs) {
+    const failedUntil = (pool.state && pool.state.failedUntil && pool.state.failedUntil.get(ref)) || 0;
+    if (failedUntil > now) continue;
+    const exp = pool.expiresAt ? pool.expiresAt[ref] : undefined;
+    if (exp !== undefined && now >= exp) continue;
+    return true;
+  }
+  return false;
+}

package/lib/concurrency.js

@@ -0,0 +1,72 @@
+// concurrency.js — per-key in-flight counter + least-connections picking (issue #193).
+
+export const CONCURRENCY_DEFAULT_LIMIT = 0;
+export const CONCURRENCY_STALE_LOCK_MS = 5 * 60 * 1000;
+
+export class ConcurrencyTracker {
+  constructor(opts) {
+    opts = opts || {};
+    const limit = opts.limit !== undefined ? opts.limit : CONCURRENCY_DEFAULT_LIMIT;
+    this._limit = (Number.isFinite(limit) && limit >= 0) ? Math.floor(limit) : 0;
+    this._staleMs = opts.staleMs || CONCURRENCY_STALE_LOCK_MS;
+    this._inFlight = new Map();
+  }
+
+  isEnabled() { return this._limit > 0; }
+  get limit() { return this._limit; }
+
+  acquire(ref, now) {
+    now = now || Date.now();
+    if (!this.isEnabled()) return true;
+    let e = this._inFlight.get(ref);
+    if (!e) {
+      e = { count: 0, lastAcquired: now };
+      this._inFlight.set(ref, e);
+    }
+    if (now - e.lastAcquired > this._staleMs) {
+      e.count = 0;
+    }
+    if (e.count >= this._limit) return false;
+    e.count += 1;
+    e.lastAcquired = now;
+    return true;
+  }
+
+  release(ref, now) {
+    now = now || Date.now();
+    const e = this._inFlight.get(ref);
+    if (!e) return;
+    e.count = Math.max(0, e.count - 1);
+    e.lastAcquired = now;
+  }
+
+  snapshot() {
+    const out = {};
+    for (const [k, v] of this._inFlight) out[k] = { count: v.count };
+    return out;
+  }
+
+  pickLeastLoaded(candidates, now) {
+    now = now || Date.now();
+    if (!Array.isArray(candidates) || candidates.length === 0) return null;
+    let best = null;
+    let bestCount = Infinity;
+    for (const ref of candidates) {
+      const e = this._inFlight.get(ref);
+      const count = e ? e.count : 0;
+      if (this.isEnabled() && count >= this._limit) continue;
+      if (count < bestCount) {
+        best = ref;
+        bestCount = count;
+      }
+    }
+    return best;
+  }
+
+  clear(ref) {
+    if (ref) this._inFlight.delete(ref);
+    else this._inFlight.clear();
+  }
+
+  get size() { return this._inFlight.size; }
+}

package/lib/index.js

@@ -55,9 +55,14 @@ const REGIONS_PATH = '/dsh-key-rotation/regions';
 const INCIDENT_RESET_PATH = '/dsh-key-rotation/incident-reset';
 const SHADOW_PATH = '/dsh-key-rotation/shadow';
 const WEBHOOK_TEST_PATH = '/dsh-key-rotation/webhook-test';
+const TEST_MATRIX_PATH = '/dsh-key-rotation/test-matrix';
 import { LastTestCache, SandboxRunner } from './sandbox.js';
 import { healIdleCooldowns } from './heal.js';
 import { LatencyHistogram } from './histogram.js';
+import { pickCascadeFallback } from './cascade.js';
+import { ConcurrencyTracker } from './concurrency.js';
+import { nextQuotaReset } from './quota-window.js';
+import { CanaryProber } from './canary.js';
 import { QuotaStore } from './quota.js';
 import { AgentBudget } from './agent-budget.js';
 import { RegionMap } from './region.js';
@@ -106,6 +111,8 @@ function ensureIncidentReporter() {
 }
 const shadowRouter = new ShadowRouter({ primary: '', secondary: '', percent: 0 });let sandboxRunner = null;
 const webhookSender = new WebhookSender({ fetchImpl: globalThis.fetch });
+const concurrencyTracker = new ConcurrencyTracker();
+let canaryProber = null;
 function ensureSandboxRunner(ctx) {
   if (sandboxRunner) return sandboxRunner;
   // provider id -> baseUrl (stripped of trailing /) for fetch /models probe
@@ -158,6 +165,17 @@ export const Config = Schema.object({
   incidentGitHubToken: Schema.string().default(''),
   incidentGitHubBaseUrl: Schema.string().default(''),
   incidentThreshold: Schema.number().default(5),
+  concurrencyLimit: Schema.number().default(0),
+  canaryProbingEnabled: Schema.boolean().default(false),
+  canaryIntervalMs: Schema.number().default(30000),
+  cascade: Schema.array(Schema.object({
+    provider: Schema.string().required(),
+    model: Schema.string(),
+  })).default([]),
+  quotaResetWindow: Schema.object({
+    type: Schema.string().default('midnight_utc'),
+    hour: Schema.number().default(0),
+  }),
   rateLimitThreshold: Schema.number().default(0.1),
   providers: Schema.array(Schema.object({
     provider: Schema.string().required(),
@@ -337,6 +355,41 @@ export function apply(ctx, config = {}) {
   // interval, low cost; skipped when selfHealCooldown is disabled in config.
   // ponytail: keep handle on the same ctx via closure so buildRuntime() reads
   // fresh config on every tick. Naive but correct: 60s cadence is cheap.
+  // #196: canary probing before key release from cooldown.
+  // Every canaryIntervalMs, probe refs that are in cooldown and close to expiry.
+  let canaryTimer = null;
+  const startCanary = () => {
+    const cfg = getConfig();
+    if (!cfg || !cfg.canaryProbingEnabled) return;
+    if (canaryTimer) return;
+    canaryTimer = setInterval(() => {
+      try {
+        const c = getConfig();
+        if (!c || !c.canaryProbingEnabled) return;
+        const runner = ensureSandboxRunner();
+        if (!runner) return;
+        if (!canaryProber) {
+          canaryProber = new CanaryProber({ sandboxRunner: runner, intervalMs: c.canaryIntervalMs });
+        }
+        const providers = Array.isArray(c.providers) ? c.providers : [];
+        for (const p of providers) {
+          const pool = buildRuntime().providerToPool.get(p.provider);
+          if (!pool) continue;
+          for (const ref of pool.refs) {
+            const until = pool.state.failedUntil.get(ref) ?? 0;
+            const now = Date.now();
+            // Probe refs in cooldown whose expiry is within canaryIntervalMs of now
+            if (until > now && until - now < (c.canaryIntervalMs ?? 30000)) {
+              canaryProber.probe(ref, ref);
+            }
+          }
+        }
+      } catch (_) { /* ponytail: never crash the timer */ }
+    }, cfg.canaryIntervalMs ?? 30000);
+    if (typeof canaryTimer.unref === 'function') canaryTimer.unref();
+  };
+  startCanary();
+
   const selfHealTimer = setInterval(() => {
     const cfg = getConfig();
     if (!cfg || cfg.selfHealCooldown === false) return;
@@ -478,6 +531,11 @@ export function apply(ctx, config = {}) {
     const rotationScheduleDays = cfg.rotationScheduleDays ?? 0;
     const rateLimitThreshold = cfg.rateLimitThreshold ?? 0.1;
     const incidentThreshold = cfg.incidentThreshold ?? 5;
+    const concurrencyLimit = cfg.concurrencyLimit ?? 0;
+    const canaryProbingEnabled = cfg.canaryProbingEnabled ?? false;
+    const canaryIntervalMs = cfg.canaryIntervalMs ?? 30000;
+    const cascade = Array.isArray(cfg.cascade) ? cfg.cascade : [];
+    const quotaResetWindow = cfg.quotaResetWindow || null;
 
     // ref -> pool (every key env of every configured provider)
     const poolByRef = new Map();
@@ -492,9 +550,24 @@ export function apply(ctx, config = {}) {
       let st = poolState.get(base);
       if (!st) {
         st = {
-          failedUntil: new Map(), failCounts: new Map(), pointer: 0, lastUsed: undefined,
-          switches: 0, lastReason: undefined, lastSwitchAt: undefined,
-          lastExhaustionAt: undefined, exhaustionCount: 0, events: [], usageCounts: new Map(), byModel: new Map(), usageDays: new Map(),
+          failedUntil: new Map(),
+          failCounts: new Map(),
+          authFailCounts: new Map(),
+          brokenUntil: new Map(),
+          costPerKey: new Map(),
+          lastUsedAt: new Map(),
+          usageCounts: new Map(),
+          byModel: new Map(),
+          usageDays: new Map(),
+          quotaWindows: new Map(),
+          pointer: 0,
+          lastUsed: undefined,
+          switches: 0,
+          lastReason: undefined,
+          lastSwitchAt: undefined,
+          lastExhaustionAt: undefined,
+          exhaustionCount: 0,
+          events: [],
         };
         poolState.set(base, st);
       }
@@ -562,7 +635,7 @@ export function apply(ctx, config = {}) {
     for (const key of [...poolState.keys()]) {
       if (![...poolByRef.values()].some((p) => p.base === key)) poolState.delete(key);
     }
-    return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
+    return { switchCodes, cooldownMs, maxCooldownMs, notifyWebhook, notifyThreshold, incidentThreshold, concurrencyLimit, canaryProbingEnabled, canaryIntervalMs, cascade, quotaResetWindow, backupDir, backupIntervalMs, backupKeep, rotationScheduleDays, rateLimitThreshold, poolByRef, providerToPool, modelPoolByProvider, cloneIds };
   }
 
   // ── patch credentials.resolve: pool refs resolve to the next healthy key ──
@@ -672,6 +745,25 @@ export function apply(ctx, config = {}) {
       let lastFailure = null;
       _rotateStartMs = Date.now();
 
+      const runtime0 = buildRuntime();
+      if (runtime0.concurrencyLimit > 0 && concurrencyTracker.isEnabled()) {
+        // #193: prefer least-loaded key within limit
+        const available = (pool.weightedRefs ?? pool.refs).filter((r) => {
+          const fu = pool.state.failedUntil.get(r) ?? 0;
+          if (fu > Date.now()) return false;
+          const exp = pool.expiresAt ? pool.expiresAt[r] : undefined;
+          if (exp !== undefined && Date.now() >= exp) return false;
+          return true;
+        });
+        const preferred = concurrencyTracker.pickLeastLoaded(available);
+        if (preferred && (pool.weightedRefs ?? pool.refs)[0] !== preferred) {
+          // Move preferred to front of the attempt list
+          const list = (pool.weightedRefs ?? pool.refs).slice();
+          const i = list.indexOf(preferred);
+          if (i > 0) { list.splice(i, 1); list.unshift(preferred); }
+          pool.weightedRefs = list;
+        }
+      }
       for (let attempt = 0; attempt < (pool.weightedRefs ?? pool.refs).length; attempt++) {
         let yielded = false;
         let switching = false;
@@ -687,6 +779,8 @@ export function apply(ctx, config = {}) {
           continue;
         }
 
+        const _pickedRef = pool.state.lastUsed;
+        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.acquire(_pickedRef);
         try {
           for await (const chunk of inner) {
             // Only actual content deltas lock the stream (no more rotation).
@@ -705,7 +799,28 @@ export function apply(ctx, config = {}) {
               const switchable = !yielded && kind === 'error' &&
                 (effectiveSwitchCodes.has(code) || SWITCHABLE_MESSAGE_PATTERN.test(message));
               if (switchable) {
-                if (pool.state.lastUsed) { const _retry = parseRetryAfter(message); const _base = pool.cooldownMs ?? cooldownMs; const _max = pool.maxCooldownMs ?? maxCooldownMs; const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base; const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max); pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b); const _code2 = String(code ?? ''); if (_code2 === 'AUTH' || /auth/i.test(message)) { const _c2 = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1; pool.state.authFailCounts.set(pool.state.lastUsed, _c2); if (_c2 >= 3) { pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30); } } else { pool.state.authFailCounts.delete(pool.state.lastUsed); } }
+                if (pool.state.lastUsed) {
+                  const _retry = parseRetryAfter(message);
+                  const _base = pool.cooldownMs ?? cooldownMs;
+                  const _max = pool.maxCooldownMs ?? maxCooldownMs;
+                  const _effBase = _retry !== undefined ? Math.max(_base, Math.min(_retry, _max ?? _base * 8)) : _base;
+                  const _b = recordFailure(pool, pool.state.lastUsed, Date.now(), _effBase, _max);
+                  pushEvent(pool, pool.state.lastUsed, code ?? 'UNKNOWN', _b);
+                  // authFailCounts/brokenUntil: lazy-init if state was created by an older plugin version
+                  if (!pool.state.authFailCounts) pool.state.authFailCounts = new Map();
+                  if (!pool.state.brokenUntil) pool.state.brokenUntil = new Map();
+                  const _code2 = String(code ?? '');
+                  if (_code2 === 'AUTH' || /auth/i.test(message)) {
+                    const _c2 = (pool.state.authFailCounts.get(pool.state.lastUsed) ?? 0) + 1;
+                    pool.state.authFailCounts.set(pool.state.lastUsed, _c2);
+                    if (_c2 >= 3) {
+                      pool.state.brokenUntil.set(pool.state.lastUsed, Date.now() + 86400000*30);
+                      pool.state.failedUntil.set(pool.state.lastUsed, Date.now() + 86400000*30);
+                    }
+                  } else {
+                    pool.state.authFailCounts.delete(pool.state.lastUsed);
+                  }
+                }
                 pool.state.switches = (pool.state.switches ?? 0) + 1;
                 pool.state.lastReason = String(code ?? 'UNKNOWN');
                 pool.state.lastSwitchAt = Date.now();
@@ -717,7 +832,10 @@ export function apply(ctx, config = {}) {
               // cost tracking if provider returns usage.cost
               if (chunk.usage?.cost != null && pool.state.lastUsed) {
                 const c = Number(chunk.usage.cost);
-                if (!isNaN(c)) pool.state.costPerKey.set(pool.state.lastUsed, (pool.state.costPerKey.get(pool.state.lastUsed) ?? 0) + c);
+                if (!isNaN(c)) {
+                  if (!pool.state.costPerKey) pool.state.costPerKey = new Map();
+                  pool.state.costPerKey.set(pool.state.lastUsed, (pool.state.costPerKey.get(pool.state.lastUsed) ?? 0) + c);
+                }
               }
               // Usage by day (#119)
               if (pool.state.lastUsed) {
@@ -758,10 +876,12 @@ export function apply(ctx, config = {}) {
             yield chunk;
           }
         } catch (e) {
+          if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
           yield finishError(e?.code ?? 'TRANSPORT', String(e?.message ?? e));
           return;
         }
 
+        if (_pickedRef && runtime0.concurrencyLimit > 0) concurrencyTracker.release(_pickedRef);
         if (switching) continue; // try the next key
         return; // clean end — served
       }
@@ -773,6 +893,24 @@ export function apply(ctx, config = {}) {
       // notify via extracted helper (see notifyExhaustion above)
       notifyExhaustion(buildRuntime(), pool, { provider: options.provider });
 
+      // #194: cross-provider cascade failover
+      const runtime = buildRuntime();
+      if (Array.isArray(runtime.cascade) && runtime.cascade.length > 0) {
+        const pools = runtime.providerToPool;
+        const fb = pickCascadeFallback(options.provider, runtime, pools);
+        if (fb && fb.pool && fb.pool !== pool) {
+          console.warn(`[dsh-key-rotation] ${options.provider}: pool exhausted — cascading to ${fb.provider}`);
+          pool.state.lastReason = 'CASCADE';
+          pool.state.lastSwitchAt = Date.now();
+          // Re-dispatch on the fallback pool (depth-1 via marker check)
+          const innerCascade = rotate({ ...options, provider: fb.provider }, fb.pool);
+          for await (const chunk of innerCascade) {
+            yield chunk;
+          }
+          return;
+        }
+      }
+
       yield lastFailure ?? finishError('TRANSPORT', 'dsh-key-rotation: all keys failed');
     })();
   }
@@ -1117,6 +1255,36 @@ export function apply(ctx, config = {}) {
     },
   }), 'dsh-key-rotation: incident-reset');
 
+  // #198: 1-click Health Matrix — parallel probe of all configured keys.
+  ctx.effect(() => ctx.webServer.register({
+    kind: 'exact',
+    path: TEST_MATRIX_PATH,
+    handler: async (req, res) => {
+      if (req.method !== 'POST') { json(res, 405, { error: { code: 'method', message: 'POST only' } }); return; }
+      if (!isTrustedBridgeRequest(req)) { json(res, 403, { error: { code: 'forbidden', message: 'dsh-key-rotation: matrix is local-only' } }); return; }
+      const cfg = getConfig();
+      const runner = ensureSandboxRunner();
+      if (!runner) { json(res, 500, { error: { code: 'no-runner', message: 'sandbox runner unavailable' } }); return; }
+      const providers = Array.isArray(cfg?.providers) ? cfg.providers : [];
+      const jobs = [];
+      for (const p of providers) {
+        for (const ref of (p.keys ?? [])) {
+          if (typeof ref !== 'string' || !ref) continue;
+          jobs.push((async () => {
+            try {
+              const probeResult = await runner.probeModels(ref, ref);
+              return { provider: p.provider, ref, ok: probeResult.ok, code: probeResult.code, latencyMs: probeResult.latencyMs, modelsCount: probeResult.modelsCount ?? 0 };
+            } catch (e) {
+              return { provider: p.provider, ref, ok: false, code: 'error', latencyMs: 0, modelsCount: 0 };
+            }
+          })());
+        }
+      }
+      const results = await Promise.all(jobs);
+      json(res, 200, { at: Date.now(), total: results.length, ok: results.filter(r => r.ok).length, results });
+    },
+  }), 'dsh-key-rotation: test-matrix');
+
   // Webhook test endpoint (#10): dry-run that validates webhookSender setup.
   ctx.effect(() => ctx.webServer.register({
     kind: 'exact',

package/lib/quota-window.js

@@ -0,0 +1,45 @@
+// quota-window.js — calendar-based quota reset windows (issue #197).
+
+export const QUOTA_WINDOW_TYPES = ['midnight_utc', 'midnight_pst', 'rolling_24h'];
+
+export function nextQuotaReset(quotaResetWindow, now) {
+  now = now || Date.now();
+  if (!quotaResetWindow || typeof quotaResetWindow !== 'object') return null;
+  const type = quotaResetWindow.type;
+  const hour = Number.isFinite(quotaResetWindow.hour) ? quotaResetWindow.hour : 0;
+
+  if (type === 'rolling_24h') {
+    const d = new Date(now);
+    d.setUTCHours(d.getUTCHours() + 24, 0, 0, 0);
+    return d.getTime();
+  }
+
+  if (type === 'midnight_utc') {
+    const d = new Date(now);
+    d.setUTCHours(hour, 0, 0, 0);
+    if (d.getTime() <= now) d.setUTCDate(d.getUTCDate() + 1);
+    return d.getTime();
+  }
+
+  if (type === 'midnight_pst') {
+    const PST_OFFSET_MS = 8 * 3600_000;
+    const shifted = now + PST_OFFSET_MS;
+    const d = new Date(shifted);
+    d.setUTCHours(hour, 0, 0, 0);
+    if (d.getTime() <= shifted) d.setUTCDate(d.getUTCDate() + 1);
+    return d.getTime() - PST_OFFSET_MS;
+  }
+
+  return null;
+}
+
+export function poolResetAt(pool, quotaResetWindow, now) {
+  return nextQuotaReset(quotaResetWindow, now);
+}
+
+export function isBlockedUntilReset(failedUntil, resetAt, now) {
+  now = now || Date.now();
+  if (!Number.isFinite(failedUntil)) return false;
+  if (resetAt === null || resetAt === undefined) return false;
+  return failedUntil >= resetAt;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.7.24",
+  "version": "0.7.26",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [
     "deepseek-harness",