@goodandready/dsh-key-rotation 0.8.28 → 0.8.30

package/lib/resolver.js

@@ -0,0 +1,159 @@
+// lib/resolver.js — the credential selector behind credentials.resolve (#381).
+//
+// Extracted from lib/index.js so the routing decisions that this feature
+// depends on — request-scoped pool ownership, model token quota enforcement,
+// fail-closed exhaustion, paused/revoked filtering, at-rest decryption —
+// are unit-testable without the cordis runtime.
+//
+// The resolver decides *which* credential ref to hand to the underlying
+// credentials service. It never sees or returns a secret value itself; it
+// delegates to `original(ref)`.
+
+import { bucketAllow, bucketRetryMs } from './bucket.js';
+import { envValue, sortAttemptList, isKeyPaused, isKeyRevoked } from './pool.js';
+import { decryptSecret } from './crypto-storage.js';
+import { resolveRefPool, isQuotaManagedRequest } from './pool-index.js';
+import {
+  isModelQuotaAvailable,
+  hasModelQuotaConfig,
+  anyModelQuotaAvailable,
+  formatModelQuotaExhaustion,
+} from './model-quota.js';
+
+/** Internal failure code for "local budgets forbid this dispatch". */
+export const LOCAL_QUOTA_CODE = 'LOCAL_MODEL_QUOTA_EXHAUSTED';
+
+/**
+ * @param {object} deps
+ * @param {() => object} deps.buildRuntime         current runtime snapshot
+ * @param {() => object|undefined} deps.currentPool request-scoped pool (ALS)
+ * @param {(pool: object, candidate: string) => void} [deps.onPicked] picked-ref hook
+ * @param {object} [deps.latencyHistogram]
+ * @param {object} [deps.concurrencyTracker]
+ * @param {() => number} [deps.now] wall clock; injectable for tests
+ * @returns {(ref: string, original: Function) => Promise<any>}
+ */
+export function createResolver({
+  buildRuntime,
+  currentPool,
+  onPicked,
+  latencyHistogram,
+  concurrencyTracker,
+  now = Date.now,
+}) {
+  return async function resolve(ref, original) {
+    const runtime = buildRuntime();
+    // A ref may belong to several model pools at once, so the request-scoped
+    // pool that rotate() actually dispatched with decides ownership. A bare
+    // resolve outside rotation falls back to the global lookup.
+    const requested = currentPool ? currentPool() ?? null : null;
+    const pool = resolveRefPool(runtime.index, ref, requested);
+    if (!pool) {
+      const res = await original(ref);
+      if (res && typeof res.value === 'string') {
+        return { ...res, value: decryptSecret(res.value) };
+      }
+      const ev = envValue(ref);
+      if (ev !== undefined) return { value: decryptSecret(ev), source: 'env' };
+      return res;
+    }
+
+    const at = now();
+    const strategy = pool.routingStrategy ?? runtime.routingStrategy ?? 'round-robin';
+    let list = pool.weightedRefs ?? pool.refs;
+    if (strategy === 'lowest-latency' || strategy === 'least-loaded') {
+      list = sortAttemptList(list, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap });
+    }
+    // Fail closed: when this request is managed by a model pool that has local
+    // token budgets, an exhausted candidate may not be dispatched and the
+    // original credential must never be used as a bypass. Decided once, before
+    // the loop, so it cannot be re-evaluated mid-iteration.
+    const quotaManaged = hasModelQuotaConfig(pool) && isQuotaManagedRequest(runtime.index, ref, requested);
+    const start = strategy === 'round-robin' ? (pool.state.pointer ?? 0) : 0;
+
+    for (let i = 0; i < list.length; i++) {
+      const index = (start + i) % list.length;
+      const candidate = list[index];
+      if (isKeyRevoked(pool, candidate)) continue;
+      if (isKeyPaused(pool, candidate)) continue;
+      const until = pool.state.failedUntil.get(candidate);
+      if (until !== undefined && until > at) continue;
+      if (pool.expiresAt?.[candidate] !== undefined && at >= pool.expiresAt[candidate]) continue;
+      // Local model token budget: a spent credential is simply not a candidate.
+      // This is deliberately NOT recorded as a failure — the key is healthy and
+      // merely out of budget — so it must never touch failedUntil / failCounts /
+      // authFailCounts / brokenUntil.
+      if (quotaManaged && !isModelQuotaAvailable(pool, candidate, at)) continue;
+      const rpmLimit = pool.rpmLimit ?? 0;
+      if (rpmLimit > 0) {
+        if (!pool.state.rpmWindows) pool.state.rpmWindows = new Map();
+        if (!bucketAllow(pool.state.rpmWindows, candidate, rpmLimit, at)) {
+          const waitMs = bucketRetryMs(pool.state.rpmWindows, candidate, rpmLimit, at);
+          if ((pool.state.failedUntil.get(candidate) ?? 0) < at + waitMs) {
+            pool.state.failedUntil.set(candidate, at + waitMs);
+          }
+          continue;
+        }
+      }
+      if (pool.perHour) {
+        if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+        let win = pool.state.quotaWindows.get(candidate);
+        if (!win || at - win.start >= 3600000) win = { count: 0, start: at };
+        if (win.count >= pool.perHour) {
+          const resetAt = win.start + 3600000;
+          if ((pool.state.failedUntil.get(candidate) ?? 0) < resetAt) pool.state.failedUntil.set(candidate, resetAt);
+          continue;
+        }
+      }
+
+      // Advance the shared cursor at selection time: concurrent requests must
+      // not all pick the same slot.
+      pool.state.pointer = (index + 1) % list.length;
+      const hit = await original(candidate);
+      if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
+        settle(pool, candidate, at, onPicked);
+        return { ...hit, value: decryptSecret(hit.value) };
+      }
+      // No stored credential: fall back to the launching environment, which is
+      // how env-bootstrapped pools work (issue #7).
+      const envVal = envValue(candidate);
+      if (envVal !== undefined) {
+        settle(pool, candidate, at, onPicked);
+        return { value: decryptSecret(envVal), source: 'env' };
+      }
+    }
+
+    // Every candidate was skipped. If the reason was local budget exhaustion,
+    // returning original(ref) would hand out a credential the budget forbids, so
+    // refuse instead of bypassing the quota. rotate() recognises this error as a
+    // local refusal: it is not penalised and not classified as an upstream failure.
+    if (quotaManaged && !anyModelQuotaAvailable(pool, at)) {
+      throw Object.assign(
+        new Error(formatModelQuotaExhaustion(pool.provider ?? pool.base, pool)),
+        { code: LOCAL_QUOTA_CODE, localQuota: true },
+      );
+    }
+    return original(ref);
+  };
+}
+
+/** Record a successful credential selection on the pool and request scope. */
+function settle(pool, candidate, now, onPicked) {
+  pool.state.lastUsed = candidate;
+  if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
+  pool.state.lastUsedAt.set(candidate, now);
+  onPicked?.(pool, candidate);
+  pool.state.failCounts?.delete(candidate);
+  pool.state.failedUntil.delete(candidate);
+  pool.state.authFailCounts?.delete(candidate);
+  pool.state.brokenUntil?.delete(candidate);
+  if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
+  pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
+  if (pool.perHour) {
+    if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+    let win = pool.state.quotaWindows.get(candidate);
+    if (!win || now - win.start >= 3600000) win = { count: 0, start: now };
+    win.count++;
+    pool.state.quotaWindows.set(candidate, win);
+  }
+}

package/lib/rotate.js

@@ -16,6 +16,34 @@ import {
 } from './pool.js';
 import { pickCascadeFallback, CASCADE_MAX_DEPTH } from './cascade.js';
 import { nextQuotaReset } from './quota-window.js';
+import {
+  isModelQuotaAvailable,
+  hasModelQuotaConfig,
+  anyModelQuotaAvailable,
+  consumeModelTokens,
+  extractUsageTokens,
+  formatModelQuotaExhaustion,
+} from './model-quota.js';
+
+export const LOCAL_QUOTA_CODE = 'LOCAL_MODEL_QUOTA_EXHAUSTED';
+
+export async function* preserveDispatchContext(iterable, storage, store) {
+  if (!storage || typeof storage.getStore !== 'function') {
+    yield* iterable;
+    return;
+  }
+  const iterator = iterable[Symbol.asyncIterator]();
+  const step = (method, arg) => storage.run(store, () => iterator[method](arg));
+  try {
+    while (true) {
+      const next = await step('next');
+      if (next.done) return next.value;
+      yield next.value;
+    }
+  } finally {
+    if (typeof iterator.return === 'function') await step('return');
+  }
+}
 
 /**
  * Create the rotate(options, pool) async generator used by llm/stream.
@@ -53,8 +81,44 @@ export function createRotate(deps) {
       let activePermit = null;
       const reqStore = { pool, pickedRef: undefined, startMs: now() };
       setRotateStartMs(reqStore.startMs);
+      const wallNow = Date.now();
+      const quotaManaged = hasModelQuotaConfig(pool);
       let attemptList = (pool.weightedRefs ?? pool.refs).slice();
       attemptList = attemptList.filter((r) => !isKeyPaused(pool, r) && !isKeyRevoked(pool, r));
+      if (quotaManaged) {
+        attemptList = attemptList.filter((ref) => isModelQuotaAvailable(pool, ref, wallNow));
+        if (attemptList.length === 0) {
+          pool.state.lastExhaustionAt = now();
+          pool.state.exhaustionCount = (pool.state.exhaustionCount ?? 0) + 1;
+          logWarn(`[dsh-key-rotation] ${options.provider}/${options.model ?? '-'}: local model token budget exhausted for all ${pool.refs.length} key(s) — no upstream request sent`);
+          const runtimeQ = buildRuntime();
+          notifyExhaustion(runtimeQ, pool, { provider: options.provider });
+          const maxDepth = runtimeQ.cascadeMaxDepth ?? CASCADE_MAX_DEPTH;
+          const currentDepth = options.__cascadeDepth ?? (options.__isCascade ? 1 : 0);
+          if (currentDepth < maxDepth && Array.isArray(runtimeQ.cascade) && runtimeQ.cascade.length > 0) {
+            const fb = pickCascadeFallback(options.provider, runtimeQ, runtimeQ.providerToPool, options.model);
+            if (fb && fb.pool && fb.pool !== pool) {
+              pool.state.lastReason = 'CASCADE';
+              pool.state.lastSwitchAt = now();
+              pool.state.cascadeCount = (pool.state.cascadeCount ?? 0) + 1;
+              pushEvent(pool, reqStore.pickedRef ?? 'pool', 'CASCADE_TRIGGERED', 0);
+              const innerCascade = rotate({
+                ...options,
+                provider: fb.provider,
+                model: fb.model || options.model,
+                __isCascade: true,
+                __cascadeDepth: currentDepth + 1,
+              }, fb.pool);
+              for await (const chunk of innerCascade) {
+                yield chunk;
+              }
+              return;
+            }
+          }
+          yield finishError(LOCAL_QUOTA_CODE, formatModelQuotaExhaustion(options.provider, pool));
+          return;
+        }
+      }
       const strategy = pool.routingStrategy ?? runtime0.routingStrategy ?? 'round-robin';
       if (strategy !== 'round-robin') {
         attemptList = sortAttemptList(attemptList, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap });
@@ -80,6 +144,7 @@ export function createRotate(deps) {
 
       const penalizeRef = (targetRef, errCode, errMsg) => {
         if (!targetRef) return;
+        if (errCode === LOCAL_QUOTA_CODE) return;
         const _retry = parseRetryAfter(errMsg);
         const _base = pool.cooldownMs ?? cooldownMs;
         const _max = pool.maxCooldownMs ?? maxCooldownMs;
@@ -144,6 +209,11 @@ export function createRotate(deps) {
               return llm.stream({ ...options, [MARKER]: true });
             });
           } catch (e) {
+            if (e?.localQuota || e?.code === LOCAL_QUOTA_CODE) {
+              lastFailure = finishError(LOCAL_QUOTA_CODE, String(e?.message ?? ''));
+              logWarn(`[dsh-key-rotation] ${options.provider}/${options.model ?? '-'}: local model token budget blocks dispatch — no upstream request sent`);
+              break;
+            }
             const curRef = reqStore.pickedRef ?? pool.state.lastUsed;
             penalizeRef(curRef, e?.code ?? 'TRANSPORT', String(e?.message ?? ''));
             lastFailure = finishError(e?.code ?? 'TRANSPORT',
@@ -156,7 +226,7 @@ export function createRotate(deps) {
           const acquired = (_pickedRef && runtime0.concurrencyLimit > 0) ? concurrencyTracker.acquire(_pickedRef) : false;
           try {
             try {
-              for await (const chunk of inner) {
+              for await (const chunk of preserveDispatchContext(inner, dispatchStorage, reqStore)) {
                 // Only actual content deltas lock the stream (no more rotation).
                 // Structural/metadata chunks (block-start/end, usage) do not.
                 if (chunk && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta' || chunk.type === 'tool-call-delta')) {
@@ -232,6 +302,20 @@ export function createRotate(deps) {
                   if (activeRef) {
                     recordSuccess(pool, activeRef, now());
                   }
+                  if (activeRef && quotaManaged) {
+                    const charged = consumeModelTokens(
+                      pool, activeRef, extractUsageTokens(chunk.usage),
+                      runtime0.quotaResetWindow, Date.now(),
+                    );
+                    if (charged) {
+                      if (typeof schedulePersist === 'function') {
+                        try { schedulePersist(); } catch (_) { /* best-effort persistence */ }
+                      }
+                      if (charged.exhausted) {
+                        logWarn(`[dsh-key-rotation] ${options.provider}/${options.model ?? '-'}: key ${activeRef} reached its local token budget (${charged.used}/${charged.limit}) — excluded from this model only`);
+                      }
+                    }
+                  }
 
                   // Proactive rate-limit (#115, #303): if response headers say this key is near
                   // its quota or has retry-after, cool it down so the NEXT request starts on a different key.

package/lib/webhook.js

@@ -81,8 +81,16 @@ export function formatInteractive(url, payload, actionToken) {
     };
   }
   if (platform === 'ntfy') {
+    let topic = 'alerts';
+    try {
+      const u = new URL(url);
+      const parts = u.pathname.split('/').filter(Boolean);
+      if (parts.length > 0) topic = parts[parts.length - 1];
+    } catch (_) {
+      topic = String(url).split('?')[0].split('/').filter(Boolean).pop() || 'alerts';
+    }
     return {
-      topic: url.split('/').pop(),
+      topic,
       title,
       message: text,
       priority: 4,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@goodandready/dsh-key-rotation",
-  "version": "0.8.28",
+  "version": "0.8.30",
   "packageManager": "pnpm@10.33.2",
   "description": "Per-provider API key rotation for DeepSeek Harness: a key pool per provider, auto-created clone routes, and switching to the next key on quota/rate-limit errors. Includes a Settings section (Key Rotation) to edit the key pools, cooldown and switch codes.",
   "keywords": [