@goodandready/dsh-key-rotation 0.8.29 → 0.8.30

package/lib/index.js

@@ -1,3 +1,4 @@
+import { createResolver } from './resolver.js';
 import { decryptSecret } from './crypto-storage.js';
 // dsh-key-rotation — per-provider API key rotation for DeepSeek Harness.
 //
@@ -20,7 +21,6 @@ import { LatencyHistogram } from './histogram.js';
 import { ConcurrencyTracker } from './concurrency.js';
 import { QuotaStore } from './quota.js';
 import { WebhookSender } from './webhook.js';
-import { bucketAllow, bucketRetryMs } from './bucket.js';
 import { compactUsage } from './usage-report.js';
 import { json, handleConfigBridge } from './http-bridge.js';
 import { createRotate } from './rotate.js';
@@ -35,7 +35,15 @@ import { notifySwitch, notifyExhaustion, pushEvent } from './notify-events.js';
 import { getLogger } from './logger.js';
 import { createSandboxService } from './sandbox-service.js';
 import { checkBudgetAndHealthAlerts } from './budget-monitor.js';
-import { buildPoolItem, cleanupRemovedProviders } from './pool-builder.js';
+import { cleanupRemovedProviders, buildPools } from './pool-builder.js';
+import { resolveRefPool, isQuotaManagedRequest } from './pool-index.js';
+import {
+  isModelQuotaAvailable,
+  hasModelQuotaConfig,
+  anyModelQuotaAvailable,
+  formatModelQuotaExhaustion,
+} from './model-quota.js';
+import { LOCAL_QUOTA_CODE } from './rotate.js';
 import { setupIdleHealEffect, setupAutoUnbreakEffect, setupPersistence } from './lifecycle.js';
 
 export const name = '@goodandready/dsh-key-rotation';
@@ -126,6 +134,9 @@ const SettingsSchema = Schema.object({
     models: Schema.dict(Schema.object({
       keys: Schema.array(Schema.string()).default([]),
       weights: Schema.array(Schema.number()).default([]),
+      quotas: Schema.dict(Schema.object({
+        tokenLimit: Schema.number(),
+      })).default({}),
     })).default({}),
     cooldownMs: Schema.number(),
     maxCooldownMs: Schema.number(),
@@ -195,9 +206,9 @@ export function apply(ctx, config = {}) {
       }
       const runtime = buildRuntime();
       const allActiveRefs = Array.from(runtime.poolByRef.keys());
-      const n = sweepExpired(poolState, now, allActiveRefs);
+      const n = sweepExpired(poolState, now, allActiveRefs, runtime.quotaResetWindow);
       if (n > 0) logger.warn(`[dsh-key-rotation] sweep: cleared ${n} expired cooldown(s)`);
-      for (const pool of runtime.poolByRef.values()) {
+      for (const pool of (runtime.pools ?? runtime.poolByRef.values())) {
         compactUsage(pool, 30, now);
       }
       checkBudgetAndHealthAlerts({
@@ -252,42 +263,9 @@ export function apply(ctx, config = {}) {
     const rpmLimit = cfg.rpmLimit ?? 0;
     const webhookActionToken = cfg.webhookActionToken ?? '';
 
-    const poolByRef = new Map();
-    const providerToPool = new Map();
-    const modelPoolByProvider = new Map();
-    const cloneIds = new Set();
-
-    const makeState = (base) => {
-      let st = poolState.get(base);
-      if (!st) {
-        st = initializePoolState();
-        poolState.set(base, st);
-      }
-      return initializePoolState(st);
-    };
-
-    const buildPool = (base, keys, weights, paused, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, revoked) =>
-      buildPoolItem({ base, keys, weights, paused, revoked, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, rpmLimit, makeState });
-
-    for (const p of cfg.providers ?? []) {
-      const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
-      const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
-      const pool = buildPool(p.provider, p.keys, p.weights, p.paused, poolCooldown, poolMax, p.expiresAt, p.routingStrategy, p.proactiveRateLimitGuard, p.revoked);
-      if (pool) {
-        for (const ref of pool.refs) poolByRef.set(ref, pool);
-        for (let i = 1; i < pool.refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
-      }
-      const models = p.models ?? {};
-      const byModel = new Map();
-      for (const [model, mp] of Object.entries(models)) {
-        const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, mp.paused, poolCooldown, poolMax, undefined, p.routingStrategy, p.proactiveRateLimitGuard, mp.revoked);
-        if (mpool) {
-          byModel.set(model, mpool);
-          for (const ref of mpool.refs) poolByRef.set(ref, mpool);
-        }
-      }
-      if (byModel.size > 0) modelPoolByProvider.set(p.provider, byModel);
-    }
+    const built = buildPools({ cfg, poolState, defaultState: initializePoolState });
+    const { index, pools, poolByRef, modelPoolByProvider, cloneIds } = built;
+    const providerToPool = new Map(built.providerToPool);
 
     let profiles = {};
     try {
@@ -318,6 +296,7 @@ export function apply(ctx, config = {}) {
       cfg,
       poolState,
       poolByRef,
+      pools,
       providerToPool,
       expectedClones,
       moduleBreaker,
@@ -335,13 +314,13 @@ export function apply(ctx, config = {}) {
       routingStrategy: cfg.routingStrategy ?? 'round-robin',
       switchNotifyThrottleMs: cfg.switchNotifyThrottleMs ?? 60000,
       warnBelowHealthy: cfg.warnBelowHealthy ?? 0, latencySloMs: cfg.latencySloMs ?? 0,
+      index, pools,
       providerTags, providerBudgets, poolByRef, providerToPool, modelPoolByProvider,
       cloneIds, expectedClones,
       circuitBreakerEnabled: cfg.circuitBreakerEnabled ?? true,
       circuitBreakerThreshold: cfg.circuitBreakerThreshold ?? 5,
       circuitBreakerOpenMs: cfg.circuitBreakerOpenMs ?? 30000,
       circuitBreakerHalfOpenProbes: cfg.circuitBreakerHalfOpenProbes ?? 1,
-      pools: [...new Set(poolByRef.values())],
     };
     return cachedRuntime;
   }
@@ -353,104 +332,17 @@ export function apply(ctx, config = {}) {
     if (credentials && typeof credentials.resolve === 'function') {
       const original = credentials.resolve.bind(credentials);
       credentials.__dshKeyRotationOriginalResolve = original;
-      credentials.resolve = async (ref) => {
-        const { poolByRef } = buildRuntime();
-        const pool = poolByRef.get(ref);
-        if (!pool) {
-          const res = await original(ref);
-          if (res && typeof res.value === 'string') {
-            return { ...res, value: decryptSecret(res.value) };
-          }
-          const ev = envValue(ref);
-          if (ev !== undefined) return { value: decryptSecret(ev) };
-          return res;
-        }
-        const now = Date.now();
-        const strat = pool.routingStrategy ?? buildRuntime().routingStrategy ?? 'round-robin';
-        let list = pool.weightedRefs ?? pool.refs;
-        if (strat === 'lowest-latency' || strat === 'least-loaded') {
-          list = sortAttemptList(list, strat, { latencyHistogram, concurrencyTracker });
-        }
-        const start = (strat === 'round-robin') ? (pool.state.pointer ?? 0) : 0;
-        for (let i = 0; i < list.length; i++) {
-          const index = (start + i) % list.length;
-          const candidate = list[index];
-          if (isKeyRevoked(pool, candidate)) continue;
-          if (isKeyPaused(pool, candidate)) continue;
-          const until = pool.state.failedUntil.get(candidate);
-          if (until !== undefined && until > now) continue;
-          if (pool.expiresAt?.[candidate] !== undefined && now >= pool.expiresAt[candidate]) continue;
-          const rpmLimit = pool.rpmLimit ?? 0;
-          if (rpmLimit > 0) {
-            if (!pool.state.rpmWindows) pool.state.rpmWindows = new Map();
-            if (!bucketAllow(pool.state.rpmWindows, candidate, rpmLimit, now)) {
-              const waitMs = bucketRetryMs(pool.state.rpmWindows, candidate, rpmLimit, now);
-              if ((pool.state.failedUntil.get(candidate) ?? 0) < now + waitMs) {
-                pool.state.failedUntil.set(candidate, now + waitMs);
-              }
-              continue;
-            }
-          }
-          if (pool.perHour) {
-            if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
-            let win = pool.state.quotaWindows.get(candidate);
-            if (!win || now - win.start >= 3600000) win = { count: 0, start: now };
-            if (win.count >= pool.perHour) {
-              const until = win.start + 3600000;
-              if ((pool.state.failedUntil.get(candidate) ?? 0) < until) pool.state.failedUntil.set(candidate, until);
-              continue;
-            }
-          }
-          pool.state.pointer = (index + 1) % list.length;
-          let hit = await original(candidate);
-          if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
-            hit = { ...hit, value: decryptSecret(hit.value) };
-            pool.state.lastUsed = candidate;
-            if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
-            pool.state.lastUsedAt.set(candidate, now);
-            const store = dispatchStorage.getStore();
-            if (store && store.pool === pool) store.pickedRef = candidate;
-            if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
-            pool.state.failedUntil.delete(candidate);
-            if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
-            if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
-            if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
-            pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
-            if (pool.perHour) {
-              if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
-              let win2 = pool.state.quotaWindows.get(candidate);
-              if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
-              win2.count++;
-              pool.state.quotaWindows.set(candidate, win2);
-            }
-            return hit;
-          }
-          const envVal = envValue(candidate);
-          if (envVal !== undefined) {
-            const decEnv = decryptSecret(envVal);
-            pool.state.lastUsed = candidate;
-            if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
-            pool.state.lastUsedAt.set(candidate, now);
-            const store = dispatchStorage.getStore();
-            if (store && store.pool === pool) store.pickedRef = candidate;
-            if (pool.state.failCounts) pool.state.failCounts.delete(candidate);
-            pool.state.failedUntil.delete(candidate);
-            if (pool.state.authFailCounts) pool.state.authFailCounts.delete(candidate);
-            if (pool.state.brokenUntil) pool.state.brokenUntil.delete(candidate);
-            if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
-            pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
-            if (pool.perHour) {
-              if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
-              let win2 = pool.state.quotaWindows.get(candidate);
-              if (!win2 || now - win2.start >= 3600000) win2 = { count: 0, start: now };
-              win2.count++;
-              pool.state.quotaWindows.set(candidate, win2);
-            }
-            return { value: envVal, source: 'env' };
-          }
-        }
-        return original(ref);
-      };
+      const resolve = createResolver({
+        buildRuntime,
+        currentPool: () => dispatchStorage.getStore()?.pool,
+        onPicked: (pool, candidate) => {
+          const store = dispatchStorage.getStore();
+          if (store && store.pool === pool) store.pickedRef = candidate;
+        },
+        latencyHistogram,
+        concurrencyTracker,
+      });
+      credentials.resolve = async (ref) => resolve(ref, original);
       credentials.__dshKeyRotationPatched = true;
       return () => {
         if (credentials.__dshKeyRotationPatched) {

package/lib/model-quota.js

@@ -0,0 +1,298 @@
+// lib/model-quota.js — locally configured per-(provider × model × credential)
+// token budgets: Provider × Credential Key × Model × Token Quota.
+//
+// Deliberately separate from lib/quota.js (QuotaStore), which mirrors
+// *provider-reported* rate-limit headers (x-ratelimit-remaining, retry-after).
+// A model token quota is a local budget: "this credential may spend at most N
+// tokens on this model pool within the current reset window".
+//
+// State lives inside the model pool itself (`pool.state.tokenUsage`), and a pool
+// is keyed by `provider::model`, so Key × Model isolation is structural: the
+// Sonnet pool and the Opus pool never read each other's counters. No composite
+// `KEY::model` global key is needed or used.
+//
+// Everything here is a pure function over pool state: no network, no disk, no
+// timers, no implicit clock (callers pass `now`), so `node --test` can exercise
+// it directly. Reads never mutate; only `consumeModelTokens()` /
+// `resetModelQuotaIfNeeded()` write, and both normalise the entry first.
+//
+// Timekeeping: `resetAt` is a wall-clock epoch timestamp computed by
+// nextQuotaReset() from lib/quota-window.js. Calendar maths must never use the
+// monotonic process clock — that is for cooldowns and durations only.
+
+import { nextQuotaReset } from './quota-window.js';
+
+/** Keys that must never be used as a credential ref (prototype pollution guard). */
+const UNSAFE_KEYS = new Set(['__proto__', 'constructor', 'prototype']);
+
+/** Used when quotaResetWindow is missing or unknown: a plain 24h budget window. */
+export const FALLBACK_WINDOW_MS = 86400000;
+
+/** Failure-free token count: a non-negative finite integer, else null. */
+function toTokenCount(value) {
+  if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) return null;
+  return Math.floor(value);
+}
+
+/**
+ * A usable token limit: a finite number greater than zero.
+ * Numeric strings are accepted because a legacy host can hand the raw YAML
+ * section to apply() without running it through Schemastery first.
+ * Missing, null, non-numeric and non-positive values all mean "no local limit".
+ */
+function toLimit(value) {
+  let n = value;
+  if (typeof n === 'string') {
+    if (!n.trim()) return null;
+    n = Number(n);
+  }
+  if (typeof n !== 'number' || !Number.isFinite(n) || n <= 0) return null;
+  return Math.floor(n);
+}
+
+/**
+ * Normalize a `models.<model>.quotas` config object into a frozen lookup.
+ *
+ * Accepts the documented `{ REF: { tokenLimit: N } }` and the shorthand
+ * `{ REF: N }`. Entries without a positive finite limit are dropped, which is
+ * what keeps "no quota configured" behaviourally identical to the pre-quota
+ * plugin — a missing quota is never read as a zero budget.
+ * A null-prototype object keeps a ref literally named `__proto__` harmless.
+ */
+export function normalizeModelQuotas(quotas) {
+  const out = Object.create(null);
+  if (!quotas || typeof quotas !== 'object') return Object.freeze(out);
+  for (const [ref, value] of Object.entries(quotas)) {
+    if (typeof ref !== 'string' || ref.length === 0 || UNSAFE_KEYS.has(ref)) continue;
+    const raw = value && typeof value === 'object' ? value.tokenLimit : value;
+    const limit = toLimit(raw);
+    if (limit === null) continue;
+    out[ref] = Object.freeze({ tokenLimit: limit });
+  }
+  return Object.freeze(out);
+}
+
+/** True when this pool carries at least one usable local token quota. */
+export function hasModelQuotaConfig(pool) {
+  if (!pool) return false;
+  if (typeof pool.hasModelQuota === 'boolean') return pool.hasModelQuota;
+  const q = pool.quotas;
+  return Boolean(q && Object.keys(q).length > 0);
+}
+
+/** Configured token limit for `ref` in `pool`, or null when unlimited. */
+export function getModelTokenLimit(pool, ref) {
+  if (!pool || typeof ref !== 'string' || ref.length === 0) return null;
+  const quota = pool.quotas?.[ref];
+  if (!quota) return null;
+  return toLimit(quota.tokenLimit ?? quota);
+}
+
+/** Wall-clock end of the current quota window (always > now). */
+function nextResetAt(quotaResetWindow, now) {
+  let at = null;
+  try {
+    at = nextQuotaReset(quotaResetWindow, now);
+  } catch (_) {
+    at = null;
+  }
+  return Number.isFinite(at) && at > now ? Math.floor(at) : now + FALLBACK_WINDOW_MS;
+}
+
+/** Live tokenUsage map, or null when `create` is false and none exists yet. */
+function usageMap(pool, create) {
+  if (!pool.state) {
+    if (!create) return null;
+    pool.state = {};
+  }
+  const map = pool.state.tokenUsage;
+  if (map instanceof Map) return map;
+  if (!create) return null;
+  const fresh = new Map();
+  pool.state.tokenUsage = fresh;
+  return fresh;
+}
+
+/**
+ * Normalized usage view for `ref` — the single place lazy reset happens.
+ *
+ * A stored entry that is missing, malformed, or past its `resetAt` reads as
+ * `{ used: 0, nextResetAt }`; otherwise its recorded values are returned as-is.
+ *
+ * `now` is intentionally required: reads must not consult an implicit clock.
+ * An omitted `now` degrades to "treat the stored window as still open" rather
+ * than throwing, so a mistaken call site cannot silently clear a budget.
+ * @returns {{used: number, resetAt: number, stored: boolean}}
+ */
+function readQuota(pool, ref, quotaResetWindow, now) {
+  const blank = { used: 0, resetAt: null, open: false, stored: false };
+  if (!(pool?.state?.tokenUsage instanceof Map)) return blank;
+  const entry = pool.state.tokenUsage.get(ref);
+  if (!entry || typeof entry !== 'object') return blank;
+  const used = toTokenCount(entry.used);
+  const resetAt = toTokenCount(entry.resetAt);
+  if (used === null || resetAt === null || resetAt <= 0) return blank;
+  if (Number.isFinite(now) && now >= resetAt) {
+    // Window elapsed: report zero, and hand back the next boundary lazily.
+    return { used: 0, resetAt: nextResetAt(quotaResetWindow ?? pool.quotaResetWindow, now), open: true, stored: false };
+  }
+  return { used, resetAt, open: true, stored: true };
+}
+
+/** Tokens consumed by `ref` in `pool` during the current window (0 = none/unlimited). */
+export function getModelTokenUsage(pool, ref, now) {
+  if (getModelTokenLimit(pool, ref) === null) return 0;
+  return readQuota(pool, ref, undefined, now).used;
+}
+
+/** Remaining tokens, or null when the credential is unlimited in this pool. */
+export function getModelTokenRemaining(pool, ref, now) {
+  const limit = getModelTokenLimit(pool, ref);
+  if (limit === null) return null;
+  return Math.max(0, limit - readQuota(pool, ref, undefined, now).used);
+}
+
+/**
+ * Quota status for one credential in one model pool (status/health APIs).
+ * @returns {{configured: true, limit: number, used: number, remaining: number,
+ *            resetAt: number, exhausted: boolean}|null} null when unlimited
+ */
+export function getModelQuotaStatus(pool, ref, now) {
+  const limit = getModelTokenLimit(pool, ref);
+  if (limit === null) return null;
+  const quota = readQuota(pool, ref, undefined, now);
+  const at = quota.resetAt ?? (Number.isFinite(now) ? nextResetAt(pool?.quotaResetWindow, now) : 0);
+  return {
+    configured: true,
+    limit,
+    used: quota.used,
+    remaining: Math.max(0, limit - quota.used),
+    resetAt: at,
+    exhausted: quota.used >= limit,
+  };
+}
+
+/**
+ * Can this credential receive a new request for this pool right now?
+ *
+ * `used < limit` stays available, so the final request is allowed to overshoot
+ * the limit — this version has no reservation system, so a bounded concurrent
+ * overshoot is accepted behaviour rather than a bug.
+ */
+export function isModelQuotaAvailable(pool, ref, now) {
+  const limit = getModelTokenLimit(pool, ref);
+  if (limit === null) return true;
+  return readQuota(pool, ref, undefined, now).used < limit;
+}
+
+/** True when at least one ref of `pool` still has local quota left. */
+export function anyModelQuotaAvailable(pool, now) {
+  const refs = pool?.refs ?? [];
+  for (const ref of refs) {
+    if (isModelQuotaAvailable(pool, ref, now)) return true;
+  }
+  return false;
+}
+
+/**
+ * Keep only refs not blocked by a local token quota. Pools without any quota
+ * config return an unchanged copy, so the no-quota hot path stays allocation light.
+ */
+export function filterQuotaEligible(pool, refs, now) {
+  const list = Array.isArray(refs) ? refs : [];
+  if (!hasModelQuotaConfig(pool)) return list.slice();
+  return list.filter((ref) => isModelQuotaAvailable(pool, ref, now));
+}
+
+/**
+ * Clear the counter when its window has elapsed. Safe on a cold pool (no-op).
+ * `isModelQuotaAvailable` / `getModelQuotaStatus` already reset lazily on read;
+ * this is the explicit maintenance-sweep entry point.
+ * @returns {boolean} true when a counter was actually cleared
+ */
+export function resetModelQuotaIfNeeded(pool, ref, quotaResetWindow, now) {
+  const map = usageMap(pool, false);
+  if (!map) return false;
+  const entry = map.get(ref);
+  if (!entry || typeof entry !== 'object') return false;
+  const resetAt = toTokenCount(entry.resetAt);
+  if (resetAt === null || resetAt <= 0 || !Number.isFinite(now) || now < resetAt) return false;
+  entry.used = 0;
+  entry.resetAt = nextResetAt(quotaResetWindow ?? pool.quotaResetWindow, now);
+  return true;
+}
+
+/**
+ * Charge `tokens` to `ref` in `pool` and return the resulting status.
+ *
+ * Only successful, usage-reporting requests call this. Unlimited credentials and
+ * non-positive / invalid token counts are no-ops, so usage is never fabricated
+ * and a request without a usage payload can never exhaust a budget.
+ * A first write normalises the entry, which is where lazy initialisation and
+ * lazy reset both happen (no per-key timers anywhere).
+ * @returns {object|null} resulting quota status, or null when nothing was charged
+ */
+export function consumeModelTokens(pool, ref, tokens, quotaResetWindow, now) {
+  const limit = getModelTokenLimit(pool, ref);
+  if (limit === null) return null;
+  const count = toTokenCount(tokens);
+  if (count === null || count <= 0) return null;
+  const window = quotaResetWindow ?? pool.quotaResetWindow;
+  const current = readQuota(pool, ref, window, now);
+  const resetAt = current.resetAt ?? (Number.isFinite(now) ? nextResetAt(window, now) : 0);
+  const used = Math.min(Number.MAX_SAFE_INTEGER, current.used + count);
+  usageMap(pool, true).set(ref, { used, resetAt });
+  return {
+    configured: true,
+    limit,
+    used,
+    remaining: Math.max(0, limit - used),
+    resetAt,
+    exhausted: used >= limit,
+  };
+}
+
+/** Token keys understood, in priority order (an explicit total wins). */
+const TOTAL_KEYS = ['total_tokens', 'totalTokens'];
+const PAIR_KEYS = [
+  ['input_tokens', 'output_tokens'],
+  ['inputTokens', 'outputTokens'],
+  ['prompt_tokens', 'completion_tokens'],
+  ['promptTokens', 'completionTokens'],
+];
+
+/**
+ * Extract a billable token count from a provider usage payload.
+ *
+ * Priority: explicit total > input + output > prompt + completion.
+ * Non-finite, negative, null and non-numeric values are ignored, so a malformed
+ * payload can never poison pool state with NaN or a negative counter, and a
+ * payload with no recognised field returns null ("no data") rather than 0
+ * ("measured zero") — the distinction that keeps unmeasured requests from
+ * marking a key exhausted.
+ * @returns {number|null} non-negative integer, or null when no data is present
+ */
+export function extractUsageTokens(usage) {
+  if (!usage || typeof usage !== 'object') return null;
+  for (const key of TOTAL_KEYS) {
+    const total = toTokenCount(usage[key]);
+    if (total !== null) return total;
+  }
+  for (const [a, b] of PAIR_KEYS) {
+    const va = toTokenCount(usage[a]);
+    const vb = toTokenCount(usage[b]);
+    if (va === null && vb === null) continue;
+    return (va ?? 0) + (vb ?? 0);
+  }
+  return null;
+}
+
+/**
+ * Message used when a model pool cannot dispatch because every credential has
+ * spent its local budget. Deliberately avoids upstream-error phrasing and
+ * numeric HTTP codes so this is never re-classified as a provider failure.
+ */
+export function formatModelQuotaExhaustion(provider, pool) {
+  const total = (pool?.refs ?? []).length;
+  return `[dsh-key-rotation] All ${total} keys for '${provider}' have reached their local token budget for this window; no upstream request was sent.`;
+}

package/lib/ops-keys.js

@@ -77,6 +77,7 @@ ctx.effect(() => ctx.webServer.register({
           st.authFailCounts?.delete(ref);
           st.brokenUntil?.delete(ref);
           st.revokedRefs?.delete(ref);
+          st.tokenUsage?.delete(ref);
           if (st.lastUsed === ref) st.lastUsed = undefined;
         }
         bestEffort('lastTestCache.delete', () => { lastTestCache?.delete?.(ref); }, ctx.logger);
@@ -130,6 +131,7 @@ ctx.effect(() => ctx.webServer.register({
       st.authFailCounts?.clear();
       st.brokenUntil?.clear();
       st.revokedRefs?.clear();
+      st.tokenUsage?.clear();
       st.switches = 0; st.lastReason = undefined; st.lastSwitchAt = undefined;
       let circuitReset = false;
       const br = circuitBreaker ?? buildRuntime().breaker;
@@ -143,12 +145,13 @@ ctx.effect(() => ctx.webServer.register({
     if (ref) {
       let found = false;
       for (const st of poolState.values()) {
-        if (st.failedUntil?.has(ref) || st.failCounts?.has(ref) || st.authFailCounts?.has(ref) || st.brokenUntil?.has(ref)) {
+        if (st.failedUntil?.has(ref) || st.failCounts?.has(ref) || st.authFailCounts?.has(ref) || st.brokenUntil?.has(ref) || st.tokenUsage?.has(ref)) {
           st.failedUntil?.delete(ref);
           st.failCounts?.delete(ref);
           st.authFailCounts?.delete(ref);
           st.brokenUntil?.delete(ref);
           st.revokedRefs?.delete(ref);
+          st.tokenUsage?.delete(ref);
           if (st.lastUsed === ref) st.lastUsed = undefined;
           found = true; break;
         }

package/lib/ops-status.js

@@ -1,3 +1,4 @@
+import { getModelQuotaStatus, isModelQuotaAvailable, hasModelQuotaConfig } from './model-quota.js';
 import { decryptSecret } from './crypto-storage.js';
 // lib/ops-status.js — status + health operational routes (#312 split from routes-ops.js).
 import { json } from './http-bridge.js';
@@ -56,7 +57,7 @@ ctx.effect(() => ctx.webServer.register({
     const now = Date.now();
     const seen = new Set();
     const providers = [];
-    for (const pool of poolByRef.values()) {
+    for (const pool of (runtime.pools ?? poolByRef.values())) {
       if (seen.has(pool.base)) continue;
       seen.add(pool.base);
       try {
@@ -89,6 +90,7 @@ ctx.effect(() => ctx.webServer.register({
           /* describe is optional — the card falls back to editable */
         }
         const until = pool.state.failedUntil.get(ref);
+        const modelQuota = getModelQuotaStatus(pool, ref, now);
         keys.push({
           ref,
           present,
@@ -111,10 +113,13 @@ ctx.effect(() => ctx.webServer.register({
           broken: pool.state.brokenUntil?.has(ref) ?? false,
           paused: isKeyPaused(pool, ref),
           revoked: isKeyRevoked(pool, ref),
+          modelQuota,
         });
       }
       providers.push({
         provider: pool.base,
+        model: pool.model ?? null,
+        quotasConfigured: hasModelQuotaConfig(pool),
         keys,
         tags: providerTags.get(pool.base) ?? [],
         // #260 circuit breaker state (may be null if not yet tripped)
@@ -149,7 +154,7 @@ ctx.effect(() => ctx.webServer.register({
       });
       } catch (e) {
         (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] status: pool ${pool.base} failed: ${String(e?.message ?? e)} ${e?.stack ?? ''}`);
-        providers.push({ provider: pool.base, keys: [], statusError: String(e?.message ?? e) });
+        providers.push({ provider: pool.base, model: pool.model ?? null, keys: [], statusError: String(e?.message ?? e) });
       }
     }
     json(res, 200, sanitizeSnapshot({
@@ -180,23 +185,34 @@ ctx.effect(() => ctx.webServer.register({
     const now = Date.now();
     const pools = {};
     let exhaustedAny = false;
-    const { poolByRef: pr, providerTags } = buildRuntime();
+    const runtimeH = buildRuntime();
     const seenH = new Set();
-    for (const pool of pr.values()) {
+    for (const pool of (runtimeH.pools ?? runtimeH.poolByRef.values())) {
       if (seenH.has(pool.base)) continue;
       seenH.add(pool.base);
+      const quotaAware = hasModelQuotaConfig(pool);
       let healthy = 0;
+      let quotaExhausted = 0;
       for (const ref of pool.refs) {
+        if (isKeyRevoked(pool, ref) || isKeyPaused(pool, ref)) continue;
         const until = pool.state.failedUntil.get(ref);
         if (until !== undefined && until > now) continue;
         const exp = pool.expiresAt?.[ref];
         if (exp !== undefined && now >= exp) continue;
+        if (quotaAware && !isModelQuotaAvailable(pool, ref, now)) { quotaExhausted++; continue; }
         healthy++;
       }
       const total = pool.refs.length;
       const exhausted = healthy === 0 && total > 0;
       if (exhausted) exhaustedAny = true;
-      pools[pool.base] = { healthy, total, exhausted, healthScore: computeHealthScore(pool.state) };
+      pools[pool.base] = {
+        healthy,
+        total,
+        exhausted,
+        healthScore: computeHealthScore(pool.state),
+        model: pool.model ?? null,
+        modelQuota: quotaAware ? { configured: true, exhaustedKeys: quotaExhausted } : null,
+      };
     }
     json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: typeof quotaStore?.snapshot === 'function' ? quotaStore.snapshot() : null });
   },