@goodandready/dsh-key-rotation 0.8.28 → 0.8.30

package/lib/ops-metrics.js

@@ -47,6 +47,9 @@ export function registerMetricsRoutes(ctx, deps) {
       lines.push('# HELP dsh_key_rotation_revoked_keys Number of permanently revoked keys');
       lines.push('# TYPE dsh_key_rotation_revoked_keys gauge');
 
+      lines.push('# HELP dsh_key_rotation_expired_keys Number of expired keys');
+      lines.push('# TYPE dsh_key_rotation_expired_keys gauge');
+
       lines.push('# HELP dsh_key_rotation_proactive_switches_total Number of proactive rate-limit switches');
       lines.push('# TYPE dsh_key_rotation_proactive_switches_total counter');
 
@@ -56,10 +59,11 @@ export function registerMetricsRoutes(ctx, deps) {
       for (const pool of pools) {
         const p = pool.base;
         const st = poolState.get(p);
-        let active = 0, cooling = 0, paused = 0, revoked = 0;
+        let active = 0, cooling = 0, paused = 0, revoked = 0, expired = 0;
         for (const r of (pool.refs ?? [])) {
           if (isKeyRevoked(pool, r)) { revoked++; continue; }
           if (isKeyPaused(pool, r)) { paused++; continue; }
+          if (pool.expiresAt?.[r] !== undefined && now >= pool.expiresAt[r]) { expired++; continue; }
           const fu = st?.failedUntil?.get(r) ?? 0;
           if (fu > now) cooling++;
           else active++;
@@ -68,6 +72,7 @@ export function registerMetricsRoutes(ctx, deps) {
         lines.push(`dsh_key_rotation_cooldown_keys{provider="${p}"} ${cooling}`);
         lines.push(`dsh_key_rotation_paused_keys{provider="${p}"} ${paused}`);
         lines.push(`dsh_key_rotation_revoked_keys{provider="${p}"} ${revoked}`);
+        lines.push(`dsh_key_rotation_expired_keys{provider="${p}"} ${expired}`);
         lines.push(`dsh_key_rotation_proactive_switches_total{provider="${p}"} ${st?.proactiveSwitches ?? 0}`);
         lines.push(`dsh_key_rotation_cascade_switches_total{provider="${p}"} ${st?.cascadeCount ?? 0}`);
       }

package/lib/ops-status.js

@@ -1,7 +1,11 @@
+import { getModelQuotaStatus, isModelQuotaAvailable, hasModelQuotaConfig } from './model-quota.js';
+import { decryptSecret } from './crypto-storage.js';
 // lib/ops-status.js — status + health operational routes (#312 split from routes-ops.js).
 import { json } from './http-bridge.js';
 import {
   isTrustedBridgeRequest,
+  isKeyPaused,
+  isKeyRevoked,
   keyTail,
   envValue,
   computeHealthScore,
@@ -53,7 +57,7 @@ ctx.effect(() => ctx.webServer.register({
     const now = Date.now();
     const seen = new Set();
     const providers = [];
-    for (const pool of poolByRef.values()) {
+    for (const pool of (runtime.pools ?? poolByRef.values())) {
       if (seen.has(pool.base)) continue;
       seen.add(pool.base);
       try {
@@ -86,6 +90,7 @@ ctx.effect(() => ctx.webServer.register({
           /* describe is optional — the card falls back to editable */
         }
         const until = pool.state.failedUntil.get(ref);
+        const modelQuota = getModelQuotaStatus(pool, ref, now);
         keys.push({
           ref,
           present,
@@ -106,10 +111,15 @@ ctx.effect(() => ctx.webServer.register({
           expiresAt: pool.expiresAt?.[ref] ?? null,
           expired: pool.expiresAt?.[ref] !== undefined && now >= pool.expiresAt[ref],
           broken: pool.state.brokenUntil?.has(ref) ?? false,
+          paused: isKeyPaused(pool, ref),
+          revoked: isKeyRevoked(pool, ref),
+          modelQuota,
         });
       }
       providers.push({
         provider: pool.base,
+        model: pool.model ?? null,
+        quotasConfigured: hasModelQuotaConfig(pool),
         keys,
         tags: providerTags.get(pool.base) ?? [],
         // #260 circuit breaker state (may be null if not yet tripped)
@@ -144,7 +154,7 @@ ctx.effect(() => ctx.webServer.register({
       });
       } catch (e) {
         (ctx?.logger ? ctx.logger('dsh-key-rotation') : null)?.warn?.(`[dsh-key-rotation] status: pool ${pool.base} failed: ${String(e?.message ?? e)} ${e?.stack ?? ''}`);
-        providers.push({ provider: pool.base, keys: [], statusError: String(e?.message ?? e) });
+        providers.push({ provider: pool.base, model: pool.model ?? null, keys: [], statusError: String(e?.message ?? e) });
       }
     }
     json(res, 200, sanitizeSnapshot({
@@ -175,23 +185,34 @@ ctx.effect(() => ctx.webServer.register({
     const now = Date.now();
     const pools = {};
     let exhaustedAny = false;
-    const { poolByRef: pr, providerTags } = buildRuntime();
+    const runtimeH = buildRuntime();
     const seenH = new Set();
-    for (const pool of pr.values()) {
+    for (const pool of (runtimeH.pools ?? runtimeH.poolByRef.values())) {
       if (seenH.has(pool.base)) continue;
       seenH.add(pool.base);
+      const quotaAware = hasModelQuotaConfig(pool);
       let healthy = 0;
+      let quotaExhausted = 0;
       for (const ref of pool.refs) {
+        if (isKeyRevoked(pool, ref) || isKeyPaused(pool, ref)) continue;
         const until = pool.state.failedUntil.get(ref);
         if (until !== undefined && until > now) continue;
         const exp = pool.expiresAt?.[ref];
         if (exp !== undefined && now >= exp) continue;
+        if (quotaAware && !isModelQuotaAvailable(pool, ref, now)) { quotaExhausted++; continue; }
         healthy++;
       }
       const total = pool.refs.length;
       const exhausted = healthy === 0 && total > 0;
       if (exhausted) exhaustedAny = true;
-      pools[pool.base] = { healthy, total, exhausted, healthScore: computeHealthScore(pool.state) };
+      pools[pool.base] = {
+        healthy,
+        total,
+        exhausted,
+        healthScore: computeHealthScore(pool.state),
+        model: pool.model ?? null,
+        modelQuota: quotaAware ? { configured: true, exhaustedKeys: quotaExhausted } : null,
+      };
     }
     json(res, 200, { status: exhaustedAny ? 'degraded' : 'ok', pools, exhaustedAny, latency: latencyHistogram.snapshotAll(), quota: typeof quotaStore?.snapshot === 'function' ? quotaStore.snapshot() : null });
   },

package/lib/persistence.js

@@ -35,11 +35,25 @@ export class StatePersistence {
             if (Number.isFinite(until)) failedUntil[ref] = until;
           });
         }
-        pools[base] = {
+        const tokenUsage = {};
+        if (st && st.tokenUsage && typeof st.tokenUsage.forEach === 'function') {
+          st.tokenUsage.forEach((entry, ref) => {
+            const used = Number(entry?.used);
+            const resetAt = Number(entry?.resetAt);
+            if (!Number.isFinite(used) || used < 0) return;
+            if (!Number.isFinite(resetAt) || resetAt <= 0) return;
+            tokenUsage[ref] = { used: Math.floor(used), resetAt: Math.floor(resetAt) };
+          });
+        }
+        const outPool = {
           failedUntil,
           pointer: Number.isFinite(st?.pointer) ? st.pointer : 0,
           lastUsed: typeof st?.lastUsed === 'string' ? st.lastUsed : null,
         };
+        if (Object.keys(tokenUsage).length > 0) {
+          outPool.tokenUsage = tokenUsage;
+        }
+        pools[base] = outPool;
       });
     }
     return {
@@ -115,6 +129,16 @@ export class StatePersistence {
             if (Number.isFinite(until)) state.failedUntil.set(ref, until);
           }
         }
+        if (saved.tokenUsage && typeof saved.tokenUsage === 'object' && !Array.isArray(saved.tokenUsage)) {
+          for (const [ref, entry] of Object.entries(saved.tokenUsage)) {
+            if (!entry || typeof entry !== 'object' || Array.isArray(entry)) continue;
+            const used = Number(entry.used);
+            const resetAt = Number(entry.resetAt);
+            if (!Number.isFinite(used) || used < 0) continue;
+            if (!Number.isFinite(resetAt) || resetAt <= 0) continue;
+            state.tokenUsage.set(ref, { used: Math.floor(used), resetAt: Math.floor(resetAt) });
+          }
+        }
         state.pointer = Number.isSafeInteger(saved.pointer) && saved.pointer >= 0 ? saved.pointer : 0;
         state.lastUsed = typeof saved.lastUsed === 'string' ? saved.lastUsed : null;
       }

package/lib/pool-builder.js

@@ -1,6 +1,9 @@
 // lib/pool-builder.js — provider & per-model pool assembly and cleanup
 import { bucketSweep } from './bucket.js';
 import { isValidRef } from './pool.js';
+import { normalizeModelQuotas } from './model-quota.js';
+import { createPoolIndex, addProviderPools } from './pool-index.js';
+import { initializePoolState } from './pool-state.js';
 
 export function parseExpiry(v) {
   if (typeof v === 'number' && Number.isFinite(v) && v > 0) return v;
@@ -24,6 +27,10 @@ export function buildPoolItem({
   poolGuard,
   rpmLimit,
   makeState,
+  provider = null,
+  model = null,
+  quotas = null,
+  quotaResetWindow = null,
 }) {
   const seen = new Set();
   const indexed = (Array.isArray(keys) ? keys : []).map((key, index) => ({ ref: typeof key === 'string' ? key.trim() : '', index }))
@@ -62,8 +69,14 @@ export function buildPoolItem({
   for (let i = 0; i < refs.length; i++) {
     weightsMap[refs[i]] = typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1;
   }
+  const normalizedQuotas = normalizeModelQuotas(quotas);
   return {
     base,
+    provider,
+    model,
+    quotas: normalizedQuotas,
+    hasModelQuota: Object.keys(normalizedQuotas).length > 0,
+    quotaResetWindow,
     refs,
     weights: refs.map((_, i) => (typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1)),
     weightsMap,
@@ -80,29 +93,106 @@ export function buildPoolItem({
   };
 }
 
+export function buildPools({ cfg = {}, poolState = new Map(), defaultState = initializePoolState } = {}) {
+  const rpmLimit = cfg.rpmLimit ?? 0;
+  const quotaResetWindow = cfg.quotaResetWindow ?? null;
+  const poolByRef = new Map();
+  const providerToPool = new Map();
+  const modelPoolByProvider = new Map();
+  const cloneIds = new Set();
+  const index = createPoolIndex();
+
+  const makeState = (base) => {
+    let st = poolState.get(base);
+    if (!st) {
+      st = initializePoolState(defaultState());
+      poolState.set(base, st);
+    }
+    return initializePoolState(st);
+  };
+
+  const buildPool = (base, keys, weights, paused, revoked, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, meta) =>
+    buildPoolItem({
+      base, keys, weights, paused, revoked, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard,
+      rpmLimit, makeState, quotaResetWindow, ...meta,
+    });
+
+  for (const p of cfg.providers ?? []) {
+    const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
+    const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
+    const pool = buildPool(p.provider, p.keys, p.weights, p.paused, p.revoked, poolCooldown, poolMax, p.expiresAt, p.routingStrategy, p.proactiveRateLimitGuard, {
+      provider: p.provider,
+      model: null,
+    });
+    if (pool) {
+      for (const ref of pool.refs) {
+        if (!poolByRef.has(ref)) poolByRef.set(ref, pool);
+      }
+      for (let i = 1; i < pool.refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
+      if (!providerToPool.has(p.provider)) providerToPool.set(p.provider, pool);
+    }
+    const byModel = new Map();
+    for (const [model, mp] of Object.entries(p.models ?? {})) {
+      const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, mp.paused, mp.revoked, poolCooldown, poolMax, undefined, p.routingStrategy, p.proactiveRateLimitGuard, {
+        provider: p.provider,
+        model,
+        quotas: mp.quotas,
+      });
+      if (!mpool) continue;
+      byModel.set(model, mpool);
+      for (const ref of mpool.refs) {
+        if (!poolByRef.has(ref)) poolByRef.set(ref, mpool);
+      }
+    }
+    if (byModel.size > 0) modelPoolByProvider.set(p.provider, byModel);
+    addProviderPools(index, p.provider, pool, byModel);
+  }
+
+  return {
+    index,
+    pools: [...index.allPools.values()],
+    poolByRef,
+    providerToPool,
+    modelPoolByProvider,
+    cloneIds,
+    poolState,
+  };
+}
+
 export function cleanupRemovedProviders({
   cfg,
   poolState,
   poolByRef,
+  pools,
   providerToPool,
   expectedClones,
   moduleBreaker,
   lowHealthNotifiedAt,
   budgetNotifiedAt,
 }) {
-  // auto-cleanup: remove poolState for providers that are now empty or removed
+  const live = Array.isArray(pools) ? pools : [...(pools?.values?.() ?? (poolByRef?.values?.() ? [...poolByRef.values()] : []))];
+  const liveBases = new Set(live.map((p) => p.base));
   for (const key of [...poolState.keys()]) {
-    if (![...poolByRef.values()].some((p) => p.base === key)) {
+    if (!liveBases.has(key)) {
       poolState.delete(key);
       lowHealthNotifiedAt?.delete?.(key);
-      budgetNotifiedAt?.delete?.(key + ':budget');
+      budgetNotifiedAt?.delete?.(`${key}:budget`);
+    }
+  }
+  const liveRefs = new Set();
+  for (const pool of live) {
+    if (Array.isArray(pool?.refs)) {
+      for (const ref of pool.refs) liveRefs.add(ref);
     }
   }
-  // #192: drop RPM windows for refs that no longer belong to any pool
   for (const st of poolState.values()) {
-    if (st.rpmWindows) bucketSweep(st.rpmWindows, new Set(poolByRef.keys()));
+    if (st.rpmWindows) bucketSweep(st.rpmWindows, liveRefs);
+    if (st.tokenUsage instanceof Map) {
+      for (const ref of [...st.tokenUsage.keys()]) {
+        if (!liveRefs.has(ref)) st.tokenUsage.delete(ref);
+      }
+    }
   }
-  // drop breaker entries for removed providers
   if (moduleBreaker) {
     for (const key of Object.keys(moduleBreaker.snapshot())) {
       if (![...providerToPool.keys()].includes(key) && !expectedClones.has(key)) {

package/lib/pool-index.js

@@ -0,0 +1,143 @@
+// lib/pool-index.js — which pool owns a credential ref for a given request.
+//
+// One credential ref can legitimately live in several pools at once:
+//
+//   anthropic::claude-sonnet  -> CLAUDE_KEY_A, CLAUDE_KEY_B
+//   anthropic::claude-opus    -> CLAUDE_KEY_A, CLAUDE_KEY_B
+//
+// A single `Map<ref, pool>` therefore cannot answer "which pool is this request
+// using?" — the last pool written wins and every other model pool becomes
+// invisible. The runtime keeps instead:
+//
+//   allPools          Map<poolBase, pool>      every pool, base and per-model
+//   basePoolByRef     Map<ref, basePool>       provider-base ownership only
+//   modelPoolsByProvider Map<provider, Map<model, pool>>
+//   modelPoolByRef    Map<ref, Set<modelPool>> every model pool a ref appears in
+//
+// `poolByRef` still exists for single-pool compatibility lookups, but it is no
+// longer the enumeration source for persistence, status, health or sweeps.
+
+/** Provider id of a pool base (`anthropic::claude-opus` -> `anthropic`). */
+export function providerOfBase(base) {
+  if (typeof base !== 'string') return '';
+  const at = base.indexOf('::');
+  return at === -1 ? base : base.slice(0, at);
+}
+
+/**
+ * Build an empty index. Call `addProvider()` once per configured provider.
+ * @returns {object} index consumed by `resolveRefPool()` / `allPools()`
+ */
+export function createPoolIndex() {
+  const index = {
+    // Complete pool inventory, keyed by pool base, so no pool is ever lost to
+    // a ref collision. Iterate `allPools`, never `poolByRef`, to enumerate.
+    allPools: new Map(),
+    basePoolByRef: new Map(),
+    modelPoolsByProvider: new Map(),
+    modelPoolByRef: new Map(),
+    basePoolByProvider: new Map(),
+  };
+  return index;
+}
+
+/** Register one provider's base pool and its `model -> pool` sub-pools. */
+export function addProviderPools(index, provider, basePool, modelPools) {
+  if (basePool) {
+    index.allPools.set(basePool.base, basePool);
+    index.basePoolByProvider.set(provider, basePool);
+    for (const ref of basePool.refs) {
+      if (!index.basePoolByRef.has(ref)) index.basePoolByRef.set(ref, basePool);
+    }
+  }
+  if (!modelPools || modelPools.size === 0) return;
+  index.modelPoolsByProvider.set(provider, modelPools);
+  for (const pool of modelPools.values()) {
+    index.allPools.set(pool.base, pool);
+    for (const ref of pool.refs) {
+      let set = index.modelPoolByRef.get(ref);
+      if (!set) { set = new Set(); index.modelPoolByRef.set(ref, set); }
+      set.add(pool);
+    }
+  }
+}
+
+/** Every pool of the index, deduplicated by base. */
+export function allPools(index) {
+  return [...index.allPools.values()];
+}
+
+/** Provider-base pool for a pool of any depth (base pool returns itself). */
+export function basePoolOf(index, pool) {
+  if (!pool) return null;
+  if (!pool.model) return pool;
+  return index.basePoolByProvider.get(pool.provider) ?? pool;
+}
+
+/** The provider-base pool that a pool base belongs to, or null. */
+export function basePoolForBase(index, base) {
+  const own = index.allPools.get(base);
+  if (!own) return null;
+  return basePoolOf(index, own);
+}
+
+/**
+ * Pool that owns `ref` for the request currently being dispatched.
+ *
+ * `requested` is the request-scoped pool chosen for this LLM call (set by
+ * rotate() in the dispatch AsyncLocalStorage). It is authoritative: a Sonnet
+ * request must rotate inside the Sonnet pool even when the incoming ref is the
+ * provider profile's original `apiKeyEnv`, because replacing that key with the
+ * model-specific key pool is the entire point of a model sub-pool.
+ *
+ * The one thing a request-scoped pool must not do is hijack an unrelated
+ * credential resolved during the same request, so substitution is limited to
+ * refs that belong to this pool or to its provider-base pool:
+ *
+ *   ref in requested.refs            -> requested  (already a model-pool key)
+ *   ref in provider base pool refs   -> requested  (substitute the model pool)
+ *   provider has no base pool        -> requested  (model pool is the only source)
+ *   anything else                    -> null       (plain original resolve)
+ *
+ * With no request-scoped pool (ordinary `credentials.resolve()` outside
+ * rotation) the global lookup runs, preferring the provider-base pool.
+ *
+ * @param {object} index result of createPoolIndex()
+ * @param {string} ref credential ref being resolved
+ * @param {object|null|undefined} requested request-scoped pool
+ * @returns {object|null} pool to rotate within, or null to use the original resolve
+ */
+export function resolveRefPool(index, ref, requested) {
+  if (typeof ref !== 'string' || ref.length === 0) return null;
+  if (requested) {
+    if (Array.isArray(requested.refs) && requested.refs.includes(ref)) return requested;
+    if (!requested.model) return null;
+    const base = index.basePoolByProvider.get(requested.provider);
+    if (!base) return requested;
+    return base.refs.includes(ref) ? requested : null;
+  }
+  const base = index.basePoolByRef.get(ref);
+  if (base) return base;
+  const set = index.modelPoolByRef.get(ref);
+  return set && set.size > 0 ? set.values().next().value : null;
+}
+
+/**
+ * True when the resolver may still rotate for this ref even though every
+ * candidate is locally out of budget. Quota enforcement is fail-closed only for
+ * requests this plugin actually manages; an ordinary `credentials.resolve()`
+ * outside rotation keeps its historical behaviour.
+ */
+export function isQuotaManagedRequest(index, ref, requested) {
+  if (!requested) return false;
+  return resolveRefPool(index, ref, requested) === requested;
+}
+
+/**
+ * Model pools a ref appears in. Used by ops routes to describe one credential
+ * across the models it serves without guessing from the pool base string.
+ */
+export function modelPoolsForRef(index, ref) {
+  const set = index.modelPoolByRef.get(ref);
+  return set ? [...set] : [];
+}

package/lib/pool-state.js

@@ -3,6 +3,7 @@
 const MAP_FIELDS = [
   'failedUntil', 'failCounts', 'authFailCounts', 'brokenUntil', 'costPerKey',
   'lastUsedAt', 'usageCounts', 'byModel', 'usageDays', 'quotaWindows', 'costDays',
+  'tokenUsage',
 ];
 
 export function initializePoolState(state = {}) {

package/lib/pool.js

@@ -1,3 +1,5 @@
+import { resetModelQuotaIfNeeded } from './model-quota.js';
+import { decryptSecret } from './crypto-storage.js';
 import { bestEffort } from './best-effort.js';
 // lib/pool.js — pure pool arithmetic and selection logic for dsh-key-rotation.
 // Isolated from cordis/dsh runtime so unit tests run in vanilla Node.js.
@@ -11,8 +13,9 @@ export const KEY_TAIL_PLACEHOLDER = '***';
 /** Mask key to show only the last KEY_TAIL_CHARS; returns placeholder for short keys (#355). */
 export function keyTail(key) {
   if (typeof key !== 'string' || key.length === 0) return '';
-  if (key.length <= KEY_TAIL_CHARS) return KEY_TAIL_PLACEHOLDER;
-  return key.slice(-KEY_TAIL_CHARS);
+  const plain = decryptSecret(key);
+  if (plain.length <= KEY_TAIL_CHARS) return KEY_TAIL_PLACEHOLDER;
+  return plain.slice(-KEY_TAIL_CHARS);
 }
 
 /** Check if an IP address is a loopback address. */
@@ -23,15 +26,26 @@ export function isLoopbackAddress(ip) {
   return false;
 }
 
-/** Check if an incoming HTTP request is from a trusted local bridge origin (#353 fail-closed). */
+/** Strip an optional :port and IPv6 brackets from a Host header value. */
+function hostNameOf(hostHeader) {
+  return String(hostHeader).replace(/:\d+$/, '').replace(/^\[|\]$/g, '').toLowerCase();
+}
+
+/** Check if an incoming HTTP request is from a trusted local bridge origin (#353 fail-closed, #378 method-aware). */
 export function isTrustedBridgeRequest(req) {
   const remoteAddress = req?.socket?.remoteAddress;
   if (!isLoopbackAddress(remoteAddress)) return false;
-  const origin = req?.headers?.origin;
-  if (!origin) return false;
   if (req?.headers?.['sec-fetch-site'] === 'cross-site') return false;
   const hostHeader = req?.headers?.host;
   if (!hostHeader) return false;
+  const hostName = hostNameOf(hostHeader);
+  if (!(isLoopbackAddress(hostName) || hostName === 'localhost')) return false;
+
+  const origin = req?.headers?.origin;
+  if (!origin) {
+    const method = String(req?.method ?? 'GET').toUpperCase();
+    return method === 'GET' || method === 'HEAD' || method === 'OPTIONS';
+  }
   try {
     const originUrl = new URL(origin);
     if (originUrl.host !== hostHeader) return false;
@@ -187,7 +201,7 @@ export function envValue(ref) {
 }
 
 /** Sweep expired cooldown entries and prune stale deleted refs from poolState. Returns count of cleared refs. */
-export function sweepExpired(poolState, now = Date.now(), activeRefs = null) {
+export function sweepExpired(poolState, now = Date.now(), activeRefs = null, quotaResetWindow = null) {
   let cleared = 0;
   const activeSet = activeRefs ? new Set(activeRefs) : null;
   for (const st of poolState.values()) {
@@ -201,9 +215,17 @@ export function sweepExpired(poolState, now = Date.now(), activeRefs = null) {
         cleared++;
       }
     }
+    // Local model token budgets roll over on their own wall-clock window, never
+    // on a cooldown. Recomputing the boundary here keeps a long-lived idle pool
+    // from carrying an expired resetAt until its next request.
+    if (st.tokenUsage instanceof Map && quotaResetWindow) {
+      for (const ref of [...st.tokenUsage.keys()]) {
+        resetModelQuotaIfNeeded({ state: st, quotaResetWindow }, ref, quotaResetWindow, now);
+      }
+    }
     // Prune stale entries for keys no longer in active configuration
     if (activeSet) {
-      for (const map of [st.failedUntil, st.failCounts, st.authFailCounts, st.brokenUntil, st.costPerKey, st.lastUsedAt, st.usageCounts, st.byModel, st.usageDays, st.quotaWindows, st.rpmWindows, st.probedAt]) {
+      for (const map of [st.failedUntil, st.failCounts, st.authFailCounts, st.brokenUntil, st.costPerKey, st.lastUsedAt, st.usageCounts, st.byModel, st.usageDays, st.quotaWindows, st.rpmWindows, st.probedAt, st.tokenUsage]) {
         if (map && typeof map.keys === 'function') {
           for (const k of [...map.keys()]) {
             if (!activeSet.has(k)) map.delete(k);
@@ -290,9 +312,10 @@ export function parseDurationToSeconds(v) {
 /** Check if a key reference is in maintenance/paused mode. */
 export function isKeyPaused(pool, ref) {
   if (!pool || !ref) return false;
-  if (pool.pausedRefs instanceof Set) return pool.pausedRefs.has(ref);
-  if (Array.isArray(pool.pausedRefs)) return pool.pausedRefs.includes(ref);
-  if (pool.state && pool.state.pausedRefs instanceof Set) return pool.state.pausedRefs.has(ref);
+  if (pool.pausedRefs instanceof Set && pool.pausedRefs.has(ref)) return true;
+  if (Array.isArray(pool.pausedRefs) && pool.pausedRefs.includes(ref)) return true;
+  if (pool.state && pool.state.pausedRefs instanceof Set && pool.state.pausedRefs.has(ref)) return true;
+  if (pool.state && Array.isArray(pool.state.pausedRefs) && pool.state.pausedRefs.includes(ref)) return true;
   return false;
 }