@goodandready/dsh-key-rotation 0.8.29 → 0.8.30

package/lib/persistence.js

@@ -35,11 +35,25 @@ export class StatePersistence {
             if (Number.isFinite(until)) failedUntil[ref] = until;
           });
         }
-        pools[base] = {
+        const tokenUsage = {};
+        if (st && st.tokenUsage && typeof st.tokenUsage.forEach === 'function') {
+          st.tokenUsage.forEach((entry, ref) => {
+            const used = Number(entry?.used);
+            const resetAt = Number(entry?.resetAt);
+            if (!Number.isFinite(used) || used < 0) return;
+            if (!Number.isFinite(resetAt) || resetAt <= 0) return;
+            tokenUsage[ref] = { used: Math.floor(used), resetAt: Math.floor(resetAt) };
+          });
+        }
+        const outPool = {
           failedUntil,
           pointer: Number.isFinite(st?.pointer) ? st.pointer : 0,
           lastUsed: typeof st?.lastUsed === 'string' ? st.lastUsed : null,
         };
+        if (Object.keys(tokenUsage).length > 0) {
+          outPool.tokenUsage = tokenUsage;
+        }
+        pools[base] = outPool;
       });
     }
     return {
@@ -115,6 +129,16 @@ export class StatePersistence {
             if (Number.isFinite(until)) state.failedUntil.set(ref, until);
           }
         }
+        if (saved.tokenUsage && typeof saved.tokenUsage === 'object' && !Array.isArray(saved.tokenUsage)) {
+          for (const [ref, entry] of Object.entries(saved.tokenUsage)) {
+            if (!entry || typeof entry !== 'object' || Array.isArray(entry)) continue;
+            const used = Number(entry.used);
+            const resetAt = Number(entry.resetAt);
+            if (!Number.isFinite(used) || used < 0) continue;
+            if (!Number.isFinite(resetAt) || resetAt <= 0) continue;
+            state.tokenUsage.set(ref, { used: Math.floor(used), resetAt: Math.floor(resetAt) });
+          }
+        }
         state.pointer = Number.isSafeInteger(saved.pointer) && saved.pointer >= 0 ? saved.pointer : 0;
         state.lastUsed = typeof saved.lastUsed === 'string' ? saved.lastUsed : null;
       }

package/lib/pool-builder.js

@@ -1,6 +1,9 @@
 // lib/pool-builder.js — provider & per-model pool assembly and cleanup
 import { bucketSweep } from './bucket.js';
 import { isValidRef } from './pool.js';
+import { normalizeModelQuotas } from './model-quota.js';
+import { createPoolIndex, addProviderPools } from './pool-index.js';
+import { initializePoolState } from './pool-state.js';
 
 export function parseExpiry(v) {
   if (typeof v === 'number' && Number.isFinite(v) && v > 0) return v;
@@ -24,6 +27,10 @@ export function buildPoolItem({
   poolGuard,
   rpmLimit,
   makeState,
+  provider = null,
+  model = null,
+  quotas = null,
+  quotaResetWindow = null,
 }) {
   const seen = new Set();
   const indexed = (Array.isArray(keys) ? keys : []).map((key, index) => ({ ref: typeof key === 'string' ? key.trim() : '', index }))
@@ -62,8 +69,14 @@ export function buildPoolItem({
   for (let i = 0; i < refs.length; i++) {
     weightsMap[refs[i]] = typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1;
   }
+  const normalizedQuotas = normalizeModelQuotas(quotas);
   return {
     base,
+    provider,
+    model,
+    quotas: normalizedQuotas,
+    hasModelQuota: Object.keys(normalizedQuotas).length > 0,
+    quotaResetWindow,
     refs,
     weights: refs.map((_, i) => (typeof w[i] === 'number' && w[i] > 0 ? Math.floor(w[i]) : 1)),
     weightsMap,
@@ -80,29 +93,106 @@ export function buildPoolItem({
   };
 }
 
+export function buildPools({ cfg = {}, poolState = new Map(), defaultState = initializePoolState } = {}) {
+  const rpmLimit = cfg.rpmLimit ?? 0;
+  const quotaResetWindow = cfg.quotaResetWindow ?? null;
+  const poolByRef = new Map();
+  const providerToPool = new Map();
+  const modelPoolByProvider = new Map();
+  const cloneIds = new Set();
+  const index = createPoolIndex();
+
+  const makeState = (base) => {
+    let st = poolState.get(base);
+    if (!st) {
+      st = initializePoolState(defaultState());
+      poolState.set(base, st);
+    }
+    return initializePoolState(st);
+  };
+
+  const buildPool = (base, keys, weights, paused, revoked, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard, meta) =>
+    buildPoolItem({
+      base, keys, weights, paused, revoked, poolCooldown, poolMax, expiresAt, poolStrategy, poolGuard,
+      rpmLimit, makeState, quotaResetWindow, ...meta,
+    });
+
+  for (const p of cfg.providers ?? []) {
+    const poolCooldown = typeof p.cooldownMs === 'number' ? p.cooldownMs : (cfg.cooldownMs ?? 60000);
+    const poolMax = typeof p.maxCooldownMs === 'number' ? p.maxCooldownMs : (cfg.maxCooldownMs ?? undefined);
+    const pool = buildPool(p.provider, p.keys, p.weights, p.paused, p.revoked, poolCooldown, poolMax, p.expiresAt, p.routingStrategy, p.proactiveRateLimitGuard, {
+      provider: p.provider,
+      model: null,
+    });
+    if (pool) {
+      for (const ref of pool.refs) {
+        if (!poolByRef.has(ref)) poolByRef.set(ref, pool);
+      }
+      for (let i = 1; i < pool.refs.length; i++) cloneIds.add(`${p.provider}-${i + 1}`);
+      if (!providerToPool.has(p.provider)) providerToPool.set(p.provider, pool);
+    }
+    const byModel = new Map();
+    for (const [model, mp] of Object.entries(p.models ?? {})) {
+      const mpool = buildPool(`${p.provider}::${model}`, mp.keys, mp.weights, mp.paused, mp.revoked, poolCooldown, poolMax, undefined, p.routingStrategy, p.proactiveRateLimitGuard, {
+        provider: p.provider,
+        model,
+        quotas: mp.quotas,
+      });
+      if (!mpool) continue;
+      byModel.set(model, mpool);
+      for (const ref of mpool.refs) {
+        if (!poolByRef.has(ref)) poolByRef.set(ref, mpool);
+      }
+    }
+    if (byModel.size > 0) modelPoolByProvider.set(p.provider, byModel);
+    addProviderPools(index, p.provider, pool, byModel);
+  }
+
+  return {
+    index,
+    pools: [...index.allPools.values()],
+    poolByRef,
+    providerToPool,
+    modelPoolByProvider,
+    cloneIds,
+    poolState,
+  };
+}
+
 export function cleanupRemovedProviders({
   cfg,
   poolState,
   poolByRef,
+  pools,
   providerToPool,
   expectedClones,
   moduleBreaker,
   lowHealthNotifiedAt,
   budgetNotifiedAt,
 }) {
-  // auto-cleanup: remove poolState for providers that are now empty or removed
+  const live = Array.isArray(pools) ? pools : [...(pools?.values?.() ?? (poolByRef?.values?.() ? [...poolByRef.values()] : []))];
+  const liveBases = new Set(live.map((p) => p.base));
   for (const key of [...poolState.keys()]) {
-    if (![...poolByRef.values()].some((p) => p.base === key)) {
+    if (!liveBases.has(key)) {
       poolState.delete(key);
       lowHealthNotifiedAt?.delete?.(key);
-      budgetNotifiedAt?.delete?.(key + ':budget');
+      budgetNotifiedAt?.delete?.(`${key}:budget`);
+    }
+  }
+  const liveRefs = new Set();
+  for (const pool of live) {
+    if (Array.isArray(pool?.refs)) {
+      for (const ref of pool.refs) liveRefs.add(ref);
     }
   }
-  // #192: drop RPM windows for refs that no longer belong to any pool
   for (const st of poolState.values()) {
-    if (st.rpmWindows) bucketSweep(st.rpmWindows, new Set(poolByRef.keys()));
+    if (st.rpmWindows) bucketSweep(st.rpmWindows, liveRefs);
+    if (st.tokenUsage instanceof Map) {
+      for (const ref of [...st.tokenUsage.keys()]) {
+        if (!liveRefs.has(ref)) st.tokenUsage.delete(ref);
+      }
+    }
   }
-  // drop breaker entries for removed providers
   if (moduleBreaker) {
     for (const key of Object.keys(moduleBreaker.snapshot())) {
       if (![...providerToPool.keys()].includes(key) && !expectedClones.has(key)) {

package/lib/pool-index.js

@@ -0,0 +1,143 @@
+// lib/pool-index.js — which pool owns a credential ref for a given request.
+//
+// One credential ref can legitimately live in several pools at once:
+//
+//   anthropic::claude-sonnet  -> CLAUDE_KEY_A, CLAUDE_KEY_B
+//   anthropic::claude-opus    -> CLAUDE_KEY_A, CLAUDE_KEY_B
+//
+// A single `Map<ref, pool>` therefore cannot answer "which pool is this request
+// using?" — the last pool written wins and every other model pool becomes
+// invisible. The runtime keeps instead:
+//
+//   allPools          Map<poolBase, pool>      every pool, base and per-model
+//   basePoolByRef     Map<ref, basePool>       provider-base ownership only
+//   modelPoolsByProvider Map<provider, Map<model, pool>>
+//   modelPoolByRef    Map<ref, Set<modelPool>> every model pool a ref appears in
+//
+// `poolByRef` still exists for single-pool compatibility lookups, but it is no
+// longer the enumeration source for persistence, status, health or sweeps.
+
+/** Provider id of a pool base (`anthropic::claude-opus` -> `anthropic`). */
+export function providerOfBase(base) {
+  if (typeof base !== 'string') return '';
+  const at = base.indexOf('::');
+  return at === -1 ? base : base.slice(0, at);
+}
+
+/**
+ * Build an empty index. Call `addProvider()` once per configured provider.
+ * @returns {object} index consumed by `resolveRefPool()` / `allPools()`
+ */
+export function createPoolIndex() {
+  const index = {
+    // Complete pool inventory, keyed by pool base, so no pool is ever lost to
+    // a ref collision. Iterate `allPools`, never `poolByRef`, to enumerate.
+    allPools: new Map(),
+    basePoolByRef: new Map(),
+    modelPoolsByProvider: new Map(),
+    modelPoolByRef: new Map(),
+    basePoolByProvider: new Map(),
+  };
+  return index;
+}
+
+/** Register one provider's base pool and its `model -> pool` sub-pools. */
+export function addProviderPools(index, provider, basePool, modelPools) {
+  if (basePool) {
+    index.allPools.set(basePool.base, basePool);
+    index.basePoolByProvider.set(provider, basePool);
+    for (const ref of basePool.refs) {
+      if (!index.basePoolByRef.has(ref)) index.basePoolByRef.set(ref, basePool);
+    }
+  }
+  if (!modelPools || modelPools.size === 0) return;
+  index.modelPoolsByProvider.set(provider, modelPools);
+  for (const pool of modelPools.values()) {
+    index.allPools.set(pool.base, pool);
+    for (const ref of pool.refs) {
+      let set = index.modelPoolByRef.get(ref);
+      if (!set) { set = new Set(); index.modelPoolByRef.set(ref, set); }
+      set.add(pool);
+    }
+  }
+}
+
+/** Every pool of the index, deduplicated by base. */
+export function allPools(index) {
+  return [...index.allPools.values()];
+}
+
+/** Provider-base pool for a pool of any depth (base pool returns itself). */
+export function basePoolOf(index, pool) {
+  if (!pool) return null;
+  if (!pool.model) return pool;
+  return index.basePoolByProvider.get(pool.provider) ?? pool;
+}
+
+/** The provider-base pool that a pool base belongs to, or null. */
+export function basePoolForBase(index, base) {
+  const own = index.allPools.get(base);
+  if (!own) return null;
+  return basePoolOf(index, own);
+}
+
+/**
+ * Pool that owns `ref` for the request currently being dispatched.
+ *
+ * `requested` is the request-scoped pool chosen for this LLM call (set by
+ * rotate() in the dispatch AsyncLocalStorage). It is authoritative: a Sonnet
+ * request must rotate inside the Sonnet pool even when the incoming ref is the
+ * provider profile's original `apiKeyEnv`, because replacing that key with the
+ * model-specific key pool is the entire point of a model sub-pool.
+ *
+ * The one thing a request-scoped pool must not do is hijack an unrelated
+ * credential resolved during the same request, so substitution is limited to
+ * refs that belong to this pool or to its provider-base pool:
+ *
+ *   ref in requested.refs            -> requested  (already a model-pool key)
+ *   ref in provider base pool refs   -> requested  (substitute the model pool)
+ *   provider has no base pool        -> requested  (model pool is the only source)
+ *   anything else                    -> null       (plain original resolve)
+ *
+ * With no request-scoped pool (ordinary `credentials.resolve()` outside
+ * rotation) the global lookup runs, preferring the provider-base pool.
+ *
+ * @param {object} index result of createPoolIndex()
+ * @param {string} ref credential ref being resolved
+ * @param {object|null|undefined} requested request-scoped pool
+ * @returns {object|null} pool to rotate within, or null to use the original resolve
+ */
+export function resolveRefPool(index, ref, requested) {
+  if (typeof ref !== 'string' || ref.length === 0) return null;
+  if (requested) {
+    if (Array.isArray(requested.refs) && requested.refs.includes(ref)) return requested;
+    if (!requested.model) return null;
+    const base = index.basePoolByProvider.get(requested.provider);
+    if (!base) return requested;
+    return base.refs.includes(ref) ? requested : null;
+  }
+  const base = index.basePoolByRef.get(ref);
+  if (base) return base;
+  const set = index.modelPoolByRef.get(ref);
+  return set && set.size > 0 ? set.values().next().value : null;
+}
+
+/**
+ * True when the resolver may still rotate for this ref even though every
+ * candidate is locally out of budget. Quota enforcement is fail-closed only for
+ * requests this plugin actually manages; an ordinary `credentials.resolve()`
+ * outside rotation keeps its historical behaviour.
+ */
+export function isQuotaManagedRequest(index, ref, requested) {
+  if (!requested) return false;
+  return resolveRefPool(index, ref, requested) === requested;
+}
+
+/**
+ * Model pools a ref appears in. Used by ops routes to describe one credential
+ * across the models it serves without guessing from the pool base string.
+ */
+export function modelPoolsForRef(index, ref) {
+  const set = index.modelPoolByRef.get(ref);
+  return set ? [...set] : [];
+}

package/lib/pool-state.js

@@ -3,6 +3,7 @@
 const MAP_FIELDS = [
   'failedUntil', 'failCounts', 'authFailCounts', 'brokenUntil', 'costPerKey',
   'lastUsedAt', 'usageCounts', 'byModel', 'usageDays', 'quotaWindows', 'costDays',
+  'tokenUsage',
 ];
 
 export function initializePoolState(state = {}) {

package/lib/pool.js

@@ -1,3 +1,4 @@
+import { resetModelQuotaIfNeeded } from './model-quota.js';
 import { decryptSecret } from './crypto-storage.js';
 import { bestEffort } from './best-effort.js';
 // lib/pool.js — pure pool arithmetic and selection logic for dsh-key-rotation.
@@ -25,15 +26,26 @@ export function isLoopbackAddress(ip) {
   return false;
 }
 
-/** Check if an incoming HTTP request is from a trusted local bridge origin (#353 fail-closed). */
+/** Strip an optional :port and IPv6 brackets from a Host header value. */
+function hostNameOf(hostHeader) {
+  return String(hostHeader).replace(/:\d+$/, '').replace(/^\[|\]$/g, '').toLowerCase();
+}
+
+/** Check if an incoming HTTP request is from a trusted local bridge origin (#353 fail-closed, #378 method-aware). */
 export function isTrustedBridgeRequest(req) {
   const remoteAddress = req?.socket?.remoteAddress;
   if (!isLoopbackAddress(remoteAddress)) return false;
-  const origin = req?.headers?.origin;
-  if (!origin) return false;
   if (req?.headers?.['sec-fetch-site'] === 'cross-site') return false;
   const hostHeader = req?.headers?.host;
   if (!hostHeader) return false;
+  const hostName = hostNameOf(hostHeader);
+  if (!(isLoopbackAddress(hostName) || hostName === 'localhost')) return false;
+
+  const origin = req?.headers?.origin;
+  if (!origin) {
+    const method = String(req?.method ?? 'GET').toUpperCase();
+    return method === 'GET' || method === 'HEAD' || method === 'OPTIONS';
+  }
   try {
     const originUrl = new URL(origin);
     if (originUrl.host !== hostHeader) return false;
@@ -189,7 +201,7 @@ export function envValue(ref) {
 }
 
 /** Sweep expired cooldown entries and prune stale deleted refs from poolState. Returns count of cleared refs. */
-export function sweepExpired(poolState, now = Date.now(), activeRefs = null) {
+export function sweepExpired(poolState, now = Date.now(), activeRefs = null, quotaResetWindow = null) {
   let cleared = 0;
   const activeSet = activeRefs ? new Set(activeRefs) : null;
   for (const st of poolState.values()) {
@@ -203,9 +215,17 @@ export function sweepExpired(poolState, now = Date.now(), activeRefs = null) {
         cleared++;
       }
     }
+    // Local model token budgets roll over on their own wall-clock window, never
+    // on a cooldown. Recomputing the boundary here keeps a long-lived idle pool
+    // from carrying an expired resetAt until its next request.
+    if (st.tokenUsage instanceof Map && quotaResetWindow) {
+      for (const ref of [...st.tokenUsage.keys()]) {
+        resetModelQuotaIfNeeded({ state: st, quotaResetWindow }, ref, quotaResetWindow, now);
+      }
+    }
     // Prune stale entries for keys no longer in active configuration
     if (activeSet) {
-      for (const map of [st.failedUntil, st.failCounts, st.authFailCounts, st.brokenUntil, st.costPerKey, st.lastUsedAt, st.usageCounts, st.byModel, st.usageDays, st.quotaWindows, st.rpmWindows, st.probedAt]) {
+      for (const map of [st.failedUntil, st.failCounts, st.authFailCounts, st.brokenUntil, st.costPerKey, st.lastUsedAt, st.usageCounts, st.byModel, st.usageDays, st.quotaWindows, st.rpmWindows, st.probedAt, st.tokenUsage]) {
         if (map && typeof map.keys === 'function') {
           for (const k of [...map.keys()]) {
             if (!activeSet.has(k)) map.delete(k);

package/lib/resolver.js

@@ -0,0 +1,159 @@
+// lib/resolver.js — the credential selector behind credentials.resolve (#381).
+//
+// Extracted from lib/index.js so the routing decisions that this feature
+// depends on — request-scoped pool ownership, model token quota enforcement,
+// fail-closed exhaustion, paused/revoked filtering, at-rest decryption —
+// are unit-testable without the cordis runtime.
+//
+// The resolver decides *which* credential ref to hand to the underlying
+// credentials service. It never sees or returns a secret value itself; it
+// delegates to `original(ref)`.
+
+import { bucketAllow, bucketRetryMs } from './bucket.js';
+import { envValue, sortAttemptList, isKeyPaused, isKeyRevoked } from './pool.js';
+import { decryptSecret } from './crypto-storage.js';
+import { resolveRefPool, isQuotaManagedRequest } from './pool-index.js';
+import {
+  isModelQuotaAvailable,
+  hasModelQuotaConfig,
+  anyModelQuotaAvailable,
+  formatModelQuotaExhaustion,
+} from './model-quota.js';
+
+/** Internal failure code for "local budgets forbid this dispatch". */
+export const LOCAL_QUOTA_CODE = 'LOCAL_MODEL_QUOTA_EXHAUSTED';
+
+/**
+ * @param {object} deps
+ * @param {() => object} deps.buildRuntime         current runtime snapshot
+ * @param {() => object|undefined} deps.currentPool request-scoped pool (ALS)
+ * @param {(pool: object, candidate: string) => void} [deps.onPicked] picked-ref hook
+ * @param {object} [deps.latencyHistogram]
+ * @param {object} [deps.concurrencyTracker]
+ * @param {() => number} [deps.now] wall clock; injectable for tests
+ * @returns {(ref: string, original: Function) => Promise<any>}
+ */
+export function createResolver({
+  buildRuntime,
+  currentPool,
+  onPicked,
+  latencyHistogram,
+  concurrencyTracker,
+  now = Date.now,
+}) {
+  return async function resolve(ref, original) {
+    const runtime = buildRuntime();
+    // A ref may belong to several model pools at once, so the request-scoped
+    // pool that rotate() actually dispatched with decides ownership. A bare
+    // resolve outside rotation falls back to the global lookup.
+    const requested = currentPool ? currentPool() ?? null : null;
+    const pool = resolveRefPool(runtime.index, ref, requested);
+    if (!pool) {
+      const res = await original(ref);
+      if (res && typeof res.value === 'string') {
+        return { ...res, value: decryptSecret(res.value) };
+      }
+      const ev = envValue(ref);
+      if (ev !== undefined) return { value: decryptSecret(ev), source: 'env' };
+      return res;
+    }
+
+    const at = now();
+    const strategy = pool.routingStrategy ?? runtime.routingStrategy ?? 'round-robin';
+    let list = pool.weightedRefs ?? pool.refs;
+    if (strategy === 'lowest-latency' || strategy === 'least-loaded') {
+      list = sortAttemptList(list, strategy, { latencyHistogram, concurrencyTracker, weights: pool.weightsMap });
+    }
+    // Fail closed: when this request is managed by a model pool that has local
+    // token budgets, an exhausted candidate may not be dispatched and the
+    // original credential must never be used as a bypass. Decided once, before
+    // the loop, so it cannot be re-evaluated mid-iteration.
+    const quotaManaged = hasModelQuotaConfig(pool) && isQuotaManagedRequest(runtime.index, ref, requested);
+    const start = strategy === 'round-robin' ? (pool.state.pointer ?? 0) : 0;
+
+    for (let i = 0; i < list.length; i++) {
+      const index = (start + i) % list.length;
+      const candidate = list[index];
+      if (isKeyRevoked(pool, candidate)) continue;
+      if (isKeyPaused(pool, candidate)) continue;
+      const until = pool.state.failedUntil.get(candidate);
+      if (until !== undefined && until > at) continue;
+      if (pool.expiresAt?.[candidate] !== undefined && at >= pool.expiresAt[candidate]) continue;
+      // Local model token budget: a spent credential is simply not a candidate.
+      // This is deliberately NOT recorded as a failure — the key is healthy and
+      // merely out of budget — so it must never touch failedUntil / failCounts /
+      // authFailCounts / brokenUntil.
+      if (quotaManaged && !isModelQuotaAvailable(pool, candidate, at)) continue;
+      const rpmLimit = pool.rpmLimit ?? 0;
+      if (rpmLimit > 0) {
+        if (!pool.state.rpmWindows) pool.state.rpmWindows = new Map();
+        if (!bucketAllow(pool.state.rpmWindows, candidate, rpmLimit, at)) {
+          const waitMs = bucketRetryMs(pool.state.rpmWindows, candidate, rpmLimit, at);
+          if ((pool.state.failedUntil.get(candidate) ?? 0) < at + waitMs) {
+            pool.state.failedUntil.set(candidate, at + waitMs);
+          }
+          continue;
+        }
+      }
+      if (pool.perHour) {
+        if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+        let win = pool.state.quotaWindows.get(candidate);
+        if (!win || at - win.start >= 3600000) win = { count: 0, start: at };
+        if (win.count >= pool.perHour) {
+          const resetAt = win.start + 3600000;
+          if ((pool.state.failedUntil.get(candidate) ?? 0) < resetAt) pool.state.failedUntil.set(candidate, resetAt);
+          continue;
+        }
+      }
+
+      // Advance the shared cursor at selection time: concurrent requests must
+      // not all pick the same slot.
+      pool.state.pointer = (index + 1) % list.length;
+      const hit = await original(candidate);
+      if (hit && typeof hit.value === 'string' && hit.value.length > 0) {
+        settle(pool, candidate, at, onPicked);
+        return { ...hit, value: decryptSecret(hit.value) };
+      }
+      // No stored credential: fall back to the launching environment, which is
+      // how env-bootstrapped pools work (issue #7).
+      const envVal = envValue(candidate);
+      if (envVal !== undefined) {
+        settle(pool, candidate, at, onPicked);
+        return { value: decryptSecret(envVal), source: 'env' };
+      }
+    }
+
+    // Every candidate was skipped. If the reason was local budget exhaustion,
+    // returning original(ref) would hand out a credential the budget forbids, so
+    // refuse instead of bypassing the quota. rotate() recognises this error as a
+    // local refusal: it is not penalised and not classified as an upstream failure.
+    if (quotaManaged && !anyModelQuotaAvailable(pool, at)) {
+      throw Object.assign(
+        new Error(formatModelQuotaExhaustion(pool.provider ?? pool.base, pool)),
+        { code: LOCAL_QUOTA_CODE, localQuota: true },
+      );
+    }
+    return original(ref);
+  };
+}
+
+/** Record a successful credential selection on the pool and request scope. */
+function settle(pool, candidate, now, onPicked) {
+  pool.state.lastUsed = candidate;
+  if (!pool.state.lastUsedAt) pool.state.lastUsedAt = new Map();
+  pool.state.lastUsedAt.set(candidate, now);
+  onPicked?.(pool, candidate);
+  pool.state.failCounts?.delete(candidate);
+  pool.state.failedUntil.delete(candidate);
+  pool.state.authFailCounts?.delete(candidate);
+  pool.state.brokenUntil?.delete(candidate);
+  if (!pool.state.usageCounts) pool.state.usageCounts = new Map();
+  pool.state.usageCounts.set(candidate, (pool.state.usageCounts.get(candidate) ?? 0) + 1);
+  if (pool.perHour) {
+    if (!pool.state.quotaWindows) pool.state.quotaWindows = new Map();
+    let win = pool.state.quotaWindows.get(candidate);
+    if (!win || now - win.start >= 3600000) win = { count: 0, start: now };
+    win.count++;
+    pool.state.quotaWindows.set(candidate, win);
+  }
+}