@goodandready/dsh-key-rotation 0.8.76 → 0.8.78

package/lib/model-catalog.js

@@ -0,0 +1,296 @@
+// lib/model-catalog.js — Centralized model catalog with Cache Stampede protection (#550).
+// Deduplicates concurrent in-flight requests and provides TTL-based caching and model capability normalization.
+
+export const DEFAULT_CATALOG_TTL_MS = 60 * 60 * 1000; // 1 hour
+export const DEFAULT_CATALOG_TIMEOUT_MS = 15_000;      // 15 seconds
+
+export const DEFAULT_CONTEXT_WINDOW = 128_000;
+export const DEFAULT_MAX_TOKENS = 8_192;
+
+export const KNOWN_MODELS_FALLBACK = {
+  'deepseek-chat': {
+    id: 'deepseek-chat',
+    name: 'DeepSeek Chat',
+    contextWindow: 64_000,
+    maxTokens: 8_000,
+    supportsReasoning: false,
+    supportsTools: true,
+    supportsVision: false,
+  },
+  'deepseek-reasoner': {
+    id: 'deepseek-reasoner',
+    name: 'DeepSeek Reasoner',
+    contextWindow: 64_000,
+    maxTokens: 8_000,
+    supportsReasoning: true,
+    supportsTools: false,
+    supportsVision: false,
+  },
+  'gpt-4o': {
+    id: 'gpt-4o',
+    name: 'GPT-4o',
+    contextWindow: 128_000,
+    maxTokens: 16_384,
+    supportsReasoning: false,
+    supportsTools: true,
+    supportsVision: true,
+  },
+  'gpt-4o-mini': {
+    id: 'gpt-4o-mini',
+    name: 'GPT-4o mini',
+    contextWindow: 128_000,
+    maxTokens: 16_384,
+    supportsReasoning: false,
+    supportsTools: true,
+    supportsVision: true,
+  },
+  'claude-3-5-sonnet-20241022': {
+    id: 'claude-3-5-sonnet-20241022',
+    name: 'Claude 3.5 Sonnet',
+    contextWindow: 200_000,
+    maxTokens: 8_192,
+    supportsReasoning: false,
+    supportsTools: true,
+    supportsVision: true,
+  },
+};
+
+/**
+ * Normalizes raw model object from provider /models endpoint.
+ * @param {object|string} raw
+ * @returns {object}
+ */
+export function normalizeModelMeta(raw) {
+  if (typeof raw === 'string') {
+    const fallback = KNOWN_MODELS_FALLBACK[raw];
+    return fallback ? { ...fallback } : {
+      id: raw,
+      name: raw,
+      contextWindow: DEFAULT_CONTEXT_WINDOW,
+      maxTokens: DEFAULT_MAX_TOKENS,
+      supportsReasoning: raw.includes('reason') || raw.includes('r1') || raw.includes('thought'),
+      supportsTools: true,
+      supportsVision: raw.includes('vision') || raw.includes('4o') || raw.includes('vl'),
+    };
+  }
+
+  const id = String(raw?.id || raw?.slug || raw?.name || '');
+  const fallback = KNOWN_MODELS_FALLBACK[id];
+
+  const name = String(raw?.name || raw?.display_name || id);
+  const contextWindow =
+    Number(raw?.context_window || raw?.max_context_window || raw?.contextWindow) ||
+    fallback?.contextWindow ||
+    DEFAULT_CONTEXT_WINDOW;
+  const maxTokens =
+    Number(raw?.max_tokens || raw?.max_output_tokens || raw?.maxTokens) ||
+    fallback?.maxTokens ||
+    DEFAULT_MAX_TOKENS;
+
+  const caps = Array.isArray(raw?.capabilities) ? raw.capabilities : [];
+  const reasoningLevels = Array.isArray(raw?.supported_reasoning_levels) ? raw.supported_reasoning_levels : [];
+
+  const supportsReasoning =
+    Boolean(raw?.supportsReasoning) ||
+    caps.includes('thinking') ||
+    caps.includes('reasoning') ||
+    reasoningLevels.length > 0 ||
+    id.includes('reason') ||
+    id.includes('r1') ||
+    fallback?.supportsReasoning ||
+    false;
+
+  const supportsTools =
+    raw?.supportsTools !== undefined
+      ? Boolean(raw.supportsTools)
+      : caps.includes('tools') || fallback?.supportsTools || true;
+
+  const inputModalities = Array.isArray(raw?.input_modalities) ? raw.input_modalities : [];
+  const supportsVision =
+    Boolean(raw?.supportsVision) ||
+    inputModalities.includes('image') ||
+    id.includes('vision') ||
+    id.includes('4o') ||
+    id.includes('vl') ||
+    fallback?.supportsVision ||
+    false;
+
+  return {
+    id,
+    name,
+    contextWindow,
+    maxTokens,
+    supportsReasoning,
+    supportsTools,
+    supportsVision,
+    raw,
+  };
+}
+
+export class ModelCatalog {
+  /**
+   * @param {object} [options]
+   * @param {number} [options.ttlMs]
+   * @param {number} [options.timeoutMs]
+   * @param {typeof fetch} [options.fetchImpl]
+   */
+  constructor(options = {}) {
+    this.ttlMs = options.ttlMs ?? DEFAULT_CATALOG_TTL_MS;
+    this.timeoutMs = options.timeoutMs ?? DEFAULT_CATALOG_TIMEOUT_MS;
+    this.fetchImpl = options.fetchImpl ?? globalThis.fetch;
+    /** @type {Map<string, { at: number, models: Array<object> }>} */
+    this.cache = new Map();
+    /** @type {Map<string, Promise<Array<object>>>} In-flight deduplication map */
+    this.inflight = new Map();
+  }
+
+  _buildKey(endpoint, apiKey) {
+    const keyHint = apiKey ? apiKey.slice(-6) : 'anonymous';
+    return `${endpoint || 'default'}|${keyHint}`;
+  }
+
+  /**
+   * Gets list of models for an endpoint.
+   * If a request for this endpoint is already in flight, reuses the promise (anti-stampede).
+   *
+   * @param {object} options
+   * @param {string} options.endpoint Base URL or /models URL
+   * @param {string} [options.apiKey]
+   * @param {boolean} [options.forceRefresh]
+   * @param {typeof fetch} [options.fetchImpl]
+   * @returns {Promise<Array<object>>}
+   */
+  async getModels(options = {}) {
+    const { endpoint, apiKey, forceRefresh = false } = options;
+    const fetchFn = options.fetchImpl || this.fetchImpl;
+    const cacheKey = this._buildKey(endpoint, apiKey);
+    const now = Date.now();
+
+    // 1. Check TTL cache if not forcing refresh
+    if (!forceRefresh) {
+      const cached = this.cache.get(cacheKey);
+      if (cached && (now - cached.at < this.ttlMs)) {
+        return cached.models;
+      }
+    }
+
+    // 2. Check in-flight deduplication map (Cache Stampede protection)
+    const existingPromise = this.inflight.get(cacheKey);
+    if (existingPromise) {
+      return existingPromise;
+    }
+
+    // 3. Initiate single fetch and record in inflight map
+    const fetchPromise = (async () => {
+      try {
+        const models = await this._fetchModels(endpoint, apiKey, fetchFn);
+        this.cache.set(cacheKey, { at: Date.now(), models });
+        return models;
+      } catch (err) {
+        // On error, if we have stale cache, serve stale cache
+        const stale = this.cache.get(cacheKey);
+        if (stale && Array.isArray(stale.models) && stale.models.length > 0) {
+          return stale.models;
+        }
+        // Fallback to known models
+        return Object.values(KNOWN_MODELS_FALLBACK).map(normalizeModelMeta);
+      } finally {
+        this.inflight.delete(cacheKey);
+      }
+    })();
+
+    this.inflight.set(cacheKey, fetchPromise);
+    return fetchPromise;
+  }
+
+  /**
+   * Gets metadata for a specific model ID.
+   * @param {string} modelId
+   * @param {object} [options]
+   * @returns {Promise<object|null>}
+   */
+  async getModel(modelId, options = {}) {
+    if (!modelId) return null;
+    const models = await this.getModels(options);
+    const found = models.find((m) => m.id === modelId);
+    if (found) return found;
+    return normalizeModelMeta(modelId);
+  }
+
+  async _fetchModels(endpoint, apiKey, fetchFn) {
+    if (!endpoint || typeof fetchFn !== 'function') {
+      return Object.values(KNOWN_MODELS_FALLBACK).map(normalizeModelMeta);
+    }
+
+    let url = endpoint;
+    if (!url.endsWith('/models') && !url.includes('/models?')) {
+      url = url.replace(/\/+$/, '') + '/models';
+    }
+
+    const headers = {
+      accept: 'application/json',
+      'user-agent': 'dsh-key-rotation/model-catalog',
+    };
+    if (apiKey) {
+      headers.authorization = `Bearer ${apiKey}`;
+    }
+
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.timeoutMs);
+
+    try {
+      const resp = await fetchFn(url, {
+        headers,
+        signal: controller.signal,
+      });
+
+      if (!resp.ok) {
+        throw new Error(`Catalog fetch failed with status ${resp.status}`);
+      }
+
+      const body = await resp.json();
+      let rawList = [];
+
+      if (Array.isArray(body?.data)) {
+        rawList = body.data;
+      } else if (Array.isArray(body?.models)) {
+        rawList = body.models;
+      } else if (body?.models && typeof body.models === 'object') {
+        rawList = Object.entries(body.models).map(([k, v]) => ({ id: k, ...(typeof v === 'object' ? v : {}) }));
+      }
+
+      if (rawList.length === 0) {
+        return Object.values(KNOWN_MODELS_FALLBACK).map(normalizeModelMeta);
+      }
+
+      return rawList.map(normalizeModelMeta);
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  invalidate(endpoint, apiKey) {
+    const key = this._buildKey(endpoint, apiKey);
+    this.cache.delete(key);
+    this.inflight.delete(key);
+  }
+
+  clear() {
+    this.cache.clear();
+    this.inflight.clear();
+  }
+
+  getStatus(endpoint, apiKey) {
+    const key = this._buildKey(endpoint, apiKey);
+    const entry = this.cache.get(key);
+    const isInflight = this.inflight.has(key);
+    return {
+      cached: Boolean(entry),
+      count: entry?.models?.length ?? 0,
+      fetchedAt: entry?.at ?? null,
+      inflight: isInflight,
+    };
+  }
+}
+
+export const defaultModelCatalog = new ModelCatalog();

package/lib/pool-selection.js

@@ -1,30 +1,98 @@
-// lib/pool-selection.js — pool selection, sticky sessions, attempt sorting and budget metrics (#512).
+// lib/pool-selection.js — pool selection, sticky sessions, attempt sorting and budget metrics (#512, #548, #549).
 // Extracted from lib/pool.js to enforce < 600 line modularity standard.
 
-export const MAX_SESSION_STICKY_ENTRIES = 500;
+import {
+  SessionRouter,
+  defaultSessionRouter,
+  MAX_SESSIONS,
+  SESSION_TTL_MS,
+} from './session-router.js';
 
-export function setSessionSticky(pool, sessionId, candidate) {
-  if (!sessionId || !candidate || !pool?.state) return;
-  if (!(pool.state.sessionSticky instanceof Map)) {
-    pool.state.sessionSticky = new Map();
+import {
+  QuotaRunway,
+  defaultQuotaRunway,
+} from './quota-runway.js';
+
+export {
+  SessionRouter,
+  defaultSessionRouter,
+  MAX_SESSIONS,
+  SESSION_TTL_MS,
+  QuotaRunway,
+  defaultQuotaRunway,
+};
+
+export const MAX_SESSION_STICKY_ENTRIES = MAX_SESSIONS;
+
+/**
+ * Get or initialize the SessionRouter on pool.state or fallback to default.
+ * @param {object} [pool]
+ * @returns {SessionRouter}
+ */
+export function getOrCreateSessionRouter(pool) {
+  if (!pool?.state) return defaultSessionRouter;
+  if (!pool.state.sessionRouter) {
+    pool.state.sessionRouter = new SessionRouter({
+      maxSessions: MAX_SESSIONS,
+      sessionTtlMs: SESSION_TTL_MS,
+    });
   }
-  if (pool.state.sessionSticky.has(sessionId)) {
-    pool.state.sessionSticky.delete(sessionId);
-  } else if (pool.state.sessionSticky.size >= MAX_SESSION_STICKY_ENTRIES) {
-    const oldest = pool.state.sessionSticky.keys().next().value;
-    if (oldest !== undefined) pool.state.sessionSticky.delete(oldest);
+  return pool.state.sessionRouter;
+}
+
+export function setSessionSticky(pool, sessionId, candidate) {
+  if (!sessionId || !candidate) return;
+  const router = getOrCreateSessionRouter(pool);
+  router.setSession(sessionId, candidate);
+  if (pool?.state) {
+    if (!(pool.state.sessionSticky instanceof Map)) {
+      pool.state.sessionSticky = new Map();
+    }
+    if (pool.state.sessionSticky.has(sessionId)) {
+      pool.state.sessionSticky.delete(sessionId);
+    } else if (pool.state.sessionSticky.size >= MAX_SESSION_STICKY_ENTRIES) {
+      const oldest = pool.state.sessionSticky.keys().next().value;
+      if (oldest !== undefined) pool.state.sessionSticky.delete(oldest);
+    }
+    pool.state.sessionSticky.set(sessionId, candidate);
   }
-  pool.state.sessionSticky.set(sessionId, candidate);
 }
 
 export function getSessionSticky(pool, sessionId) {
-  if (!sessionId || !pool?.state?.sessionSticky) return null;
-  return pool.state.sessionSticky.get(sessionId) ?? null;
+  if (!sessionId) return null;
+  const router = getOrCreateSessionRouter(pool);
+  const fromRouter = router.getSession(sessionId);
+  if (fromRouter) return fromRouter;
+  return pool?.state?.sessionSticky?.get?.(sessionId) ?? null;
 }
 
 export function clearSessionSticky(pool, sessionId) {
-  if (!sessionId || !pool?.state?.sessionSticky) return;
-  pool.state.sessionSticky.delete(sessionId);
+  if (!sessionId) return;
+  const router = getOrCreateSessionRouter(pool);
+  router.unstick(sessionId);
+  if (pool?.state?.sessionSticky) {
+    pool.state.sessionSticky.delete(sessionId);
+  }
+}
+
+/**
+ * Proactively unstick all sessions pinned to a cooling or failed key (#548).
+ * @param {object} pool
+ * @param {string} keyRef
+ * @returns {number}
+ */
+export function unstickFailedKey(pool, keyRef) {
+  if (!keyRef) return 0;
+  const router = getOrCreateSessionRouter(pool);
+  const count = router.unstickKey(keyRef);
+  if (pool?.state?.sessionSticky) {
+    for (const [sId, ref] of pool.state.sessionSticky.entries()) {
+      if (ref === keyRef) {
+        pool.state.sessionSticky.delete(sId);
+      }
+    }
+  }
+  return count;
 }
 
 /** Pick a key pool for a (provider, model) pair. */
@@ -42,16 +110,31 @@ export function selectPool(modelPoolByProvider, providerToPool, provider, model)
 
 /**
  * Sort candidate refs according to pool routing strategy.
+ * Preemptively deprioritizes keys projected to exhaust their quota before reset (#549).
  * Returns a new array; does not mutate input.
  */
 export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
   if (!Array.isArray(refs) || refs.length <= 1) return (refs ?? []).slice();
-  const list = refs.slice();
+  let list = refs.slice();
+
   if (strategy === 'sticky') {
-    const last = deps.sessionSticky ||
+    const router = getOrCreateSessionRouter(deps.pool);
+    let chosenKey = null;
+
+    if (deps.sessionId) {
+      const resolved = router.resolveSessionKey(deps.sessionId, list);
+      if (resolved && resolved.keyRef) {
+        chosenKey = resolved.keyRef;
+        if (deps.pool) setSessionSticky(deps.pool, deps.sessionId, chosenKey);
+      }
+    }
+
+    const last = chosenKey ||
+      deps.sessionSticky ||
       (deps.sessionId && (deps.pool?.state?.sessionSticky?.get?.(deps.sessionId) || deps.sessionStickyMap?.get?.(deps.sessionId))) ||
       deps.lastUsed ||
       (deps.pool && deps.pool.state && deps.pool.state.lastUsed);
+
     if (last && list.includes(last)) {
       const idx = list.indexOf(last);
       if (idx > 0) {
@@ -59,19 +142,16 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
         list.unshift(last);
       }
     }
-    return list;
-  }
-  if (strategy === 'tier-priority') {
+  } else if (strategy === 'tier-priority') {
     const tiers = deps.tiers || (deps.pool && deps.pool.tiers) || {};
-    return list.sort((a, b) => {
+    list = list.sort((a, b) => {
       const ta = (tiers[a] && Number(tiers[a]) >= 1) ? Number(tiers[a]) : 1;
       const tb = (tiers[b] && Number(tiers[b]) >= 1) ? Number(tiers[b]) : 1;
       return ta - tb;
     });
-  }
-  if (strategy === 'least-loaded' && deps.concurrencyTracker && typeof deps.concurrencyTracker.getActive === 'function') {
+  } else if (strategy === 'least-loaded' && deps.concurrencyTracker && typeof deps.concurrencyTracker.getActive === 'function') {
     const weights = deps.weights || (deps.pool && deps.pool.weightsMap) || {};
-    return list.sort((a, b) => {
+    list = list.sort((a, b) => {
       const ca = deps.concurrencyTracker.getActive(a) ?? 0;
       const cb = deps.concurrencyTracker.getActive(b) ?? 0;
       const wa = (weights[a] && Number(weights[a]) > 0) ? Number(weights[a]) : 1;
@@ -81,9 +161,8 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
       if (scoreA !== scoreB) return scoreA - scoreB;
       return ca - cb;
     });
-  }
-  if (strategy === 'lowest-latency' && deps.latencyHistogram && typeof deps.latencyHistogram.snapshot === 'function') {
-    return list.sort((a, b) => {
+  } else if (strategy === 'lowest-latency' && deps.latencyHistogram && typeof deps.latencyHistogram.snapshot === 'function') {
+    list = list.sort((a, b) => {
       const sa = deps.latencyHistogram.snapshot(a);
       const sb = deps.latencyHistogram.snapshot(b);
       const la = (sa && Number.isFinite(sa.p95)) ? sa.p95 : ((sa && Number.isFinite(sa.avg)) ? sa.avg : 500);
@@ -91,6 +170,25 @@ export function sortAttemptList(refs, strategy = 'round-robin', deps = {}) {
       return la - lb;
     });
   }
+
+  // Preemptively deprioritize keys projected to exhaust before quota window reset (#549)
+  const runway = deps.quotaRunway ?? deps.pool?.state?.quotaRunway;
+  if (runway && typeof runway.isExhaustingBeforeReset === 'function' && list.length > 1) {
+    const now = typeof deps.now === 'function' ? deps.now() : (Number(deps.now) || Date.now());
+    const healthy = [];
+    const exhausting = [];
+    for (const ref of list) {
+      if (runway.isExhaustingBeforeReset(ref, now)) {
+        exhausting.push(ref);
+      } else {
+        healthy.push(ref);
+      }
+    }
+    if (healthy.length > 0 && exhausting.length > 0) {
+      return healthy.concat(exhausting);
+    }
+  }
+
   return list;
 }
 

package/lib/pool.js

@@ -1,16 +1,20 @@
 // lib/pool.js — pure pool arithmetic and selection logic for dsh-key-rotation.
 import {
   MAX_SESSION_STICKY_ENTRIES, setSessionSticky, getSessionSticky, clearSessionSticky,
+  unstickFailedKey, SessionRouter, defaultSessionRouter, MAX_SESSIONS, SESSION_TTL_MS,
   selectPool, sortAttemptList, expiringSoon, shouldNotifyDaily,
   costForDay, costForWeek, budgetVerdict,
 } from './pool-selection.js';
 
 export {
   MAX_SESSION_STICKY_ENTRIES, setSessionSticky, getSessionSticky, clearSessionSticky,
+  unstickFailedKey, SessionRouter, defaultSessionRouter, MAX_SESSIONS, SESSION_TTL_MS,
   selectPool, sortAttemptList, expiringSoon, shouldNotifyDaily,
   costForDay, costForWeek, budgetVerdict,
 };
 
+import { classifyResponseError, CONTEXT_WINDOW_EXCEEDED_CODE } from './errors.js';
+export { classifyResponseError };
 import { resetModelQuotaIfNeeded } from './model-quota.js';
 import { decryptSecret } from './crypto-storage.js';
 import { bestEffort } from './best-effort.js';
@@ -190,6 +194,7 @@ export function pickNext(pool, now, refsCount = pool.refs.length) {
 
 /** Apply a failed-key cooldown to pool state. Pure: returns next state shape. */
 export function applyCooldown(pool, ref, cooldownMs, now = Date.now()) {
+  unstickFailedKey(pool, ref);
   return {
     ...pool,
     state: {
@@ -218,6 +223,7 @@ export function computeBackoff(baseMs, failCount, maxMs, isSoft = false) {
 
 /** Record a failure for `ref` in `pool.state`. */
 export function recordFailure(pool, ref, now, baseMs, maxMs, isSoft = false, jitter = false) {
+  unstickFailedKey(pool, ref);
   if (!pool.state.failCounts) pool.state.failCounts = new Map();
   let next = pool.state.failCounts.get(ref) ?? 0;
   if (!isSoft) {
@@ -494,6 +500,18 @@ export function isSwitchableError(failureOrPayload, switchCodes = new Set(DEFAUL
     status = Number(rawCode);
   }
 
+  // 0. Structured response error classification (#547)
+  if (status || failure.body || message.startsWith('{') || (code && code !== 'ERROR')) {
+    const typed = classifyResponseError(status, failure.body ?? message, failure.headers ?? failure.response?.headers);
+    if (typed.code === CONTEXT_WINDOW_EXCEEDED_CODE) return false;
+    if ((typed.code === 'QUOTA' || typed.code === 'ACCOUNT_QUOTA') && (switchCodes.has('QUOTA') || switchCodes.has('RATE_LIMIT') || switchCodes.has('429'))) return true;
+    if (typed.code === 'RATE_LIMIT' && (switchCodes.has('RATE_LIMIT') || switchCodes.has('429'))) return true;
+    if ((typed.code === 'AUTH' || typed.code === 'INVALID_CREDENTIAL') && (switchCodes.has('AUTH') || switchCodes.has('401') || switchCodes.has('403'))) return true;
+    if (typed.code === 'SERVER' && (switchCodes.has('SERVER') || (status >= 500 && status <= 504 && switchCodes.has(String(status))))) return true;
+    if (typed.code === 'TIMEOUT' && (switchCodes.has('TIMEOUT') || switchCodes.has('408') || switchCodes.has('504'))) return true;
+    if (switchCodes.has(typed.code)) return true;
+  }
+
   // 1. Direct HTTP status codes (#267: also 408/425 transient, #545: numeric status/code mapped to semantic switch codes)
   if ((status === 429 || status === 408 || status === 425) && (switchCodes.has('RATE_LIMIT') || switchCodes.has('QUOTA') || switchCodes.has('429') || switchCodes.has('TIMEOUT') || switchCodes.has(String(status)))) return true;
   if ((status === 401 || status === 403) && (switchCodes.has('AUTH') || switchCodes.has('401') || switchCodes.has('403') || switchCodes.has(String(status)))) return true;
@@ -576,3 +594,4 @@ export function isKeyBroken(pool, ref, now = Date.now()) {
   if (pool.basePool) return isKeyBroken(pool.basePool, ref, now);
   return false;
 }
+