adaptive-memory-multi-model-router 2.15.2 → 2.15.4

package/src/routing/advancedRouter.ts

@@ -16,6 +16,86 @@ import { estimateCost } from "../utils/tokenUtils";
 import { logScaleCostScore } from "../utils/costUtils";
 import { quickselectTopK, selectTop } from "../utils/sorting";
 
+// ============================================================
+// TRAFFIC SHARES FOR ADAPTIVE DIVERSITY WEIGHT (EXP3-inspired)
+// Based on negative frequency-dependent selection — as provider i's traffic
+// share f_i grows above uniform (1/n), impose a diversity penalty.
+// Formula: gamma = sqrt(n * log(n) / (T * G^2))  [Auer et al. 2002, EXP3]
+// diversityPenalty_i = gamma * (f_i - 1/n)
+// ============================================================
+
+/** Number of times each provider has been selected (cumulative) */
+const _selectionCount: Record<string, number> = {};
+
+/** Total routing decisions since last reset */
+let _totalDecisions = 0;
+
+/** Reward range estimate for gamma computation (quality_score scale 0-1) */
+const _REWARD_RANGE = 1.0;
+
+/**
+ * Compute the EXP3-inspired diversity penalty for a provider.
+ * Based on: Auer et al. 2002 — "The Nonstochastic Multiarmed Bandit Problem"
+ * and the insight that as provider share f_i grows above uniform (1/n),
+ * imposing a penalty proportional to (f_i - 1/n) prevents monoculture
+ * (competitive exclusion — negative frequency-dependent selection).
+ *
+ * @param providerName - provider key
+ * @param nProviders - total number of available providers
+ * @param recentDecayFactor - optional decay for non-stationary environments
+ */
+function computeDiversityPenalty(
+  providerName: string,
+  nProviders: number,
+  recentDecayFactor = 0.0
+): number {
+  if (nProviders < 2) return 0;
+  if (_totalDecisions < 2) return 0;
+
+  const share = _selectionCount[providerName] || 0;
+  const uniform = 1.0 / nProviders;
+
+  // Deviation from uniform distribution (can be negative if under-used)
+  const deviation = share - uniform;
+  if (Math.abs(deviation) < 1e-6) return 0;
+
+  // EXP3 learning rate: gamma = sqrt(n * log(n) / (T * G^2))
+  // G = reward range (quality_score in [0,1] → G = 1)
+  const T = Math.max(_totalDecisions, 10);
+  const gamma = Math.sqrt((nProviders * Math.log(nProviders)) / (T * _REWARD_RANGE * _REWARD_RANGE));
+
+  // Clamp gamma to prevent extreme penalties when T is very small
+  const clampedGamma = Math.min(gamma, 0.5);
+
+  // Optional: decay factor for non-stationary environments
+  // (higher share = stronger penalty, but decays over time)
+  // Currently disabled (recentDecayFactor=0) — re-enable if providers change frequently
+  const effectiveGamma = clampedGamma * (1.0 - recentDecayFactor);
+
+  // Penalty is proportional to how far above uniform the provider's share is
+  // (negative deviation = under-used = reward, not penalty)
+  if (deviation <= 0) return 0; // Reward already captured implicitly by below-uniform penalty
+
+  return effectiveGamma * deviation;
+}
+
+/**
+ * Record that a provider was selected (call after each routing decision).
+ * Used to compute the diversity penalty on subsequent decisions.
+ */
+function recordSelection(providerName: string): void {
+  _selectionCount[providerName] = (_selectionCount[providerName] || 0) + 1;
+  _totalDecisions += 1;
+}
+
+/**
+ * Reset traffic tracking (call when provider pool changes or for eval resets).
+ */
+function resetDiversityState(): void {
+  Object.keys(_selectionCount).forEach(k => delete _selectionCount[k]);
+  _totalDecisions = 0;
+}
+
 // ============================================================
 // CACHE FOR MODEL PROFILES (avoids O(n*m) rebuild on every routeQuery)
 // ============================================================
@@ -767,6 +847,24 @@ export function routeQuery(prompt: string, available_models?: string[], budget_m
 
   let topCandidates = quickselectTopK(candidates, 4, scoreFn);
 
+  // === DIVERSITY PENALTY (EXP3-inspired, negative frequency-dependence) ===
+  // As provider share f_i grows above uniform (1/n), impose a diversity penalty.
+  // This prevents provider monoculture (competitive exclusion) — the core insight
+  // from the Paradox of the Plankton / negative frequency-dependent selection.
+  // Penalty is applied only to the quality dimension (complexity_bias-weighted),
+  // since cost already naturally distributes across providers.
+  // See: Auer et al. 2002, "The Nonstochastic Multiarmed Bandit Problem"
+  const nProviders = candidates.length;
+  for (const c of candidates) {
+    const divPenalty = computeDiversityPenalty(c.name, nProviders);
+    // Diversity penalty applies to quality dimension only (cost already disperses traffic)
+    c.total_score -= divPenalty * complexity_bias;
+  }
+
+  // Re-rank after diversity adjustment
+  candidates.sort((a, b) => b.total_score - a.total_score);
+  topCandidates = candidates.slice(0, 4);
+
   // Adaptive quality floor: for complex queries, prefer models above the floor
   if (adaptiveQualityFloor > 0) {
     const qualified = topCandidates.filter(c => c.quality_score >= adaptiveQualityFloor);
@@ -779,6 +877,9 @@ export function routeQuery(prompt: string, available_models?: string[], budget_m
   const primary = topCandidates[0];
   const secondary = topCandidates.slice(1, 3);
 
+  // Record selection for diversity tracking (after final decision, before return)
+  recordSelection(primary.name);
+
   // Calculate confidence based on score gap
   let confidence = 0.5;
   if (candidates.length > 1) {

package/src/routing/providerHealth.ts

@@ -49,6 +49,17 @@ export interface ProviderHealth {
   cooldownUntil: number;
   /** Health score 0-1 (higher is better) */
   healthScore: number;
+  /** === MVT RATE-LIMIT TRACKING (Charnov 1976 optimal foraging) === */
+  /** Tokens used in current rate-limit window */
+  tokensUsedThisWindow: number;
+  /** Timestamp when current rate-limit window started */
+  rateLimitWindowStart: number;
+  /** Maximum tokens per rate-limit window */
+  rateLimitTokens: number;
+  /** Rate-limit window duration in ms (default: 60000 = 1 min) */
+  rateLimitWindowMs: number;
+  /** Rolling average tokens per successful request */
+  avgTokensPerRequest: number;
 }
 
 export interface ProviderMetrics {
@@ -64,6 +75,8 @@ export interface ProviderMetrics {
   totalLatency: number;
   /** Last measured latency */
   lastLatency: number;
+  /** Tokens consumed this request (for rate-limit tracking) */
+  tokensUsed: number;
 }
 
 export interface HealthManagerConfig {
@@ -132,8 +145,11 @@ export class ProviderHealthManager extends EventEmitter {
 
   /**
    * Record a successful request
+   * @param provider - provider name
+   * @param latencyMs - response latency in ms
+   * @param tokensUsed - tokens consumed this request (for MVT rate-limit tracking)
    */
-  recordSuccess(provider: string, latencyMs: number): void {
+  recordSuccess(provider: string, latencyMs: number, tokensUsed: number = 0): void {
     this.ensureProviderExists(provider);
     
     const now = Date.now();
@@ -145,6 +161,7 @@ export class ProviderHealthManager extends EventEmitter {
       failedRequests: 0,
       totalLatency: latencyMs,
       lastLatency: latencyMs,
+      tokensUsed,
     });
     
     // Trim to window size
@@ -152,8 +169,29 @@ export class ProviderHealthManager extends EventEmitter {
       window.shift();
     }
     
-    // Update health state
+    // === MVT RATE-LIMIT WINDOW MANAGEMENT ===
     const health = this.health.get(provider)!;
+    const windowElapsed = now - health.rateLimitWindowStart;
+    
+    // If window has elapsed (rolled over), reset the token counter
+    if (windowElapsed >= health.rateLimitWindowMs) {
+      health.tokensUsedThisWindow = 0;
+      health.rateLimitWindowStart = now;
+    }
+    
+    // Accumulate tokens used
+    if (tokensUsed > 0) {
+      health.tokensUsedThisWindow += tokensUsed;
+    }
+    
+    // Update rolling average tokens per request
+    const successfulReqs = window.filter(m => m.successfulRequests > 0);
+    if (successfulReqs.length > 0) {
+      const totalTokens = successfulReqs.reduce((s, m) => s + (m.tokensUsed || 0), 0);
+      health.avgTokensPerRequest = totalTokens / successfulReqs.length;
+    }
+    
+    // Update health state
     health.lastSuccess = now;
     health.consecutiveErrors = 0;
     health.cooldownUntil = 0;
@@ -180,6 +218,7 @@ export class ProviderHealthManager extends EventEmitter {
       failedRequests: 1,
       totalLatency: 0,
       lastLatency: 0,
+      tokensUsed: 0,
     });
     
     // Trim to window size
@@ -310,6 +349,129 @@ export class ProviderHealthManager extends EventEmitter {
     return scored.map(s => s.provider);
   }
 
+  // ================================================================
+  // MVT RATE-LIMIT ROTATION (Charnov 1976 Optimal Foraging)
+  // ================================================================
+
+  /**
+   * Configure rate-limit parameters for a provider.
+   * Call this once during provider registration with the provider's actual limits.
+   * 
+   * @param provider - provider name
+   * @param rateLimitTokens - max tokens per window (e.g., 1000000 for 1M)
+   * @param rateLimitWindowMs - window duration in ms (e.g., 60000 for 1 min)
+   */
+  setRateLimitConfig(provider: string, rateLimitTokens: number, rateLimitWindowMs: number): void {
+    this.ensureProviderExists(provider);
+    const health = this.health.get(provider)!;
+    health.rateLimitTokens = rateLimitTokens;
+    health.rateLimitWindowMs = rateLimitWindowMs;
+    // Reset window on config change
+    health.tokensUsedThisWindow = 0;
+    health.rateLimitWindowStart = Date.now();
+  }
+
+  /**
+   * Estimate cold-start latency for switching to a fallback provider.
+   * Based on the provider's average latency as a proxy.
+   * In production, this would include TLS handshake, DNS, and model warmup costs.
+   */
+  private estimateColdStartLatency(fallbackProvider: string): number {
+    const fallbackHealth = this.health.get(fallbackProvider);
+    if (!fallbackHealth) return 1000; // conservative default
+    
+    const baseLatency = fallbackHealth.latency || 500;
+    // Cold start typically 1.5-3x warm latency depending on provider
+    // Add TLS + DNS overhead (typically 50-200ms)
+    const coldStartMultiplier = 2.0;
+    const tlsOverhead = 100;
+    return baseLatency * coldStartMultiplier + tlsOverhead;
+  }
+
+  /**
+   * Should we rotate away from this provider due to rate-limit depletion?
+   * 
+   * Implements Charnov's Marginal Value Theorem (1976):
+   *   g'(t*) = g(t*) / (t* + τ)
+   * 
+   * where:
+   *   g(t*) = cumulative successful tokens used so far in this window
+   *   g'(t*) = marginal rate = remaining tokens / time remaining in window
+   *   τ = cold-start latency for the fallback provider
+   * 
+   * LEAVE when marginal rate ≤ average rate (including switch cost).
+   * STAY when marginal rate > average rate (still worth staying).
+   * 
+   * @param provider - current provider to evaluate
+   * @param fallbackProvider - candidate fallback provider
+   * @returns true if rotation is recommended (marginal rate ≤ break-even rate)
+   */
+  shouldRotateForRateLimit(provider: string, fallbackProvider: string): boolean {
+    const health = this.health.get(provider);
+    if (!health) return false;
+    
+    const now = Date.now();
+    const windowElapsed = now - health.rateLimitWindowStart;
+    
+    // If window hasn't started or is fresh, don't rotate
+    if (health.rateLimitWindowStart === 0 || windowElapsed < 100) return false;
+    
+    // If already depleted (tokens used ≥ limit), recommend rotation
+    if (health.tokensUsedThisWindow >= health.rateLimitTokens) return true;
+    
+    // Remaining token budget in current window
+    const remainingBudget = Math.max(0, health.rateLimitTokens - health.tokensUsedThisWindow);
+    const remainingTimeMs = Math.max(1, health.rateLimitWindowMs - windowElapsed);
+    
+    // Marginal rate: tokens per ms we can still consume this window
+    // High marginal rate = plenty of budget left = stay
+    // Low marginal rate = running out = consider leaving
+    const marginalRate = remainingBudget / remainingTimeMs;
+    
+    // Cumulative successful tokens so far
+    const g_t = health.tokensUsedThisWindow;
+    
+    // Cold-start cost for switching to fallback
+    const tau = this.estimateColdStartLatency(fallbackProvider);
+    
+    // Break-even rate: the rate at which staying = switching
+    // From MVT: g'(t*) = g(t*) / (t* + τ)
+    // In our terms: marginal_rate = cumulative_rate * (t* / (t* + τ))
+    // But here we use: avg_rate_including_switch = g_t / (windowElapsed + τ)
+    // This is the rate INCLUDING the cost of switching (we lose τ ms of this window)
+    const avgRateIncludingSwitch = g_t / (windowElapsed + tau);
+    
+    // MVT says: LEAVE when marginal_rate ≤ avg_rate_including_switch
+    // (the marginal gain from staying ≤ the average gain achievable including switch cost)
+    // STAY when marginal_rate > avg_rate_including_switch
+    // (we can still get more from this window than the switch costs us)
+    const ROTATION_THRESHOLD_FACTOR = 1.0; // 1.0 = exact MVT; >1 = leave earlier, <1 = stay longer
+    
+    if (marginalRate <= avgRateIncludingSwitch * ROTATION_THRESHOLD_FACTOR) {
+      return true; // MVT says: leave this patch
+    }
+    
+    return false; // MVT says: stay in this patch
+  }
+
+  /**
+   * Get the marginal rate for a provider (tokens/ms remaining in window).
+   * Useful for monitoring and debugging MVT decisions.
+   */
+  getMarginalRate(provider: string): { marginalRate: number; remainingBudget: number; remainingTimeMs: number; utilizationPct: number } | null {
+    const health = this.health.get(provider);
+    if (!health || health.rateLimitWindowStart === 0) return null;
+    
+    const now = Date.now();
+    const windowElapsed = now - health.rateLimitWindowStart;
+    const remainingBudget = Math.max(0, health.rateLimitTokens - health.tokensUsedThisWindow);
+    const remainingTimeMs = Math.max(1, health.rateLimitWindowMs - windowElapsed);
+    const marginalRate = remainingBudget / remainingTimeMs;
+    const utilizationPct = (health.tokensUsedThisWindow / health.rateLimitTokens) * 100;
+    
+    return { marginalRate, remainingBudget, remainingTimeMs, utilizationPct };
+  }
+
   /**
    * Mark provider as disabled (manual circuit breaker)
    */
@@ -410,6 +572,13 @@ export class ProviderHealthManager extends EventEmitter {
         isHealthy: true,
         cooldownUntil: 0,
         healthScore: 1.0,
+        // === MVT RATE-LIMIT DEFAULTS ===
+        // Conservative defaults: 1M tokens/min (most free tier providers)
+        tokensUsedThisWindow: 0,
+        rateLimitWindowStart: now,
+        rateLimitTokens: 1_000_000,
+        rateLimitWindowMs: 60_000,
+        avgTokensPerRequest: 500, // conservative default estimate
       });
       this.metrics.set(provider, []);
     }
@@ -476,7 +645,68 @@ export class ProviderHealthManager extends EventEmitter {
 }
 
 // ============================================================
-// Exports
 // ============================================================
+// EXPORTS
+// ============================================================
+
+export default ProviderHealthManager;
+
+/** Singleton instance for use across the app without DI */
+export const globalHealthManager = new ProviderHealthManager();
+
+/**
+ * Stateless MVT rate-limit rotation helper.
+ * Call this after routeQuery returns to check if the selected provider
+ * should be rotated away from due to rate-limit depletion.
+ *
+ * Uses Charnov (1976): g'(t*) = g(t*) / (t* + τ)
+ * Leave when marginal rate ≤ avg rate including switch cost.
+ *
+ * @param providerHealth - current provider health state (from healthManager.getHealth())
+ * @param fallbackProviderLatencyMs - estimated cold-start latency for fallback
+ * @param estimatedTokensThisCall - estimated tokens for this request
+ * @returns true if MVT recommends rotation
+ */
+export function mvtShouldRotate(
+  providerHealth: ProviderHealth,
+  fallbackProviderLatencyMs: number,
+  estimatedTokensThisCall: number = 500,
+): boolean {
+  const now = Date.now();
+
+  // No window started yet — stay
+  if (providerHealth.rateLimitWindowStart === 0) return false;
+
+  const windowElapsed = now - providerHealth.rateLimitWindowStart;
+
+  // Window is fresh — stay
+  if (windowElapsed < 100) return false;
+
+  // Already depleted — rotate immediately
+  if (providerHealth.tokensUsedThisWindow >= providerHealth.rateLimitTokens) return true;
+
+  // Remaining budget after this call
+  const budgetAfter = providerHealth.rateLimitTokens - providerHealth.tokensUsedThisWindow - estimatedTokensThisCall;
+
+  // If this call would exceed the limit, recommend rotation
+  if (budgetAfter < 0) return true;
+
+  const remainingTimeMs = Math.max(1, providerHealth.rateLimitWindowMs - windowElapsed);
+
+  // Marginal rate: tokens/ms we can still consume this window after this call
+  const marginalRate = budgetAfter / remainingTimeMs;
+
+  // Cumulative tokens used so far (proxy for g(t*))
+  const g_t = providerHealth.tokensUsedThisWindow;
+
+  // τ = cold-start latency for fallback
+  const tau = fallbackProviderLatencyMs;
+
+  // Break-even rate: avg rate including switch cost
+  // From MVT: g'(t*) = g(t*) / (t* + τ)
+  // Our marginal rate should exceed this to justify staying
+  const avgRateIncludingSwitch = g_t / (windowElapsed + tau);
 
-export default ProviderHealthManager;
+  // Leave when marginal ≤ break-even (MVT optimality condition)
+  return marginalRate <= avgRateIncludingSwitch;
+}