adaptive-memory-multi-model-router 2.2.5 → 2.2.6

package/src/cache/semanticCache.ts

@@ -7,8 +7,12 @@
  * No external embedding API needed. Trigram overlap catches paraphrases like:
  *   "What is Python?"    ≈ "Tell me about Python" ≈ "Explain Python"
  *   "Write a sort fn"    ≈ "Create a sorting fn"  ≈ "How to sort an array"
+ *
+ * Also supports cross-provider cache key generation via generateCacheKey().
  */
 
+import * as crypto from 'crypto';
+
 // ============================================================
 // Types
 // ============================================================
@@ -53,6 +57,150 @@ function normalize(text: string): string {
     .trim();
 }
 
+// ============================================================
+// Cross-Provider Cache Key Generation
+// ============================================================
+
+/**
+ * Provider-specific formatting patterns to strip for cross-provider cache keys.
+ */
+const PROVIDER_SYSTEM_PATTERNS: Record<string, RegExp[]> = {
+  anthropic: [
+    /<anthropic_thinking>[\s\S]*?<\/anthropic_thinking>/gi,
+    /<thinking>[\s\S]*?<\/thinking>/gi,
+    /Human:/gi,
+    /Assistant:/gi,
+  ],
+  openai: [
+    /<|im_start|>/gi,
+    /<|im_end|>/gi,
+  ],
+  google: [
+    /<content>[\s\S]*?<\/content>/gi,
+    /[Parts|thought]/gi,
+  ],
+};
+
+export interface GenerateCacheKeyOptions {
+  /** Target provider (affects normalization rules) */
+  provider?: string;
+  /** Target model (for model-specific normalization) */
+  model?: string;
+  /** Custom normalization rules */
+  customRules?: Array<{ pattern: RegExp; replacement: string }>;
+}
+
+export interface CacheKeyResult {
+  /** The normalized cache key string */
+  key: string;
+  /** Hash of the normalized content */
+  hash: string;
+  /** Metadata about what was normalized */
+  metadata: {
+    originalLength: number;
+    normalizedLength: number;
+    rulesApplied: number;
+    provider?: string;
+  };
+}
+
+/**
+ * Generate a deterministic cache key from a query.
+ * Same semantic content = same key across providers.
+ *
+ * @example
+ * const key1 = semanticCache.generateCacheKey("What is Python?", { provider: "openai" });
+ * const key2 = semanticCache.generateCacheKey("What is Python?", { provider: "anthropic" });
+ * // key1.key === key2.key (same semantic content = same key)
+ */
+export function generateCacheKey(
+  query: string,
+  options: GenerateCacheKeyOptions = {}
+): CacheKeyResult {
+  const originalLength = query.length;
+
+  // Step 1: Unicode normalization (NFC)
+  let normalized = query.normalize('NFC');
+
+  // Step 2: Collapse whitespace
+  normalized = normalized.replace(/\s+/g, ' ');
+
+  // Step 3: Remove control characters
+  normalized = normalized.replace(/[\x00-\x1F\x7F]/g, '');
+
+  // Step 4: Strip provider-specific formatting
+  if (options.provider) {
+    const patterns = PROVIDER_SYSTEM_PATTERNS[options.provider] || [];
+    for (const pattern of patterns) {
+      normalized = normalized.replace(pattern, '');
+    }
+  }
+
+  // Step 5: General system/assistant role removal
+  normalized = normalized
+    .replace(/\b(system|user|assistant|human|bot)\s*:/gi, '')
+    .replace(/^(system|user|assistant|human|bot)\s*/gim, '');
+
+  // Step 6: Remove markdown formatting
+  normalized = normalized
+    .replace(/```[\s\S]*?```/g, '[CODE_BLOCK]')
+    .replace(/`([^`]+)`/g, '$1')
+    .replace(/\*\*([^*]+)\*\*/g, '$1')
+    .replace(/_([^_]+)_/g, '$1')
+    .replace(/#+\s*/g, '')
+    .replace(/^\s*[-*+]\s+/gm, '')
+    .replace(/^\s*\d+\.\s+/gm, '');
+
+  // Step 7: Apply custom rules
+  if (options.customRules) {
+    for (const rule of options.customRules) {
+      normalized = normalized.replace(rule.pattern, rule.replacement);
+    }
+  }
+
+  // Step 8: Final whitespace collapse
+  normalized = normalized.replace(/\s+/g, ' ').trim();
+
+  // Count rules applied
+  let rulesApplied = 3; // Base normalizations
+  if (options.provider) rulesApplied += 2;
+  if (options.customRules) rulesApplied += options.customRules.length;
+
+  // Generate hash
+  const hash = crypto
+    .createHash('sha256')
+    .update(normalized)
+    .digest('hex')
+    .substring(0, 16);
+
+  // Build key
+  let key = `v1:${hash}`;
+  if (options.provider) {
+    key += `:${options.provider}`;
+    if (options.model) {
+      key += `:${options.model}`;
+    }
+  }
+
+  return {
+    key,
+    hash,
+    metadata: {
+      originalLength,
+      normalizedLength: normalized.length,
+      rulesApplied,
+      provider: options.provider,
+    },
+  };
+}
+
+/**
+ * Quick cache key generation (simplified API).
+ */
+export function toCacheKey(query: string, provider?: string): string {
+  return generateCacheKey(query, { provider }).key;
+}
+
 /**
  * Extract character trigrams from text.
  * Pads with spaces so short words still produce trigrams.

package/src/cost/preCallCostEstimator.ts

@@ -0,0 +1,345 @@
+/**
+ * A3M Router - Pre-Call Cost Estimator
+ *
+ * Estimates cost BEFORE making an API call based on input features.
+ * Uses historical data patterns and token count estimation via character ratio.
+ *
+ * No external API calls - all estimation is local.
+ *
+ * Usage:
+ *   const estimator = new PreCallCostEstimator();
+ *   const estimate = estimator.estimate({
+ *     query: "What is Python?",
+ *     modelTier: 'mid',
+ *     provider: 'groq'
+ *   });
+ *   console.log(estimate); // { estimatedTokens: 24, estimatedCost: 0.0014, estimatedLatency: 200 }
+ */
+
+import { ProviderTier } from '../providers/providerConfig';
+
+// ============================================================
+// Types
+// ============================================================
+
+export interface CostEstimateInput {
+  /** Query text to estimate tokens for */
+  query: string;
+  /** Model tier (free|cheap|mid|premium|enterprise) */
+  modelTier?: ProviderTier;
+  /** Provider ID for provider-specific cost lookup */
+  provider?: string;
+  /** Optional: explicit token count (if already known) */
+  explicitTokens?: number;
+  /** System prompt length (if using a system prompt) */
+  systemPromptLength?: number;
+}
+
+export interface CostEstimate {
+  /** Estimated total tokens (input + output) */
+  estimatedTokens: number;
+  /** Estimated input tokens */
+  estimatedInputTokens: number;
+  /** Estimated output tokens */
+  estimatedOutputTokens: number;
+  /** Estimated cost in USD */
+  estimatedCost: number;
+  /** Estimated latency in milliseconds */
+  estimatedLatency: number;
+  /** Confidence score 0-1 */
+  confidence: number;
+  /** Breakdown of estimation */
+  breakdown: {
+    inputCostPerM: number;
+    outputCostPerM: number;
+    charToTokenRatio: number;
+  };
+}
+
+export interface ProviderCostConfig {
+  input: number;  // per 1M tokens
+  output: number; // per 1M tokens
+}
+
+// Default latency estimates per tier (ms)
+const TIER_LATENCY: Record<ProviderTier, { min: number; max: number; avg: number }> = {
+  free:   { min: 500,  max: 5000,  avg: 2000 },
+  cheap:  { min: 100,  max: 800,   avg: 300 },
+  mid:    { min: 200,  max: 1500,  avg: 600 },
+  premium:{ min: 300,  max: 2000,  avg: 800 },
+  enterprise: { min: 200, max: 1500, avg: 500 },
+};
+
+// ============================================================
+// PreCallCostEstimator
+// ============================================================
+
+export class PreCallCostEstimator {
+  // Historical data for regression: [charCount, actualTokens][] tuples
+  private historicalData: Array<{ chars: number; tokens: number }> = [];
+  // Linear regression coefficients
+  private slope = 0.25; // chars per token ratio
+  private intercept = 10;
+  // EWMA for latency estimation
+  private latencyHistory: number[] = [];
+  private ewmaAlpha = 0.3;
+
+  constructor(historicalData?: Array<{ chars: number; tokens: number }>) {
+    if (historicalData && historicalData.length > 0) {
+      this.fitLinearRegression(historicalData);
+    }
+  }
+
+  /**
+   * Main estimation method - estimates tokens, cost, and latency.
+   */
+  estimate(input: CostEstimateInput): CostEstimate {
+    const {
+      query,
+      modelTier = 'mid',
+      provider,
+      explicitTokens,
+      systemPromptLength = 0,
+    } = input;
+
+    // Token estimation
+    let inputTokens: number;
+    if (explicitTokens !== undefined) {
+      inputTokens = explicitTokens;
+    } else {
+      inputTokens = this.estimateTokens(query, systemPromptLength);
+    }
+
+    // Output token estimation based on query complexity
+    const outputTokens = this.estimateOutputTokens(query, inputTokens);
+
+    const totalTokens = inputTokens + outputTokens;
+
+    // Cost estimation
+    const costConfig = this.getCostConfig(provider, modelTier);
+    const estimatedCost = this.calculateCost(totalTokens, costConfig);
+
+    // Latency estimation
+    const estimatedLatency = this.estimateLatency(modelTier, inputTokens, totalTokens);
+
+    // Confidence based on amount of historical data we have
+    const confidence = this.calculateConfidence();
+
+    return {
+      estimatedTokens: totalTokens,
+      estimatedInputTokens: inputTokens,
+      estimatedOutputTokens: outputTokens,
+      estimatedCost: Math.round(estimatedCost * 1000000) / 1000000, // 6 decimal places
+      estimatedLatency,
+      confidence,
+      breakdown: {
+        inputCostPerM: costConfig.input,
+        outputCostPerM: costConfig.output,
+        charToTokenRatio: this.slope,
+      },
+    };
+  }
+
+  /**
+   * Estimate input tokens using character-to-token ratio.
+   * Uses linear regression if historical data is available.
+   */
+  estimateTokens(text: string, systemPromptLength = 0): number {
+    const totalChars = text.length + systemPromptLength;
+
+    if (this.historicalData.length >= 5) {
+      // Use linear regression
+      return Math.max(1, Math.round(this.slope * totalChars + this.intercept));
+    }
+
+    // Fallback: general English average ~4 chars per token
+    return Math.max(1, Math.round(totalChars / 4));
+  }
+
+  /**
+   * Estimate output tokens based on query complexity.
+   * More complex queries (code, analysis) tend to need more output.
+   */
+  estimateOutputTokens(query: string, inputTokens: number): number {
+    const lower = query.toLowerCase();
+
+    // Base estimate: ~30% of input tokens as output
+    let multiplier = 0.3;
+
+    // Complexity adjustments
+    if (this.containsKeyword(lower, ['code', 'implement', 'function', 'class', 'algorithm'])) {
+      multiplier = 0.5; // Code needs more output
+    } else if (this.containsKeyword(lower, ['explain', 'describe', 'what is', 'how does'])) {
+      multiplier = 0.35; // Explanations need moderate output
+    } else if (this.containsKeyword(lower, ['list', 'count', 'find all'])) {
+      multiplier = 0.4; // List queries need more output
+    } else if (this.containsKeyword(lower, ['yes', 'no', 'is', 'are', 'does'])) {
+      multiplier = 0.1; // Simple questions need minimal output
+    }
+
+    // Cap to reasonable bounds
+    return Math.min(
+      Math.max(10, Math.round(inputTokens * multiplier)),
+      4000 // Max 4k output tokens
+    );
+  }
+
+  /**
+   * Calculate cost in USD.
+   */
+  calculateCost(totalTokens: number, config: ProviderCostConfig): number {
+    const inputM = this.estimateInputTokensFromTotal(totalTokens) / 1_000_000;
+    const outputM = this.estimateOutputTokensFromTotal(totalTokens) / 1_000_000;
+    return inputM * config.input + outputM * config.output;
+  }
+
+  /**
+   * Estimate latency in milliseconds.
+   */
+  estimateLatency(tier: ProviderTier, inputTokens: number, totalTokens: number): number {
+    const tierLatency = TIER_LATENCY[tier];
+
+    // Base latency from tier
+    let latency = tierLatency.avg;
+
+    // Scale by token count (rough linear approximation)
+    const tokenScale = totalTokens / 100;
+    latency *= Math.max(0.5, Math.min(3, tokenScale));
+
+    // Adjust for historical EWMA if available
+    if (this.latencyHistory.length > 0) {
+      const ewmaLatency = this.calculateEwma();
+      latency = latency * 0.7 + ewmaLatency * 0.3;
+    }
+
+    return Math.round(latency);
+  }
+
+  /**
+   * Record actual tokens for future regression improvements.
+   */
+  recordActualTokens(queryLength: number, actualTokens: number): void {
+    this.historicalData.push({ chars: queryLength, tokens: actualTokens });
+
+    // Keep only last 100 data points
+    if (this.historicalData.length > 100) {
+      this.historicalData.shift();
+    }
+
+    // Refit regression periodically
+    if (this.historicalData.length % 10 === 0) {
+      this.fitLinearRegression(this.historicalData);
+    }
+  }
+
+  /**
+   * Record actual latency for EWMA updates.
+   */
+  recordActualLatency(latencyMs: number): void {
+    this.latencyHistory.push(latencyMs);
+    if (this.latencyHistory.length > 50) {
+      this.latencyHistory.shift();
+    }
+  }
+
+  /**
+   * Fit linear regression to historical data.
+   * Uses ordinary least squares.
+   */
+  fitLinearRegression(data: Array<{ chars: number; tokens: number }>): void {
+    if (data.length < 2) return;
+
+    const n = data.length;
+    let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0;
+
+    for (const { chars, tokens } of data) {
+      sumX += chars;
+      sumY += tokens;
+      sumXY += chars * tokens;
+      sumX2 += chars * chars;
+    }
+
+    const denominator = n * sumX2 - sumX * sumX;
+    if (denominator === 0) return;
+
+    this.slope = (n * sumXY - sumX * sumY) / denominator;
+    this.intercept = (sumY - this.slope * sumX) / n;
+
+    // Sanity check
+    if (this.slope <= 0 || this.slope > 1) {
+      this.slope = 0.25; // Reset to default if outlier
+      this.intercept = 10;
+    }
+  }
+
+  // ---- Private helpers ----
+
+  private getCostConfig(provider?: string, tier?: ProviderTier): ProviderCostConfig {
+    // Provider-specific costs (from providerConfig.ts patterns)
+    const providerCosts: Partial<Record<string, ProviderCostConfig>> = {
+      groq:      { input: 0.59,  output: 0.79 },
+      cerebras:  { input: 0.60,  output: 0.60 },
+      deepseek:  { input: 0.14,  output: 0.28 },
+      deepinfra: { input: 0.05,  output: 0.05 },
+      together:  { input: 0.18,  output: 0.18 },
+      fireworks: { input: 0.20,  output: 0.20 },
+      mistral:   { input: 0.20,  output: 0.60 },
+      openai:    { input: 2.50,  output: 10.00 },
+      anthropic: { input: 3.00,  output: 15.00 },
+    };
+
+    if (provider && providerCosts[provider]) {
+      return providerCosts[provider]!;
+    }
+
+    // Tier fallback
+    const tierCosts: Record<ProviderTier, ProviderCostConfig> = {
+      free:       { input: 0,     output: 0 },
+      cheap:      { input: 0.20,  output: 0.40 },
+      mid:        { input: 1.00,  output: 3.00 },
+      premium:    { input: 3.00,  output: 12.00 },
+      enterprise: { input: 5.00,  output: 20.00 },
+    };
+
+    return tier ? tierCosts[tier] : tierCosts.mid;
+  }
+
+  private containsKeyword(text: string, keywords: string[]): boolean {
+    return keywords.some(kw => text.includes(kw));
+  }
+
+  private calculateEwma(): number {
+    if (this.latencyHistory.length === 0) return 0;
+    let ewma = this.latencyHistory[0];
+    for (let i = 1; i < this.latencyHistory.length; i++) {
+      ewma = this.ewmaAlpha * this.latencyHistory[i] + (1 - this.ewmaAlpha) * ewma;
+    }
+    return ewma;
+  }
+
+  private calculateConfidence(): number {
+    // More historical data = higher confidence
+    const dataFactor = Math.min(this.historicalData.length / 50, 1.0);
+    // More latency history = higher confidence
+    const latencyFactor = Math.min(this.latencyHistory.length / 20, 1.0);
+    return Math.round((dataFactor * 0.6 + latencyFactor * 0.4) * 100) / 100;
+  }
+
+  private estimateInputTokensFromTotal(total: number): number {
+    return Math.round(total * 0.7); // Assume 70% input
+  }
+
+  private estimateOutputTokensFromTotal(total: number): number {
+    return Math.round(total * 0.3); // Assume 30% output
+  }
+}
+
+// ============================================================
+// Factory
+// ============================================================
+
+export function createPreCallCostEstimator(
+  historicalData?: Array<{ chars: number; tokens: number }>
+): PreCallCostEstimator {
+  return new PreCallCostEstimator(historicalData);
+}