adaptive-memory-multi-model-router 2.2.9 → 2.4.0

package/src/cache/cacheKeyGenerator.ts

@@ -1,242 +0,0 @@
-/**
- * A3M Router - Cross-Provider Cache Key Generator
- *
- * Normalizes prompts so same semantic content maps to same cache key
- * regardless of provider-specific formatting, system prompts, etc.
- *
- * Usage:
- *   const cacheKey = generateCacheKey("What is Python?", { provider: "openai" });
- *   const key2 = generateCacheKey("What is Python?", { provider: "anthropic" });
- *   // key === key2 (same semantic content = same key)
- */
-
-import * as crypto from 'crypto';
-
-// ============================================================
-// Types
-// ============================================================
-
-export interface CacheKeyOptions {
-  /** Target provider (affects normalization rules) */
-  provider?: string;
-  /** Target model (for model-specific normalization) */
-  model?: string;
-  /** Whether to include system prompt in normalization */
-  includeSystemPrompt?: boolean;
-  /** Custom normalization rules */
-  customRules?: NormalizationRule[];
-}
-
-export interface NormalizationRule {
-  pattern: RegExp;
-  replacement: string;
-}
-
-export interface CacheKeyResult {
-  /** The normalized cache key string */
-  key: string;
-  /** Hash of the normalized content */
-  hash: string;
-  /** Metadata about what was normalized */
-  metadata: {
-    originalLength: number;
-    normalizedLength: number;
-    rulesApplied: number;
-    provider?: string;
-  };
-}
-
-// ============================================================
-// Provider-specific system prompt patterns
-// ============================================================
-
-const PROVIDER_SYSTEM_PATTERNS: Record<string, RegExp[]> = {
-  anthropic: [
-    /<anthropic_thinking>[\s\S]*?<\/anthropic_thinking>/gi,
-    /<thinking>[\s\S]*?<\/thinking>/gi,
-    /Human:/gi,
-    /Assistant:/gi,
-  ],
-  openai: [
-    /<|im_start|>/gi,
-    /<|im_end|>/gi,
-  ],
-  google: [
-    /<content>[\s\S]*?<\/content>/gi,
-    /[\Parts|thought]/gi,
-  ],
-};
-
-// ============================================================
-// Core normalizer
-// ============================================================
-
-/**
- * Normalize text for cross-provider cache key generation.
- * Removes provider-specific formatting while preserving semantic content.
- */
-export function normalizeForCacheKey(
-  text: string,
-  options: CacheKeyOptions = {}
-): string {
-  let normalized = text;
-
-  // Step 1: Unicode normalization (NFC)
-  normalized = normalized.normalize('NFC');
-
-  // Step 2: Collapse whitespace
-  normalized = normalized.replace(/\s+/g, ' ');
-
-  // Step 3: Remove control characters
-  normalized = normalized.replace(/[\x00-\x1F\x7F]/g, '');
-
-  // Step 4: Strip provider-specific formatting
-  if (options.provider) {
-    const patterns = PROVIDER_SYSTEM_PATTERNS[options.provider] || [];
-    for (const pattern of patterns) {
-      normalized = normalized.replace(pattern, '');
-    }
-  }
-
-  // Step 5: General system/assistant role removal
-  normalized = normalized
-    .replace(/\b(system|user|assistant|human|bot)\s*:/gi, '')
-    .replace(/^(system|user|assistant|human|bot)\s*/gim, '');
-
-  // Step 6: Remove markdown formatting (often provider-specific)
-  normalized = normalized
-    .replace(/```[\s\S]*?```/g, '[CODE_BLOCK]')  // Preserve code block indicator
-    .replace(/`([^`]+)`/g, '$1')                 // Inline code content
-    .replace(/\*\*([^*]+)\*\*/g, '$1')           // Bold
-    .replace(/_([^_]+)_/g, '$1')                  // Italic
-    .replace(/#+\s*/g, '')                        // Headers
-    .replace(/^\s*[-*+]\s+/gm, '')                // List bullets
-    .replace(/^\s*\d+\.\s+/gm, '');              // Numbered lists
-
-  // Step 7: Apply custom rules
-  if (options.customRules) {
-    for (const rule of options.customRules) {
-      normalized = normalized.replace(rule.pattern, rule.replacement);
-    }
-  }
-
-  // Step 8: Collapse whitespace again after removals
-  normalized = normalized.replace(/\s+/g, ' ').trim();
-
-  return normalized;
-}
-
-/**
- * Generate a deterministic cache key from a query.
- * Same semantic content = same key across providers.
- */
-export function generateCacheKey(
-  query: string,
-  options: CacheKeyOptions = {}
-): CacheKeyResult {
-  const originalLength = query.length;
-
-  // Normalize the query
-  let normalized = normalizeForCacheKey(query, {
-    ...options,
-    includeSystemPrompt: false, // Always exclude for user query matching
-  });
-
-  // Count rules that were applied (approximate)
-  let rulesApplied = 3; // Base normalizations
-  if (options.provider) rulesApplied += 2;
-  if (options.customRules) rulesApplied += options.customRules.length;
-
-  // Generate hash
-  const hash = crypto
-    .createHash('sha256')
-    .update(normalized)
-    .digest('hex')
-    .substring(0, 16); // First 16 chars = 64-bit key
-
-  // Final key format: v1:{hash}:{provider?[:model]?}
-  let key = `v1:${hash}`;
-  if (options.provider) {
-    key += `:${options.provider}`;
-    if (options.model) {
-      key += `:${options.model}`;
-    }
-  }
-
-  return {
-    key,
-    hash,
-    metadata: {
-      originalLength,
-      normalizedLength: normalized.length,
-      rulesApplied,
-      provider: options.provider,
-    },
-  };
-}
-
-// ============================================================
-// SemanticCache enhancement
-// ============================================================
-
-/**
- * Add cross-provider cache key methods to existing SemanticCache.
- * Call this to enhance the cache with provider-normalized lookups.
- */
-export function createCacheKeyGenerator(
-  defaultOptions?: CacheKeyOptions
-): {
-  generateKey: (query: string, options?: CacheKeyOptions) => CacheKeyResult;
-  createNormalizedMatcher: (cache: Map<string, any>) => (query: string, options?: CacheKeyOptions) => string | null;
-} {
-  return {
-    /**
-     * Generate a cache key for a query.
-     */
-    generateKey: (query: string, options?: CacheKeyOptions): CacheKeyResult => {
-      return generateCacheKey(query, { ...defaultOptions, ...options });
-    },
-
-    /**
-     * Create a matcher function that finds existing cache entries
-     * by comparing normalized keys.
-     */
-    createNormalizedMatcher: (cache: Map<string, any>) => {
-      return (query: string, options?: CacheKeyOptions): string | null => {
-        const { key } = generateCacheKey(query, { ...defaultOptions, ...options });
-
-        // Check exact match
-        if (cache.has(key)) {
-          return key;
-        }
-
-        // Check hash-only match (v1:{hash} prefix)
-        const hashPrefix = key.split(':').slice(0, 2).join(':');
-        for (const cachedKey of cache.keys()) {
-          if (cachedKey.startsWith(hashPrefix + ':')) {
-            return cachedKey;
-          }
-        }
-
-        return null;
-      };
-    },
-  };
-}
-
-// ============================================================
-// Convenience exports
-// ============================================================
-
-/**
- * Quick cache key generation (simplified API).
- * Use this for simple cross-provider cache lookups.
- *
- * @example
- * const key1 = toCacheKey("What is Python?", "openai");
- * const key2 = toCacheKey("What is Python?", "anthropic");
- * console.log(key1 === key2); // true
- */
-export function toCacheKey(query: string, provider?: string): string {
-  return generateCacheKey(query, { provider }).key;
-}

package/src/cost/preCallCostEstimator.ts

@@ -1,345 +0,0 @@
-/**
- * A3M Router - Pre-Call Cost Estimator
- *
- * Estimates cost BEFORE making an API call based on input features.
- * Uses historical data patterns and token count estimation via character ratio.
- *
- * No external API calls - all estimation is local.
- *
- * Usage:
- *   const estimator = new PreCallCostEstimator();
- *   const estimate = estimator.estimate({
- *     query: "What is Python?",
- *     modelTier: 'mid',
- *     provider: 'groq'
- *   });
- *   console.log(estimate); // { estimatedTokens: 24, estimatedCost: 0.0014, estimatedLatency: 200 }
- */
-
-import { ProviderTier } from '../providers/providerConfig';
-
-// ============================================================
-// Types
-// ============================================================
-
-export interface CostEstimateInput {
-  /** Query text to estimate tokens for */
-  query: string;
-  /** Model tier (free|cheap|mid|premium|enterprise) */
-  modelTier?: ProviderTier;
-  /** Provider ID for provider-specific cost lookup */
-  provider?: string;
-  /** Optional: explicit token count (if already known) */
-  explicitTokens?: number;
-  /** System prompt length (if using a system prompt) */
-  systemPromptLength?: number;
-}
-
-export interface CostEstimate {
-  /** Estimated total tokens (input + output) */
-  estimatedTokens: number;
-  /** Estimated input tokens */
-  estimatedInputTokens: number;
-  /** Estimated output tokens */
-  estimatedOutputTokens: number;
-  /** Estimated cost in USD */
-  estimatedCost: number;
-  /** Estimated latency in milliseconds */
-  estimatedLatency: number;
-  /** Confidence score 0-1 */
-  confidence: number;
-  /** Breakdown of estimation */
-  breakdown: {
-    inputCostPerM: number;
-    outputCostPerM: number;
-    charToTokenRatio: number;
-  };
-}
-
-export interface ProviderCostConfig {
-  input: number;  // per 1M tokens
-  output: number; // per 1M tokens
-}
-
-// Default latency estimates per tier (ms)
-const TIER_LATENCY: Record<ProviderTier, { min: number; max: number; avg: number }> = {
-  free:   { min: 500,  max: 5000,  avg: 2000 },
-  cheap:  { min: 100,  max: 800,   avg: 300 },
-  mid:    { min: 200,  max: 1500,  avg: 600 },
-  premium:{ min: 300,  max: 2000,  avg: 800 },
-  enterprise: { min: 200, max: 1500, avg: 500 },
-};
-
-// ============================================================
-// PreCallCostEstimator
-// ============================================================
-
-export class PreCallCostEstimator {
-  // Historical data for regression: [charCount, actualTokens][] tuples
-  private historicalData: Array<{ chars: number; tokens: number }> = [];
-  // Linear regression coefficients
-  private slope = 0.25; // chars per token ratio
-  private intercept = 10;
-  // EWMA for latency estimation
-  private latencyHistory: number[] = [];
-  private ewmaAlpha = 0.3;
-
-  constructor(historicalData?: Array<{ chars: number; tokens: number }>) {
-    if (historicalData && historicalData.length > 0) {
-      this.fitLinearRegression(historicalData);
-    }
-  }
-
-  /**
-   * Main estimation method - estimates tokens, cost, and latency.
-   */
-  estimate(input: CostEstimateInput): CostEstimate {
-    const {
-      query,
-      modelTier = 'mid',
-      provider,
-      explicitTokens,
-      systemPromptLength = 0,
-    } = input;
-
-    // Token estimation
-    let inputTokens: number;
-    if (explicitTokens !== undefined) {
-      inputTokens = explicitTokens;
-    } else {
-      inputTokens = this.estimateTokens(query, systemPromptLength);
-    }
-
-    // Output token estimation based on query complexity
-    const outputTokens = this.estimateOutputTokens(query, inputTokens);
-
-    const totalTokens = inputTokens + outputTokens;
-
-    // Cost estimation
-    const costConfig = this.getCostConfig(provider, modelTier);
-    const estimatedCost = this.calculateCost(totalTokens, costConfig);
-
-    // Latency estimation
-    const estimatedLatency = this.estimateLatency(modelTier, inputTokens, totalTokens);
-
-    // Confidence based on amount of historical data we have
-    const confidence = this.calculateConfidence();
-
-    return {
-      estimatedTokens: totalTokens,
-      estimatedInputTokens: inputTokens,
-      estimatedOutputTokens: outputTokens,
-      estimatedCost: Math.round(estimatedCost * 1000000) / 1000000, // 6 decimal places
-      estimatedLatency,
-      confidence,
-      breakdown: {
-        inputCostPerM: costConfig.input,
-        outputCostPerM: costConfig.output,
-        charToTokenRatio: this.slope,
-      },
-    };
-  }
-
-  /**
-   * Estimate input tokens using character-to-token ratio.
-   * Uses linear regression if historical data is available.
-   */
-  estimateTokens(text: string, systemPromptLength = 0): number {
-    const totalChars = text.length + systemPromptLength;
-
-    if (this.historicalData.length >= 5) {
-      // Use linear regression
-      return Math.max(1, Math.round(this.slope * totalChars + this.intercept));
-    }
-
-    // Fallback: general English average ~4 chars per token
-    return Math.max(1, Math.round(totalChars / 4));
-  }
-
-  /**
-   * Estimate output tokens based on query complexity.
-   * More complex queries (code, analysis) tend to need more output.
-   */
-  estimateOutputTokens(query: string, inputTokens: number): number {
-    const lower = query.toLowerCase();
-
-    // Base estimate: ~30% of input tokens as output
-    let multiplier = 0.3;
-
-    // Complexity adjustments
-    if (this.containsKeyword(lower, ['code', 'implement', 'function', 'class', 'algorithm'])) {
-      multiplier = 0.5; // Code needs more output
-    } else if (this.containsKeyword(lower, ['explain', 'describe', 'what is', 'how does'])) {
-      multiplier = 0.35; // Explanations need moderate output
-    } else if (this.containsKeyword(lower, ['list', 'count', 'find all'])) {
-      multiplier = 0.4; // List queries need more output
-    } else if (this.containsKeyword(lower, ['yes', 'no', 'is', 'are', 'does'])) {
-      multiplier = 0.1; // Simple questions need minimal output
-    }
-
-    // Cap to reasonable bounds
-    return Math.min(
-      Math.max(10, Math.round(inputTokens * multiplier)),
-      4000 // Max 4k output tokens
-    );
-  }
-
-  /**
-   * Calculate cost in USD.
-   */
-  calculateCost(totalTokens: number, config: ProviderCostConfig): number {
-    const inputM = this.estimateInputTokensFromTotal(totalTokens) / 1_000_000;
-    const outputM = this.estimateOutputTokensFromTotal(totalTokens) / 1_000_000;
-    return inputM * config.input + outputM * config.output;
-  }
-
-  /**
-   * Estimate latency in milliseconds.
-   */
-  estimateLatency(tier: ProviderTier, inputTokens: number, totalTokens: number): number {
-    const tierLatency = TIER_LATENCY[tier];
-
-    // Base latency from tier
-    let latency = tierLatency.avg;
-
-    // Scale by token count (rough linear approximation)
-    const tokenScale = totalTokens / 100;
-    latency *= Math.max(0.5, Math.min(3, tokenScale));
-
-    // Adjust for historical EWMA if available
-    if (this.latencyHistory.length > 0) {
-      const ewmaLatency = this.calculateEwma();
-      latency = latency * 0.7 + ewmaLatency * 0.3;
-    }
-
-    return Math.round(latency);
-  }
-
-  /**
-   * Record actual tokens for future regression improvements.
-   */
-  recordActualTokens(queryLength: number, actualTokens: number): void {
-    this.historicalData.push({ chars: queryLength, tokens: actualTokens });
-
-    // Keep only last 100 data points
-    if (this.historicalData.length > 100) {
-      this.historicalData.shift();
-    }
-
-    // Refit regression periodically
-    if (this.historicalData.length % 10 === 0) {
-      this.fitLinearRegression(this.historicalData);
-    }
-  }
-
-  /**
-   * Record actual latency for EWMA updates.
-   */
-  recordActualLatency(latencyMs: number): void {
-    this.latencyHistory.push(latencyMs);
-    if (this.latencyHistory.length > 50) {
-      this.latencyHistory.shift();
-    }
-  }
-
-  /**
-   * Fit linear regression to historical data.
-   * Uses ordinary least squares.
-   */
-  fitLinearRegression(data: Array<{ chars: number; tokens: number }>): void {
-    if (data.length < 2) return;
-
-    const n = data.length;
-    let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0;
-
-    for (const { chars, tokens } of data) {
-      sumX += chars;
-      sumY += tokens;
-      sumXY += chars * tokens;
-      sumX2 += chars * chars;
-    }
-
-    const denominator = n * sumX2 - sumX * sumX;
-    if (denominator === 0) return;
-
-    this.slope = (n * sumXY - sumX * sumY) / denominator;
-    this.intercept = (sumY - this.slope * sumX) / n;
-
-    // Sanity check
-    if (this.slope <= 0 || this.slope > 1) {
-      this.slope = 0.25; // Reset to default if outlier
-      this.intercept = 10;
-    }
-  }
-
-  // ---- Private helpers ----
-
-  private getCostConfig(provider?: string, tier?: ProviderTier): ProviderCostConfig {
-    // Provider-specific costs (from providerConfig.ts patterns)
-    const providerCosts: Partial<Record<string, ProviderCostConfig>> = {
-      groq:      { input: 0.59,  output: 0.79 },
-      cerebras:  { input: 0.60,  output: 0.60 },
-      deepseek:  { input: 0.14,  output: 0.28 },
-      deepinfra: { input: 0.05,  output: 0.05 },
-      together:  { input: 0.18,  output: 0.18 },
-      fireworks: { input: 0.20,  output: 0.20 },
-      mistral:   { input: 0.20,  output: 0.60 },
-      openai:    { input: 2.50,  output: 10.00 },
-      anthropic: { input: 3.00,  output: 15.00 },
-    };
-
-    if (provider && providerCosts[provider]) {
-      return providerCosts[provider]!;
-    }
-
-    // Tier fallback
-    const tierCosts: Record<ProviderTier, ProviderCostConfig> = {
-      free:       { input: 0,     output: 0 },
-      cheap:      { input: 0.20,  output: 0.40 },
-      mid:        { input: 1.00,  output: 3.00 },
-      premium:    { input: 3.00,  output: 12.00 },
-      enterprise: { input: 5.00,  output: 20.00 },
-    };
-
-    return tier ? tierCosts[tier] : tierCosts.mid;
-  }
-
-  private containsKeyword(text: string, keywords: string[]): boolean {
-    return keywords.some(kw => text.includes(kw));
-  }
-
-  private calculateEwma(): number {
-    if (this.latencyHistory.length === 0) return 0;
-    let ewma = this.latencyHistory[0];
-    for (let i = 1; i < this.latencyHistory.length; i++) {
-      ewma = this.ewmaAlpha * this.latencyHistory[i] + (1 - this.ewmaAlpha) * ewma;
-    }
-    return ewma;
-  }
-
-  private calculateConfidence(): number {
-    // More historical data = higher confidence
-    const dataFactor = Math.min(this.historicalData.length / 50, 1.0);
-    // More latency history = higher confidence
-    const latencyFactor = Math.min(this.latencyHistory.length / 20, 1.0);
-    return Math.round((dataFactor * 0.6 + latencyFactor * 0.4) * 100) / 100;
-  }
-
-  private estimateInputTokensFromTotal(total: number): number {
-    return Math.round(total * 0.7); // Assume 70% input
-  }
-
-  private estimateOutputTokensFromTotal(total: number): number {
-    return Math.round(total * 0.3); // Assume 30% output
-  }
-}
-
-// ============================================================
-// Factory
-// ============================================================
-
-export function createPreCallCostEstimator(
-  historicalData?: Array<{ chars: number; tokens: number }>
-): PreCallCostEstimator {
-  return new PreCallCostEstimator(historicalData);
-}