adaptive-memory-multi-model-router 2.5.5 → 2.7.0

package/src/cache/semanticCache.ts

@@ -1,83 +1,184 @@
 /**
- * A3M Router - Semantic Cache
+ * A3M Router - Semantic Cache (Embedding-based)
  *
  * Stores previous query->response pairs and returns cached responses
- * for semantically similar queries using character n-gram Jaccard similarity.
+ * for semantically similar queries using cosine similarity on embeddings.
  *
- * No external embedding API needed. Trigram overlap catches paraphrases like:
- *   "What is Python?"    ≈ "Tell me about Python" ≈ "Explain Python"
- *   "Write a sort fn"    ≈ "Create a sorting fn"  ≈ "How to sort an array"
+ * Supports embedders: nomic (via Ollama), openai, or local Ollama.
+ * Uses nomic-embed-text by default via local Ollama.
  */
 
 // ============================================================
 // Types
 // ============================================================
 
-export interface CachedResponse {
-  query: string;
-  response: string;
-  metadata?: any;
-  cachedAt: number;
-  hitCount: number;
+export type EmbedderType = 'nomic' | 'openai' | 'local';
+
+export interface SemanticCacheConfig {
+  similarityThreshold: number;  // cosine similarity, e.g., 0.92
+  ttlSeconds: number;          // default TTL in seconds
+  maxEntries?: number;         // LRU cache size limit (default: 1000)
+  embedder?: EmbedderType;     // which embedder to use (default: 'nomic')
+  embedderUrl?: string;        // custom embedder URL
+  embedderApiKey?: string;      // API key for external embedder
 }
 
-export interface SemanticCacheOptions {
-  maxSize?: number;             // Max entries (default: 1000)
-  similarityThreshold?: number; // 0-1, min similarity for hit (default: 0.92)
-  ttl?: number;                 // TTL in ms (default: 3600000 = 1 hour)
+export interface CacheEntry {
+  key: string;                  // hash of normalized query
+  embedding: number[];
+  response: string;
+  provider: string;
+  model: string;
+  cost: number;
+  createdAt: number;
+  ttl: number;                  // TTL in ms
+  hitCount: number;
+  lastAccessedAt: number;       // for LRU tracking
 }
 
-interface CacheEntry extends CachedResponse {
-  trigrams: Set<string>;
-  expiresAt: number;
+export interface SemanticCacheGetResult {
+  hit: boolean;
+  response?: string;
+  provider?: string;
+  model?: string;
+  cost?: number;
+  similarity?: number;
 }
 
 export interface SemanticCacheStats {
+  size: number;
   hits: number;
   misses: number;
   hitRate: number;
-  size: number;
 }
 
 // ============================================================
-// N-gram utilities
+// Embedding Service
+// ============================================================
+
+interface Embedder {
+  embed(text: string): Promise<number[]>;
+}
+
+class NomicEmbedder implements Embedder {
+  constructor(private url: string = 'http://127.0.0.1:11434/api/embeddings') {}
+
+  async embed(text: string): Promise<number[]> {
+    const response = await fetch(this.url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        model: 'nomic-embed-text',
+        prompt: text,
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`Nomic embedder error: ${response.status} ${response.statusText}`);
+    }
+
+    const data = await response.json() as { embedding: number[] };
+    return data.embedding;
+  }
+}
+
+class OpenAIEmbedder implements Embedder {
+  constructor(private apiKey: string, private url: string = 'https://api.openai.com/v1/embeddings') {}
+
+  async embed(text: string): Promise<number[]> {
+    const response = await fetch(this.url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${this.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: 'text-embedding-3-small',
+        input: text,
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`OpenAI embedder error: ${response.status} ${response.statusText}`);
+    }
+
+    const data = await response.json() as { data: Array<{ embedding: number[] }> };
+    return data.data[0]?.embedding ?? [];
+  }
+}
+
+class LocalOllamaEmbedder implements Embedder {
+  constructor(
+    private baseUrl: string = 'http://127.0.0.1:11434/api/embeddings',
+    private model: string = 'nomic-embed-text'
+  ) {}
+
+  async embed(text: string): Promise<number[]> {
+    const response = await fetch(this.baseUrl, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        model: this.model,
+        prompt: text,
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`Local Ollama embedder error: ${response.status} ${response.statusText}`);
+    }
+
+    const data = await response.json() as { embedding: number[] };
+    return data.embedding;
+  }
+}
+
+// ============================================================
+// Utilities
 // ============================================================
 
 /**
- * Normalize text: lowercase, collapse whitespace, strip punctuation edges.
+ * Compute cosine similarity between two vectors.
  */
-function normalize(text: string): string {
-  return text
-    .toLowerCase()
-    .replace(/\s+/g, " ")
-    .trim();
+function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length) return 0;
+  if (a.length === 0) return 0;
+
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+  return denominator === 0 ? 0 : dotProduct / denominator;
 }
 
 /**
- * Extract character trigrams from text.
- * Pads with spaces so short words still produce trigrams.
+ * Normalize text for hashing: lowercase, collapse whitespace.
  */
-function extractTrigrams(text: string): Set<string> {
-  const normalized = " " + normalize(text) + " ";
-  const trigrams = new Set<string>();
-  for (let i = 0; i <= normalized.length - 3; i++) {
-    trigrams.add(normalized.substring(i, i + 3));
-  }
-  return trigrams;
+function normalize(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/\s+/g, ' ')
+    .trim();
 }
 
 /**
- * Compute Jaccard similarity between two sets.
- * |A ∩ B| / |A ∪ B|
+ * Simple hash for cache key (deterministic, fast).
  */
-function jaccard(a: Set<string>, b: Set<string>): number {
-  if (a.size === 0 && b.size === 0) return 1.0;
-  let intersection = 0;
-  for (const item of a) {
-    if (b.has(item)) intersection++;
+function hashKey(text: string): string {
+  let hash = 0;
+  const normalized = normalize(text);
+  for (let i = 0; i < normalized.length; i++) {
+    const char = normalized.charCodeAt(i);
+    hash = ((hash << 5) - hash) + char;
+    hash = hash & hash; // Convert to 32-bit integer
   }
-  const union = a.size + b.size - intersection;
-  return union === 0 ? 0 : intersection / union;
+  return hash.toString(36) + '_' + normalized.slice(0, 20);
 }
 
 // ============================================================
@@ -85,101 +186,176 @@ function jaccard(a: Set<string>, b: Set<string>): number {
 // ============================================================
 
 export class SemanticCache {
-  private entries: CacheEntry[] = [];
-  private maxSize: number;
+  private entries: Map<string, CacheEntry> = new Map();
+  private accessOrder: string[] = []; // track LRU order
+  private embedder: Embedder;
   private similarityThreshold: number;
-  private ttl: number;
+  private ttlMs: number;
+  private maxEntries: number;
   private hits = 0;
   private misses = 0;
 
-  constructor(options?: SemanticCacheOptions) {
-    this.maxSize = options?.maxSize ?? 1000;
-    this.similarityThreshold = options?.similarityThreshold ?? 0.92;
-    this.ttl = options?.ttl ?? 3600000; // 1 hour
+  constructor(config: SemanticCacheConfig) {
+    this.similarityThreshold = config.similarityThreshold;
+    this.ttlMs = (config.ttlSeconds || 3600) * 1000;
+    this.maxEntries = config.maxEntries || 1000;
+
+    // Initialize embedder based on config
+    switch (config.embedder) {
+      case 'openai':
+        if (!config.embedderApiKey) {
+          throw new Error('OpenAI embedder requires apiKey in config.embedderApiKey');
+        }
+        this.embedder = new OpenAIEmbedder(config.embedderApiKey, config.embedderUrl);
+        break;
+      case 'local':
+        this.embedder = new LocalOllamaEmbedder(
+          config.embedderUrl || 'http://127.0.0.1:11434/api/embeddings',
+          'nomic-embed-text'
+        );
+        break;
+      case 'nomic':
+      default:
+        this.embedder = new NomicEmbedder(
+          config.embedderUrl || 'http://127.0.0.1:11434/api/embeddings'
+        );
+        break;
+    }
   }
 
   /**
    * Get cached response for a semantically similar query.
-   * Returns the best match above the similarity threshold, or null.
+   * Returns the best match above the similarity threshold, or { hit: false }.
    */
-  async get(query: string): Promise<CachedResponse | null> {
+  async get(query: string): Promise<SemanticCacheGetResult> {
     const now = Date.now();
-    const queryTrigrams = extractTrigrams(query);
+
+    // Generate embedding for the query
+    let queryEmbedding: number[];
+    try {
+      queryEmbedding = await this.embedder.embed(query);
+    } catch (error) {
+      console.warn('SemanticCache: Failed to generate embedding for query:', error);
+      this.misses++;
+      return { hit: false };
+    }
 
     let bestEntry: CacheEntry | null = null;
-    let bestScore = 0;
+    let bestSimilarity = 0;
 
-    for (const entry of this.entries) {
-      // Skip expired
-      if (now > entry.expiresAt) continue;
+    for (const entry of this.entries.values()) {
+      // Skip expired entries
+      if (now > entry.createdAt + entry.ttl) continue;
 
-      const score = jaccard(queryTrigrams, entry.trigrams);
-      if (score > bestScore) {
-        bestScore = score;
+      const similarity = cosineSimilarity(queryEmbedding, entry.embedding);
+      if (similarity > bestSimilarity) {
+        bestSimilarity = similarity;
         bestEntry = entry;
       }
     }
 
-    if (bestEntry && bestScore >= this.similarityThreshold) {
+    if (bestEntry && bestSimilarity >= this.similarityThreshold) {
       this.hits++;
       bestEntry.hitCount++;
+      bestEntry.lastAccessedAt = now;
+      this.updateAccessOrder(bestEntry.key);
+
       return {
-        query: bestEntry.query,
+        hit: true,
         response: bestEntry.response,
-        metadata: bestEntry.metadata,
-        cachedAt: bestEntry.cachedAt,
-        hitCount: bestEntry.hitCount,
+        provider: bestEntry.provider,
+        model: bestEntry.model,
+        cost: bestEntry.cost,
+        similarity: bestSimilarity,
       };
     }
 
     this.misses++;
-    return null;
+    return { hit: false };
   }
 
   /**
    * Store a query->response pair in the cache.
    */
-  async set(query: string, response: string, metadata?: any): Promise<void> {
+  async set(
+    query: string,
+    response: string,
+    metadata: {
+      provider: string;
+      model: string;
+      cost: number;
+      ttl?: number;
+    }
+  ): Promise<void> {
     const now = Date.now();
+    const key = hashKey(query);
+
+    // Generate embedding
+    let embedding: number[];
+    try {
+      embedding = await this.embedder.embed(query);
+    } catch (error) {
+      console.warn('SemanticCache: Failed to generate embedding for set:', error);
+      return;
+    }
 
-    // Evict expired entries first
+    // Evict expired entries
     this.evictExpired();
 
-    // Evict oldest if at capacity
-    if (this.entries.length >= this.maxSize) {
-      this.evictOldest();
+    // Evict oldest if at capacity (LRU)
+    if (this.entries.size >= this.maxEntries) {
+      this.evictLRU();
     }
 
-    // Check if an exact-match entry already exists and update it
-    const normalized = normalize(query);
-    const existing = this.entries.find(
-      (e) => normalize(e.query) === normalized && now <= e.expiresAt
-    );
-    if (existing) {
+    // Check if entry already exists and update it
+    if (this.entries.has(key)) {
+      const existing = this.entries.get(key)!;
       existing.response = response;
-      existing.metadata = metadata;
-      existing.cachedAt = now;
-      existing.expiresAt = now + this.ttl;
-      existing.trigrams = extractTrigrams(query);
+      existing.embedding = embedding;
+      existing.provider = metadata.provider;
+      existing.model = metadata.model;
+      existing.cost = metadata.cost;
+      existing.createdAt = now;
+      existing.ttl = (metadata.ttl !== undefined ? metadata.ttl : this.ttlMs / 1000) * 1000;
+      existing.lastAccessedAt = now;
+      existing.hitCount = existing.hitCount; // preserve hit count
       return;
     }
 
-    this.entries.push({
-      query,
+    const entry: CacheEntry = {
+      key,
+      embedding,
       response,
-      metadata,
-      cachedAt: now,
-      expiresAt: now + this.ttl,
+      provider: metadata.provider,
+      model: metadata.model,
+      cost: metadata.cost,
+      createdAt: now,
+      ttl: (metadata.ttl !== undefined ? metadata.ttl : this.ttlMs / 1000) * 1000,
       hitCount: 0,
-      trigrams: extractTrigrams(query),
-    });
+      lastAccessedAt: now,
+    };
+
+    this.entries.set(key, entry);
+    this.accessOrder.push(key);
+  }
+
+  /**
+   * Delete a specific query from the cache.
+   */
+  async delete(query: string): Promise<void> {
+    const key = hashKey(query);
+    if (this.entries.has(key)) {
+      this.entries.delete(key);
+      this.accessOrder = this.accessOrder.filter(k => k !== key);
+    }
   }
 
   /**
    * Clear all cache entries.
    */
   clear(): void {
-    this.entries = [];
+    this.entries.clear();
+    this.accessOrder = [];
   }
 
   /**
@@ -188,34 +364,48 @@ export class SemanticCache {
   getStats(): SemanticCacheStats {
     const total = this.hits + this.misses;
     return {
+      size: this.entries.size,
       hits: this.hits,
       misses: this.misses,
       hitRate: total > 0 ? this.hits / total : 0,
-      size: this.entries.length,
     };
   }
 
+  /**
+   * Update access order for LRU tracking.
+   */
+  private updateAccessOrder(key: string): void {
+    this.accessOrder = this.accessOrder.filter(k => k !== key);
+    this.accessOrder.push(key);
+  }
+
   /**
    * Purge expired entries.
    */
   private evictExpired(): void {
     const now = Date.now();
-    this.entries = this.entries.filter((e) => now <= e.expiresAt);
+    for (const [key, entry] of this.entries.entries()) {
+      if (now > entry.createdAt + entry.ttl) {
+        this.entries.delete(key);
+        this.accessOrder = this.accessOrder.filter(k => k !== key);
+      }
+    }
   }
 
   /**
-   * Evict the oldest (by cachedAt) entry.
+   * Evict the least recently used entry.
    */
-  private evictOldest(): void {
-    if (this.entries.length === 0) return;
-    let oldestIdx = 0;
-    let oldestTime = Infinity;
-    for (let i = 0; i < this.entries.length; i++) {
-      if (this.entries[i].cachedAt < oldestTime) {
-        oldestTime = this.entries[i].cachedAt;
-        oldestIdx = i;
-      }
+  private evictLRU(): void {
+    if (this.accessOrder.length === 0) return;
+    const lruKey = this.accessOrder.shift();
+    if (lruKey && this.entries.has(lruKey)) {
+      this.entries.delete(lruKey);
     }
-    this.entries.splice(oldestIdx, 1);
   }
 }
+
+// ============================================================
+// Exports
+// ============================================================
+
+export default SemanticCache;