@nano-llm-cache/core 1.0.0

package/dist/index.mjs

@@ -0,0 +1,423 @@
+// src/storage.ts
+import { get, set, del, keys } from "idb-keyval";
+var CacheStorage = class {
+  constructor(prefix = "nano-llm-cache") {
+    this.prefix = prefix;
+  }
+  /**
+   * Generate storage key
+   */
+  getKey(id) {
+    return `${this.prefix}:${id}`;
+  }
+  /**
+   * Save a cache entry
+   */
+  async save(id, entry) {
+    await set(this.getKey(id), entry);
+  }
+  /**
+   * Get a cache entry by ID
+   */
+  async get(id) {
+    return await get(this.getKey(id));
+  }
+  /**
+   * Get all cache entries
+   */
+  async getAll() {
+    const allKeys = await keys();
+    const cacheKeys = allKeys.filter(
+      (key) => typeof key === "string" && key.startsWith(this.prefix)
+    );
+    const entries = [];
+    for (const key of cacheKeys) {
+      const entry = await get(key);
+      if (entry) {
+        entries.push(entry);
+      }
+    }
+    return entries;
+  }
+  /**
+   * Delete a cache entry
+   */
+  async delete(id) {
+    await del(this.getKey(id));
+  }
+  /**
+   * Clear all cache entries
+   */
+  async clear() {
+    const allKeys = await keys();
+    const cacheKeys = allKeys.filter(
+      (key) => typeof key === "string" && key.startsWith(this.prefix)
+    );
+    for (const key of cacheKeys) {
+      await del(key);
+    }
+  }
+  /**
+   * Remove expired entries based on maxAge
+   */
+  async removeExpired(maxAge) {
+    const now = Date.now();
+    const entries = await this.getAll();
+    let removedCount = 0;
+    for (const entry of entries) {
+      if (now - entry.timestamp > maxAge) {
+        const id = this.hashPrompt(entry.prompt);
+        await this.delete(id);
+        removedCount++;
+      }
+    }
+    return removedCount;
+  }
+  /**
+   * Simple hash function for prompt
+   */
+  hashPrompt(prompt) {
+    let hash = 0;
+    for (let i = 0; i < prompt.length; i++) {
+      const char = prompt.charCodeAt(i);
+      hash = (hash << 5) - hash + char;
+      hash = hash & hash;
+    }
+    return Math.abs(hash).toString(36);
+  }
+  /**
+   * Get cache statistics
+   */
+  async getStats() {
+    const entries = await this.getAll();
+    if (entries.length === 0) {
+      return {
+        totalEntries: 0,
+        oldestEntry: null,
+        newestEntry: null
+      };
+    }
+    const timestamps = entries.map((e) => e.timestamp);
+    return {
+      totalEntries: entries.length,
+      oldestEntry: Math.min(...timestamps),
+      newestEntry: Math.max(...timestamps)
+    };
+  }
+};
+
+// src/embeddings.ts
+import { pipeline, env } from "@xenova/transformers";
+
+// src/similarity.ts
+function calculateSimilarity(vecA, vecB) {
+  if (vecA.length !== vecB.length) {
+    throw new Error("Vectors must have the same length");
+  }
+  if (vecA.length === 0) {
+    return 0;
+  }
+  let dotProduct = 0;
+  let magnitudeA = 0;
+  let magnitudeB = 0;
+  for (let i = 0; i < vecA.length; i++) {
+    dotProduct += vecA[i] * vecB[i];
+    magnitudeA += vecA[i] * vecA[i];
+    magnitudeB += vecB[i] * vecB[i];
+  }
+  magnitudeA = Math.sqrt(magnitudeA);
+  magnitudeB = Math.sqrt(magnitudeB);
+  if (magnitudeA === 0 || magnitudeB === 0) {
+    return 0;
+  }
+  const similarity = dotProduct / (magnitudeA * magnitudeB);
+  return Math.max(0, Math.min(1, similarity));
+}
+function normalizeVector(vec) {
+  const magnitude = Math.sqrt(vec.reduce((sum, val) => sum + val * val, 0));
+  if (magnitude === 0) {
+    return vec;
+  }
+  return vec.map((val) => val / magnitude);
+}
+function toArray(arrayLike) {
+  return Array.from(arrayLike);
+}
+
+// src/embeddings.ts
+env.allowLocalModels = false;
+env.useBrowserCache = true;
+var EmbeddingGenerator = class {
+  constructor(modelName = "Xenova/all-MiniLM-L6-v2", debug = false) {
+    this.model = null;
+    this.loading = null;
+    this.modelName = modelName;
+    this.debug = debug;
+  }
+  /**
+   * Initialize the embedding model (lazy loading)
+   */
+  async initialize() {
+    if (this.model) {
+      return;
+    }
+    if (this.loading) {
+      await this.loading;
+      return;
+    }
+    this.loading = (async () => {
+      try {
+        if (this.debug) {
+          console.log(`[NanoCache] Loading embedding model: ${this.modelName}`);
+        }
+        this.model = await pipeline("feature-extraction", this.modelName);
+        if (this.debug) {
+          console.log("[NanoCache] Embedding model loaded successfully");
+        }
+      } catch (error) {
+        this.loading = null;
+        throw new Error(`Failed to load embedding model: ${error}`);
+      }
+    })();
+    await this.loading;
+  }
+  /**
+   * Generate embedding for a text prompt
+   */
+  async generate(text) {
+    await this.initialize();
+    if (!this.model) {
+      throw new Error("Embedding model not initialized");
+    }
+    try {
+      const output = await this.model(text, {
+        pooling: "mean",
+        normalize: true
+      });
+      let embedding;
+      if (output.data) {
+        embedding = toArray(output.data);
+      } else if (Array.isArray(output)) {
+        embedding = output;
+      } else {
+        throw new Error("Unexpected embedding output format");
+      }
+      if (this.debug) {
+        console.log(`[NanoCache] Generated embedding of length ${embedding.length}`);
+      }
+      return embedding;
+    } catch (error) {
+      throw new Error(`Failed to generate embedding: ${error}`);
+    }
+  }
+  /**
+   * Generate embeddings for multiple texts in batch
+   */
+  async generateBatch(texts) {
+    await this.initialize();
+    if (!this.model) {
+      throw new Error("Embedding model not initialized");
+    }
+    const embeddings = [];
+    for (const text of texts) {
+      const embedding = await this.generate(text);
+      embeddings.push(embedding);
+    }
+    return embeddings;
+  }
+  /**
+   * Check if model is loaded
+   */
+  isLoaded() {
+    return this.model !== null;
+  }
+  /**
+   * Unload the model to free memory
+   */
+  async unload() {
+    this.model = null;
+    this.loading = null;
+    if (this.debug) {
+      console.log("[NanoCache] Embedding model unloaded");
+    }
+  }
+};
+
+// src/cache.ts
+var NanoCache = class {
+  constructor(config = {}) {
+    this.config = {
+      similarityThreshold: config.similarityThreshold ?? 0.95,
+      maxAge: config.maxAge ?? 0,
+      modelName: config.modelName ?? "Xenova/all-MiniLM-L6-v2",
+      debug: config.debug ?? false,
+      storagePrefix: config.storagePrefix ?? "nano-llm-cache"
+    };
+    this.storage = new CacheStorage(this.config.storagePrefix);
+    this.embeddings = new EmbeddingGenerator(this.config.modelName, this.config.debug);
+  }
+  /**
+   * Query the cache for a similar prompt
+   */
+  async query(prompt) {
+    try {
+      if (this.config.maxAge > 0) {
+        await this.storage.removeExpired(this.config.maxAge);
+      }
+      const queryEmbedding = await this.embeddings.generate(prompt);
+      const entries = await this.storage.getAll();
+      if (entries.length === 0) {
+        if (this.config.debug) {
+          console.log("[NanoCache] Cache is empty");
+        }
+        return { hit: false };
+      }
+      let bestMatch = null;
+      let bestSimilarity = 0;
+      for (const entry of entries) {
+        const similarity = calculateSimilarity(queryEmbedding, entry.embedding);
+        if (similarity > bestSimilarity) {
+          bestSimilarity = similarity;
+          bestMatch = entry;
+        }
+      }
+      if (bestMatch && bestSimilarity >= this.config.similarityThreshold) {
+        if (this.config.debug) {
+          console.log(`[NanoCache] Cache HIT! Similarity: ${bestSimilarity.toFixed(4)}`);
+          console.log(`[NanoCache] Original: "${bestMatch.prompt}"`);
+          console.log(`[NanoCache] Query: "${prompt}"`);
+        }
+        return {
+          hit: true,
+          response: bestMatch.response,
+          similarity: bestSimilarity,
+          entry: bestMatch
+        };
+      }
+      if (this.config.debug) {
+        console.log(`[NanoCache] Cache MISS. Best similarity: ${bestSimilarity.toFixed(4)}`);
+      }
+      return { hit: false, similarity: bestSimilarity };
+    } catch (error) {
+      console.error("[NanoCache] Query error:", error);
+      return { hit: false };
+    }
+  }
+  /**
+   * Save a prompt-response pair to the cache
+   */
+  async save(prompt, response, metadata) {
+    try {
+      const embedding = await this.embeddings.generate(prompt);
+      const entry = {
+        prompt,
+        embedding,
+        response,
+        timestamp: Date.now(),
+        metadata
+      };
+      const id = this.hashPrompt(prompt);
+      await this.storage.save(id, entry);
+      if (this.config.debug) {
+        console.log(`[NanoCache] Saved entry for prompt: "${prompt}"`);
+      }
+    } catch (error) {
+      console.error("[NanoCache] Save error:", error);
+      throw error;
+    }
+  }
+  /**
+   * Clear all cached entries
+   */
+  async clear() {
+    await this.storage.clear();
+    if (this.config.debug) {
+      console.log("[NanoCache] Cache cleared");
+    }
+  }
+  /**
+   * Get cache statistics
+   */
+  async getStats() {
+    return await this.storage.getStats();
+  }
+  /**
+   * Check if embedding model is loaded
+   */
+  isModelLoaded() {
+    return this.embeddings.isLoaded();
+  }
+  /**
+   * Preload the embedding model
+   */
+  async preloadModel() {
+    await this.embeddings.generate("warmup");
+    if (this.config.debug) {
+      console.log("[NanoCache] Model preloaded");
+    }
+  }
+  /**
+   * Unload the embedding model to free memory
+   */
+  async unloadModel() {
+    await this.embeddings.unload();
+  }
+  /**
+   * Simple hash function for prompt
+   */
+  hashPrompt(prompt) {
+    let hash = 0;
+    for (let i = 0; i < prompt.length; i++) {
+      const char = prompt.charCodeAt(i);
+      hash = (hash << 5) - hash + char;
+      hash = hash & hash;
+    }
+    return Math.abs(hash).toString(36);
+  }
+  /**
+   * Create a wrapper for OpenAI-compatible chat completion
+   * This allows drop-in replacement of openai.chat.completions.create
+   */
+  createChatWrapper(originalFn) {
+    const self = this;
+    return (async function wrappedCreate(request) {
+      const userMessage = request.messages.filter((m) => m.role === "user").map((m) => m.content).join("\n");
+      if (!userMessage) {
+        return await originalFn(request);
+      }
+      const cacheResult = await self.query(userMessage);
+      if (cacheResult.hit && cacheResult.response) {
+        return {
+          id: `nano-cache-${Date.now()}`,
+          object: "chat.completion",
+          created: Math.floor(Date.now() / 1e3),
+          model: request.model,
+          choices: [
+            {
+              index: 0,
+              message: {
+                role: "assistant",
+                content: cacheResult.response
+              },
+              finish_reason: "stop"
+            }
+          ]
+        };
+      }
+      const response = await originalFn(request);
+      const assistantMessage = response.choices[0]?.message?.content;
+      if (assistantMessage) {
+        await self.save(userMessage, assistantMessage, {
+          model: request.model,
+          timestamp: response.created
+        });
+      }
+      return response;
+    });
+  }
+};
+export {
+  NanoCache,
+  calculateSimilarity,
+  normalizeVector
+};

package/package.json

@@ -0,0 +1,68 @@
+{
+    "name": "@nano-llm-cache/core",
+    "publishConfig": {
+        "access": "public"
+    },
+    "version": "1.0.0",
+    "description": "A semantic cache for LLM API calls using local embeddings and vector similarity",
+    "main": "./dist/index.js",
+    "module": "./dist/index.mjs",
+    "types": "./dist/index.d.ts",
+    "exports": {
+        ".": {
+            "require": "./dist/index.js",
+            "import": "./dist/index.mjs",
+            "types": "./dist/index.d.ts"
+        }
+    },
+    "files": [
+        "dist"
+    ],
+    "scripts": {
+        "build": "tsup src/index.ts --format cjs,esm --dts --clean",
+        "dev": "tsup src/index.ts --format cjs,esm --dts --watch",
+        "test": "vitest",
+        "test:ui": "vitest --ui",
+        "test:coverage": "vitest --coverage",
+        "lint": "eslint src --ext .ts",
+        "prepublishOnly": "npm run build"
+    },
+    "keywords": [
+        "llm",
+        "cache",
+        "semantic",
+        "vector",
+        "embeddings",
+        "openai",
+        "ai",
+        "machine-learning",
+        "transformers"
+    ],
+    "author": "",
+    "license": "MIT",
+    "devDependencies": {
+        "@types/node": "^20.11.5",
+        "@typescript-eslint/eslint-plugin": "^6.19.0",
+        "@typescript-eslint/parser": "^6.19.0",
+        "@vitest/ui": "^1.2.1",
+        "eslint": "^8.56.0",
+        "tsup": "^8.0.1",
+        "typescript": "^5.3.3",
+        "vitest": "^1.2.1"
+    },
+    "dependencies": {
+        "@xenova/transformers": "^2.17.1",
+        "idb-keyval": "^6.2.1"
+    },
+    "peerDependencies": {
+        "openai": "^4.0.0"
+    },
+    "peerDependenciesMeta": {
+        "openai": {
+            "optional": true
+        }
+    },
+    "engines": {
+        "node": ">=18.0.0"
+    }
+}