@o-lang/semantic-doc-search 1.0.12 → 1.0.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@o-lang/semantic-doc-search",
-  "version": "1.0.12",
+  "version": "1.0.13",
   "description": "O-lang Semantic Document Search Resolver with hybrid search, embeddings, rerank, and streaming.",
   "main": "src/index.js",
   "type": "commonjs",

package/src/embeddings/local.js

@@ -1,53 +1,151 @@
 // src/embeddings/local.js
-const crypto = require("crypto");
+const { pipeline, env } = require('@xenova/transformers');
+
+// Configure transformers to work in Node.js
+env.allowLocalModels = true;
+env.backends.onnx.warmup = false; // Faster startup
 
 /**
- * LocalEmbedding
- * Generates deterministic "fake" embeddings for offline testing or fallback.
- * Each string will produce a consistent vector based on a hash.
- * Note: Not semantic, just a placeholder for testing.
+ * LocalEmbedding - REAL semantic embeddings using all-MiniLM-L6-v2
+ * This is a compact, high-quality sentence transformer that:
+ * - Understands semantic meaning of text
+ * - Produces embeddings with negative and positive values (-1 to 1)
+ * - Works offline after first download
+ * - Is optimized for CPU (no GPU required)
+ * - Produces 384-dimensional vectors compatible with pgvector
  */
 class LocalEmbedding {
-  constructor(dim = 512) {
-    this.dim = dim;
+  constructor() {
+    this.dim = 384; // all-MiniLM-L6-v2 output dimension
+    this.modelPromise = null;
+    this.isModelLoading = false;
   }
 
   /**
-   * Convert text → deterministic pseudo-vector
+   * Lazy-load the embedding model (only loads when first needed)
    */
-  embed(text) {
-    if (!text || !text.trim()) return new Array(this.dim).fill(0);
+  async getModel() {
+    if (!this.modelPromise) {
+      this.isModelLoading = true;
+      console.log('🔄 Loading local embedding model (first run may take 1-2 minutes)...');
+      
+      this.modelPromise = pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', {
+        revision: 'main',
+        cache_dir: './.cache/embeddings' // Cache model locally
+      }).then(model => {
+        console.log('✅ Local embedding model loaded successfully!');
+        this.isModelLoading = false;
+        return model;
+      }).catch(error => {
+        console.error('❌ Failed to load local embedding model:', error.message);
+        this.isModelLoading = false;
+        throw error;
+      });
+    }
+    return this.modelPromise;
+  }
 
-    const hash = crypto.createHash("sha256").update(text).digest();
-    const vector = [];
+  /**
+   * Generate REAL semantic embedding for text
+   * @param {string} text - Input text to embed
+   * @returns {number[]} - 384-dimensional embedding vector with values typically between -1 and 1
+   */
+  async embed(text) {
+    if (!text || !text.trim()) {
+      // Return zero vector for empty text
+      return new Array(this.dim).fill(0);
+    }
 
-    for (let i = 0; i < this.dim; i++) {
-      vector.push(hash[i % hash.length] / 255); // normalize 0–1
+    try {
+      const model = await this.getModel();
+      
+      // Generate embedding with mean pooling and normalization
+      // This matches the standard sentence-transformers approach
+      const output = await model(text, { 
+        pooling: 'mean', 
+        normalize: true 
+      });
+      
+      // Convert Float32Array to regular array
+      const embedding = Array.from(output.data);
+      
+      // Verify dimension
+      if (embedding.length !== this.dim) {
+        console.warn(`⚠️ Expected ${this.dim} dimensions, got ${embedding.length}`);
+        // Pad or truncate to correct dimension
+        if (embedding.length < this.dim) {
+          return [...embedding, ...new Array(this.dim - embedding.length).fill(0)];
+        } else {
+          return embedding.slice(0, this.dim);
+        }
+      }
+      
+      return embedding;
+      
+    } catch (error) {
+      console.error(`❌ Embedding generation failed for text: "${text.substring(0, 50)}..."`);
+      console.error('Error:', error.message);
+      
+      // Fallback to zero vector to prevent complete failure
+      return new Array(this.dim).fill(0);
     }
+  }
 
-    return vector;
+  /**
+   * Batch embedding for multiple strings (processed sequentially to manage memory)
+   */
+  async embedBatch(textArray = []) {
+    if (!Array.isArray(textArray)) {
+      throw new Error("embedBatch expects an array of strings");
+    }
+    
+    const embeddings = [];
+    for (const text of textArray) {
+      const embedding = await this.embed(text);
+      embeddings.push(embedding);
+    }
+    return embeddings;
   }
 
   /**
-   * Batch embedding for multiple strings
+   * Get embedding dimension
    */
-  embedBatch(textArray = []) {
-    if (!Array.isArray(textArray)) throw new Error("embedBatch expects an array");
-    return textArray.map(text => this.embed(text));
+  getDimension() {
+    return this.dim;
   }
 }
 
 /**
- * Convenience function for index.js
- * Retries local embedding generation (mostly placeholder, but keeps API compatible)
+ * Convenience function for compatibility with existing code
+ * Creates embedding with retry logic
  */
-async function createEmbeddingWithRetry(text, options = {}, retries = 1) {
+async function createEmbeddingWithRetry(text, options = {}, retries = 2) {
   const embedder = new LocalEmbedding();
+  
   for (let attempt = 1; attempt <= retries; attempt++) {
     try {
-      return embedder.embed(text);
+      const embedding = await embedder.embed(text);
+      
+      // Verify embedding is valid (not all zeros)
+      const isAllZeros = embedding.every(val => val === 0);
+      if (isAllZeros && (text || '').trim()) {
+        if (attempt === retries) {
+          console.warn(`⚠️ Embedding is all zeros for text: "${text.substring(0, 50)}..."`);
+        }
+        throw new Error('Embedding returned all zeros');
+      }
+      
+      return embedding;
+      
     } catch (err) {
-      if (attempt === retries) throw err;
+      if (attempt === retries) {
+        console.error(`❌ All ${retries} attempts failed for embedding text: "${text.substring(0, 50)}..."`);
+        throw err;
+      }
+      
+      console.warn(`⚠️ Embedding attempt ${attempt} failed, retrying...`);
+      // Add small delay before retry
+      await new Promise(resolve => setTimeout(resolve, 100 * attempt));
     }
   }
 }