@o-lang/semantic-doc-search 1.0.33 → 1.0.35

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@o-lang/semantic-doc-search",
-  "version": "1.0.33",
+  "version": "1.0.35",
   "description": "O-lang Semantic Document Search Resolver with hybrid search, embeddings, rerank, and streaming.",
   "main": "src/index.js",
   "type": "commonjs",

package/src/embeddings/local.js

@@ -7,6 +7,8 @@
  * - No zero vectors
  * - Deterministic behavior
  * - DEFENSIVE against method detaching & invalid vectors
+ * - WINDOWS-SAFE (disables SIMD, threads, proxy)
+ * - TENSOR-SAFE (handles Float32Array, Array, and all ONNX tensor types)
  */
 
 class LocalEmbedding {
@@ -29,13 +31,21 @@ class LocalEmbedding {
 
     if (!this.loading) {
       this.loading = (async () => {
-        const { pipeline, env } = await import("@xenova/transformers");
+        // ⚠️ CRITICAL: Configure environment BEFORE loading model
+        const { env } = await import("@xenova/transformers");
 
-        // Safe defaults
+        // Safe settings for all platforms (harmless on macOS/Linux, essential on Windows)
+        env.backends.onnx.wasm.simd = false;      // Avoids AVX/SIMD crashes on older CPUs
+        env.backends.onnx.wasm.threads = false;   // Prevents threading issues in Node
+        env.backends.onnx.wasm.proxy = false;     // Avoids proxy complications
         env.allowLocalModels = true;
         env.backends.onnx.warmup = false;
+        env.cacheDir = "./.cache/embeddings";     // Explicit, project-local cache
 
         console.log("🔄 Loading local embedding model (first run only)...");
+        console.log("⚙️  Using WASM (SIMD disabled) for cross-platform compatibility");
+
+        const { pipeline } = await import("@xenova/transformers");
 
         const model = await pipeline(
           "feature-extraction",
@@ -57,9 +67,6 @@ class LocalEmbedding {
 
   /* ---------------- PUBLIC API ---------------- */
 
-  /**
-   * Generate embedding for a single string
-   */
   async embed(text) {
     if (typeof text !== "string" || !text.trim()) {
       throw new Error("Embedding input must be a non-empty string");
@@ -73,11 +80,45 @@ class LocalEmbedding {
         normalize: true,
       });
 
-      // ✅ Defensive: ensure vector is a valid array
-      const vector = Array.isArray(output?.data) ? Array.from(output.data) : null;
+      // 🔍 DEBUG: Inspect output structure
+      console.log("🔍 Model output type:", typeof output);
+      if (output && typeof output === 'object') {
+        console.log("🔍 Output keys:", Object.keys(output));
+        console.log("🔍 Output dims:", output.dims);
+        console.log("🔍 output.data type:", Object.prototype.toString.call(output.data));
+        console.log("🔍 Is TypedArray?", ArrayBuffer.isView(output.data));
+      }
 
+      // ✅ UNIVERSAL EXTRACTION: handles Float32Array, Array, and all tensor forms
+      let vector = null;
+
+      if (output && output.data !== undefined) {
+        // Handle Float32Array, Uint8Array, etc. (standard in ONNX/WASM)
+        if (ArrayBuffer.isView(output.data)) {
+          vector = Array.from(output.data);
+        }
+        // Handle plain JS array (older backends or CPU mode)
+        else if (Array.isArray(output.data)) {
+          vector = Array.from(output.data);
+        }
+      }
+      // Handle batch output: [tensor]
+      else if (Array.isArray(output) && output[0]?.data !== undefined) {
+        if (ArrayBuffer.isView(output[0].data)) {
+          vector = Array.from(output[0].data);
+        } else if (Array.isArray(output[0].data)) {
+          vector = Array.from(output[0].data);
+        }
+      }
+      // Fallback: raw array (rare)
+      else if (Array.isArray(output)) {
+        vector = output;
+      }
+
+      // Final validation
       if (!Array.isArray(vector) || vector.length !== this.dim) {
-        console.error("❌ Invalid embedding vector returned:", vector);
+        console.error("❌ Invalid embedding vector length:", vector?.length);
+        console.error("❌ First few values:", vector?.slice?.(0, 5));
         throw new Error(`Invalid embedding dimension: ${vector?.length || 0} (expected ${this.dim})`);
       }
 
@@ -85,15 +126,12 @@ class LocalEmbedding {
     } catch (err) {
       console.error(
         `❌ Embedding failed for text: "${text.slice(0, 60)}..."`,
-        err
+        err.message
       );
       throw err;
     }
   }
 
-  /**
-   * Batch embedding (sequential, safe)
-   */
   async embedBatch(texts = []) {
     if (!Array.isArray(texts)) {
       throw new Error("embedBatch expects an array of strings");
@@ -106,15 +144,10 @@ class LocalEmbedding {
     return results;
   }
 
-  /**
-   * Return embedding dimension
-   */
   getDimension() {
     return this.dim;
   }
 }
 
-/* ---------------- SINGLETON EXPORT ---------------- */
-
 const embedder = new LocalEmbedding();
-module.exports = embedder;
+module.exports = embedder;

package/src/resolver.js

@@ -37,6 +37,39 @@ function sanitizeTextForEmbedding(text) {
   return text.replace(/^["']|["']$/g, "").trim();
 }
 
+/**
+ * Load documents from doc_root if provided
+ */
+function loadDocumentsFromContext(context) {
+  if (context.documents && Array.isArray(context.documents)) {
+    return context.documents;
+  }
+
+  if (context.doc_root) {
+    const baseDir = path.resolve(process.cwd(), context.doc_root);
+    if (fs.existsSync(baseDir)) {
+      const files = fs.readdirSync(baseDir).filter(f => f.endsWith('.txt') || f.endsWith('.md'));
+      return files.map(file => ({
+        id: file,
+        content: fs.readFileSync(path.join(baseDir, file), 'utf8'),
+        source: `file:${file}`
+      }));
+    }
+  }
+
+  return [];
+}
+
+/**
+ * Convert any array-like (Float32Array, etc.) to plain JS array
+ */
+function toPlainArray(input) {
+  if (!input) return null;
+  if (Array.isArray(input)) return input;
+  if (ArrayBuffer.isView(input)) return Array.from(input);
+  return null;
+}
+
 /**
  * Semantic Doc Search Resolver
  */
@@ -56,21 +89,24 @@ async function resolver(action, context = {}) {
     console.log("🩺 Vector health:", await vectorStore.health());
   }
 
-  // Ingestion guard cache
-  const cache = loadCache();
-
   // Ensure backend supports search
   if (!vectorStore.supports("vector.search")) {
     throw new Error("Vector backend does not support vector.search");
   }
 
-  // --- Document ingestion (optional) ---
-  if (context.documents && Array.isArray(context.documents)) {
+  // Load documents (from context.documents OR doc_root)
+  const documents = loadDocumentsFromContext(context);
+
+  // Ingestion guard cache
+  const cache = loadCache();
+
+  // --- Document ingestion ---
+  if (documents.length > 0) {
     if (!vectorStore.supports("vector.insert")) {
       throw new Error("Vector backend does not support vector.insert");
     }
 
-    for (const doc of context.documents) {
+    for (const doc of documents) {
       const chunks = doc.chunks || [doc.content];
       for (let i = 0; i < chunks.length; i++) {
         const text = sanitizeTextForEmbedding(chunks[i]);
@@ -79,8 +115,18 @@ async function resolver(action, context = {}) {
         const hash = hashContent(text);
         if (cache[hash]) continue; // Skip already ingested
 
-        const vector = await embedder.embed(text);
-        if (!vector || vector.every(v => v === 0)) continue;
+        const rawVector = await embedder.embed(text);
+        if (!rawVector || rawVector.every(v => v === 0)) {
+          console.warn("⚠️ Skipping chunk with zero embedding:", text.slice(0, 50));
+          continue;
+        }
+
+        // 🔒 Normalize to plain JS array
+        const vector = toPlainArray(rawVector);
+        if (!vector) {
+          console.warn("⚠️ Skipping chunk with invalid embedding type:", typeof rawVector);
+          continue;
+        }
 
         await vectorStore.upsert({
           id: `${doc.id}:${i}`,
@@ -96,12 +142,18 @@ async function resolver(action, context = {}) {
   }
 
   // Embed query & search
-  const queryVector = await embedder.embed(query);
-  if (!queryVector || queryVector.every(v => v === 0)) {
+  const rawQueryVector = await embedder.embed(query);
+  if (!rawQueryVector || rawQueryVector.every(v => v === 0)) {
     console.warn("⚠️ Query embedding invalid");
     return { text: "(Query could not be embedded)", meta: { matches: 0 } };
   }
 
+  // 🔒 Normalize query vector too (for consistency)
+  const queryVector = toPlainArray(rawQueryVector);
+  if (!queryVector) {
+    return { text: "(Query vector invalid)", meta: { matches: 0 } };
+  }
+
   // Top-K + similarity threshold
   const results = await vectorStore.query({
     vector: queryVector,
@@ -112,4 +164,4 @@ async function resolver(action, context = {}) {
   return formatResults(results, query);
 }
 
-module.exports = resolver;
+module.exports = resolver;

package/test-embed.js

@@ -0,0 +1,10 @@
+// test-embed.js
+const embedder = require("./src/embeddings/local");
+
+async function test() {
+  console.log("Model dimension:", embedder.getDimension());
+  const vector = await embedder.embed("hello world");
+  console.log("Embedding result:", vector?.length, vector);
+}
+
+test().catch(console.error);