@o-lang/semantic-doc-search 1.0.41 → 1.0.43

package/package.json

@@ -1,38 +1,40 @@
-{
-  "name": "@o-lang/semantic-doc-search",
-  "version": "1.0.41",
-  "description": "O-lang Semantic Document Search Resolver with hybrid search, embeddings, rerank, and streaming.",
-  "main": "src/index.js",
-  "type": "commonjs",
-  "bin": {
-    "olang-doc-search": "bin/cli.js"
-  },
-  "scripts": {
-    "start": "node bin/cli.js",
-    "test": "echo \"No tests yet\""
-  },
-  "dependencies": {
-    "@anthropic-ai/sdk": "*",
-    "@xenova/transformers": "^2.17.2",
-    "axios": "^1.7.2",
-    "docx": "^7.0.0",
-    "dotenv": "^17.2.3",
-    "express": "^4.18.2",
-    "groq-sdk": "^0.5.0",
-    "jsdom": "^22.1.0",
-    "minimist": "^1.2.8",
-    "node-stream-zip": "*",
-    "openai": "^4.3.1",
-    "pdf-parse": "^1.1.1",
-    "pg": "^8.16.3",
-    "pgvector": "^0.2.1",
-    "pinecone-client": "^1.0.0",
-    "readline": "^1.3.0",
-    "redis": "^5.2.0"
-  },
-  "devDependencies": {
-    "eslint": "^8.46.0",
-    "jest": "^29.7.0",
-    "nodemon": "^2.0.22"
-  }
-}
+{
+  "name": "@o-lang/semantic-doc-search",
+  "version": "1.0.43",
+  "description": "O-Lang semantic document search resolver with vector embeddings",
+  "main": "src/index.js",
+  "exports": {
+    ".": "./src/index.js",
+    "./resolver": "./src/resolver.js",
+    "./embeddings/local": "./src/embeddings/local.js"
+  },
+  "files": [
+    "src/",
+    "package.json",
+    "README.md"
+  ],
+  "scripts": {
+    "test": "node src/test-doc-search.js",
+    "start": "node src/index.js"
+  },
+  "keywords": [
+    "o-lang",
+    "resolver",
+    "semantic-search",
+    "rag",
+    "embeddings"
+  ],
+  "author": "O-Lang Team <info@olang.cloud>",
+  "license": "MIT",
+  "dependencies": {
+    "@pinecone-database/pinecone": "^2.2.2",
+    "@xenova/transformers": "^2.14.0",
+    "axios": "^1.6.0",
+    "dotenv": "^16.6.1",
+    "pg": "^8.20.0",
+    "redis": "^4.7.1"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

package/src/embeddings/local.js

@@ -149,5 +149,15 @@ class LocalEmbedding {
   }
 }
 
-const embedder = new LocalEmbedding();
-module.exports = embedder;
+// ✅ EXPORT AS FACTORY FUNCTION (what resolver expects)
+// Usage: const embed = await embedder({ dimension: 384 })
+// Returns: async (text) => vector
+const embedderInstance = new LocalEmbedding();
+
+module.exports = async ({ dimension = 384 } = {}) => {
+  if (dimension && typeof dimension === 'number') {
+    embedderInstance.dim = dimension;
+  }
+  // Return bound embed method that resolver can call: await embed(text)
+  return embedderInstance.embed.bind(embedderInstance);
+};

package/src/embeddings/local.js.bak

@@ -0,0 +1,153 @@
+/**
+ * LocalEmbedding
+ * ----------------
+ * Real semantic embeddings using all-MiniLM-L6-v2
+ * - Singleton model load
+ * - No silent failures
+ * - No zero vectors
+ * - Deterministic behavior
+ * - DEFENSIVE against method detaching & invalid vectors
+ * - WINDOWS-SAFE (disables SIMD, threads, proxy)
+ * - TENSOR-SAFE (handles Float32Array, Array, and all ONNX tensor types)
+ */
+
+class LocalEmbedding {
+  constructor() {
+    this.dim = 384;
+    this.model = null;
+    this.loading = null;
+
+    // 🔒 Bind methods to prevent resolver breakage
+    this.loadModel = this.loadModel.bind(this);
+    this.embed = this.embed.bind(this);
+    this.embedBatch = this.embedBatch.bind(this);
+    this.getDimension = this.getDimension.bind(this);
+  }
+
+  /* ---------------- INTERNAL ---------------- */
+
+  async loadModel() {
+    if (this.model) return this.model;
+
+    if (!this.loading) {
+      this.loading = (async () => {
+        // ⚠️ CRITICAL: Configure environment BEFORE loading model
+        const { env } = await import("@xenova/transformers");
+
+        // Safe settings for all platforms (harmless on macOS/Linux, essential on Windows)
+        env.backends.onnx.wasm.simd = false;      // Avoids AVX/SIMD crashes on older CPUs
+        env.backends.onnx.wasm.threads = false;   // Prevents threading issues in Node
+        env.backends.onnx.wasm.proxy = false;     // Avoids proxy complications
+        env.allowLocalModels = true;
+        env.backends.onnx.warmup = false;
+        env.cacheDir = "./.cache/embeddings";     // Explicit, project-local cache
+
+        console.log("🔄 Loading local embedding model (first run only)...");
+        console.log("⚙️  Using WASM (SIMD disabled) for cross-platform compatibility");
+
+        const { pipeline } = await import("@xenova/transformers");
+
+        const model = await pipeline(
+          "feature-extraction",
+          "Xenova/all-MiniLM-L6-v2",
+          {
+            revision: "main",
+            cache_dir: "./.cache/embeddings",
+          }
+        );
+
+        console.log("✅ Local embedding model ready");
+        return model;
+      })();
+    }
+
+    this.model = await this.loading;
+    return this.model;
+  }
+
+  /* ---------------- PUBLIC API ---------------- */
+
+  async embed(text) {
+    if (typeof text !== "string" || !text.trim()) {
+      throw new Error("Embedding input must be a non-empty string");
+    }
+
+    const model = await this.loadModel();
+
+    try {
+      const output = await model(text, {
+        pooling: "mean",
+        normalize: true,
+      });
+
+      // 🔍 DEBUG: Inspect output structure
+      console.log("🔍 Model output type:", typeof output);
+      if (output && typeof output === 'object') {
+        console.log("🔍 Output keys:", Object.keys(output));
+        console.log("🔍 Output dims:", output.dims);
+        console.log("🔍 output.data type:", Object.prototype.toString.call(output.data));
+        console.log("🔍 Is TypedArray?", ArrayBuffer.isView(output.data));
+      }
+
+      // ✅ UNIVERSAL EXTRACTION: handles Float32Array, Array, and all tensor forms
+      let vector = null;
+
+      if (output && output.data !== undefined) {
+        // Handle Float32Array, Uint8Array, etc. (standard in ONNX/WASM)
+        if (ArrayBuffer.isView(output.data)) {
+          vector = Array.from(output.data);
+        }
+        // Handle plain JS array (older backends or CPU mode)
+        else if (Array.isArray(output.data)) {
+          vector = Array.from(output.data);
+        }
+      }
+      // Handle batch output: [tensor]
+      else if (Array.isArray(output) && output[0]?.data !== undefined) {
+        if (ArrayBuffer.isView(output[0].data)) {
+          vector = Array.from(output[0].data);
+        } else if (Array.isArray(output[0].data)) {
+          vector = Array.from(output[0].data);
+        }
+      }
+      // Fallback: raw array (rare)
+      else if (Array.isArray(output)) {
+        vector = output;
+      }
+
+      // Final validation
+      if (!Array.isArray(vector) || vector.length !== this.dim) {
+        console.error("❌ Invalid embedding vector length:", vector?.length);
+        console.error("❌ First few values:", vector?.slice?.(0, 5));
+        throw new Error(`Invalid embedding dimension: ${vector?.length || 0} (expected ${this.dim})`);
+      }
+
+      return vector;
+    } catch (err) {
+      console.error(
+        `❌ Embedding failed for text: "${text.slice(0, 60)}..."`,
+        err.message
+      );
+      throw err;
+    }
+  }
+
+  async embedBatch(texts = []) {
+    if (!Array.isArray(texts)) {
+      throw new Error("embedBatch expects an array of strings");
+    }
+
+    const results = [];
+    for (const text of texts) {
+      results.push(await this.embed(text));
+    }
+    return results;
+  }
+
+  getDimension() {
+    return this.dim;
+  }
+}
+
+const embedder = new LocalEmbedding();
+module.exports = embedder;

package/src/index.js

@@ -1,4 +1,3 @@
-// index.js (6 lines)
 const semanticResolver = require("./resolver");
 
 async function docSearchResolver(action, context) {
@@ -6,4 +5,5 @@ async function docSearchResolver(action, context) {
 }
 
 docSearchResolver.resolverName = "doc-search";
+docSearchResolver.version = "1.0.41";
 module.exports = docSearchResolver;

package/.cache/embeddings/Xenova/all-MiniLM-L6-v2/config.json

@@ -1,25 +0,0 @@
-{
-  "_name_or_path": "sentence-transformers/all-MiniLM-L6-v2",
-  "architectures": [
-    "BertModel"
-  ],
-  "attention_probs_dropout_prob": 0.1,
-  "classifier_dropout": null,
-  "gradient_checkpointing": false,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 384,
-  "initializer_range": 0.02,
-  "intermediate_size": 1536,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 512,
-  "model_type": "bert",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 6,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "transformers_version": "4.29.2",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 30522
-}