@o-lang/semantic-doc-search 1.0.40 → 1.0.42

package/src/services/docQA.js

@@ -2,20 +2,22 @@ const VectorRouter = require("../adapters/vectorRouter");
 const embedder = require("../embeddings/local");
 const extractText = require("../utils/extractText");
 const chunkText = require("../utils/chunker");
+const formatResults = require("../utils/formatResults");
 const fs = require("fs");
 const path = require("path");
 
 async function performDocQA(
   query,
   {
-    doc_root,
-    vectorBackend = "pgvector",
+    doc_root = "./docs",
+    vectorBackend = "memory", // 🔥 default to memory like Python fallback
     dimension = 384,
-    migrate_on_demand = false,
     POSTGRES_URL,
+    topK = 5,
     ...config
   } = {}
 ) {
+  // ── Create vector store
   const store = VectorRouter.create({
     backend: vectorBackend,
     dimension,
@@ -25,30 +27,65 @@ async function performDocQA(
 
   const embed = await embedder({ dimension });
 
-  if (migrate_on_demand && doc_root) {
-    for (const file of fs.readdirSync(doc_root)) {
+  // ─────────────────────────────────────────────
+  // 🔥 ALWAYS INGEST (Python parity)
+  // ─────────────────────────────────────────────
+  if (doc_root && fs.existsSync(doc_root)) {
+    const files = fs.readdirSync(doc_root);
+
+    for (const file of files) {
       const fullPath = path.join(doc_root, file);
+
       if (!fs.statSync(fullPath).isFile()) continue;
+      if (!file.endsWith(".txt") && !file.endsWith(".md")) continue;
+
+      try {
+        const text = await extractText(fullPath);
+        if (!text || !text.trim()) continue;
+
+        const chunks = chunkText(text, 500, 50) || [text];
+
+        for (let i = 0; i < chunks.length; i++) {
+          const chunk = chunks[i];
+          if (!chunk.trim()) continue;
 
-      const text = await extractText(fullPath);
-      const chunks = chunkText(text);
-
-      for (let i = 0; i < chunks.length; i++) {
-        await store.upsert({
-          id: `${file}-${i}`,
-          vector: await embed(chunks[i]),
-          content: chunks[i],
-          source: file,
-          metadata: { chunk: i }
-        });
+          try {
+            await store.upsert({
+              id: `${file}:${i}`,
+              vector: await embed(chunk),
+              content: chunk,
+              source: `file:${file}`,
+              metadata: { chunk: i }
+            });
+          } catch (err) {
+            console.warn("⚠️ Chunk failed:", err.message);
+          }
+        }
+
+      } catch (err) {
+        console.error("❌ Failed to process file:", file, err.message);
       }
     }
   }
 
-  const results = await store.query(await embed(query), { topK: 5 });
+  // ─────────────────────────────────────────────
+  // 🔍 SEARCH
+  // ─────────────────────────────────────────────
+  let matches = [];
+
+  try {
+    const queryVector = await embed(query);
+    matches = await store.query(queryVector, { topK });
+  } catch (err) {
+    console.error("❌ Search failed:", err.message);
+  }
 
   if (store.close) await store.close();
-  return results;
+
+  // ─────────────────────────────────────────────
+  // ✅ FORMAT LIKE PYTHON
+  // ─────────────────────────────────────────────
+  return formatResults(matches, query);
 }
 
-module.exports = performDocQA;
+module.exports = performDocQA;

package/src/utils/formatResults.js

@@ -1,15 +1,14 @@
-/**
- * Normalizes vector search results for O-Lang workflows.
- * Returns both structured matches AND a plain .text field for LLM prompts.
- */
-function formatResults(results = [], query) {
-  // ✅ Generate plain text from all matches
-  const text = results.map(r => r.content).join('\n\n');
-  
+function formatResults(results = [], query = "") {
+  const safeResults = Array.isArray(results) ? results : [];
+
+  const text = safeResults.length
+    ? safeResults.map(r => r.content).join('\n\n')
+    : "";
+
   return {
     query,
-    text, // ← THIS IS THE KEY ADDITION
-    matches: results.map(r => ({
+    text,
+    matches: safeResults.map(r => ({
       id: r.id,
       content: r.content,
       source: r.source,

package/.cache/embeddings/Xenova/all-MiniLM-L6-v2/config.json

@@ -1,25 +0,0 @@
-{
-  "_name_or_path": "sentence-transformers/all-MiniLM-L6-v2",
-  "architectures": [
-    "BertModel"
-  ],
-  "attention_probs_dropout_prob": 0.1,
-  "classifier_dropout": null,
-  "gradient_checkpointing": false,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 384,
-  "initializer_range": 0.02,
-  "intermediate_size": 1536,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 512,
-  "model_type": "bert",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 6,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "transformers_version": "4.29.2",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 30522
-}