@o-lang/semantic-doc-search 1.0.18 → 1.0.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@o-lang/semantic-doc-search",
-  "version": "1.0.18",
+  "version": "1.0.21",
   "description": "O-lang Semantic Document Search Resolver with hybrid search, embeddings, rerank, and streaming.",
   "main": "src/index.js",
   "type": "commonjs",

package/src/adapters/vectorRouter.js

@@ -1,31 +1,30 @@
-const PgVectorAdapter = require("./pgvectorAdapter");
-const PineconeAdapter = require("./pineconeAdapter");
-const RedisAdapter = require("./redisAdapter");
-const InMemoryAdapter = require("./inMemoryAdapter");
-
 class VectorRouter {
   static create(config = {}) {
-    const backend = config.backend;
-
-    if (!backend) {
-      throw new Error("Vector backend not specified");
-    }
+    const backend = config.backend || "pgvector";
 
     switch (backend) {
-      case "pgvector":
+      case "pgvector": {
+        const PgVectorAdapter = require("./pgvectorAdapter");
         return new PgVectorAdapter(config);
+      }
 
-      case "pinecone":
-        return new PineconeAdapter(config);
+      case "memory": {
+        const InMemoryAdapter = require("./inMemoryAdapter");
+        return new InMemoryAdapter(config);
+      }
 
-      case "redis":
+      case "redis": {
+        const RedisAdapter = require("./redisAdapter");
         return new RedisAdapter(config);
+      }
 
-      case "memory":
-        return new InMemoryAdapter(config);
+      case "pinecone": {
+        const PineconeAdapter = require("./pineconeAdapter");
+        return new PineconeAdapter(config);
+      }
 
       default:
-        throw new Error(`Unsupported vector backend: ${backend}`);
+        throw new Error(`Unknown vector backend: ${backend}`);
     }
   }
 }

package/src/index.js

@@ -1,4 +1,3 @@
-// doc-search.js
 const fs = require("fs");
 const path = require("path");
 const { createLLM } = require("./llm/router.js");
@@ -20,23 +19,36 @@ function safeResolve(base, userPath) {
 
 function loadCache() {
   try {
-    if (fs.existsSync(CACHE_PATH)) return JSON.parse(fs.readFileSync(CACHE_PATH, "utf8")) || {};
+    if (fs.existsSync(CACHE_PATH)) {
+      return JSON.parse(fs.readFileSync(CACHE_PATH, "utf8")) || {};
+    }
   } catch {}
   return {};
 }
 
 function saveCache(cache) {
-  try { fs.writeFileSync(CACHE_PATH, JSON.stringify(cache, null, 2)); } catch {}
+  try {
+    fs.writeFileSync(CACHE_PATH, JSON.stringify(cache, null, 2));
+  } catch {}
 }
 
 // ------------------- DATABASE ADAPTER -------------------
 class DatabaseAdapter {
-  constructor() { this.initialized = false; }
+  constructor() {
+    this.initialized = false;
+  }
+
   async initialize(context) {
     if (this.initialized) return;
-    if (context.db_type === "mongodb" || context.MONGO_URI) await this.initMongo(context);
-    else if (context.db_type === "sqlite" || context.db_path) await this.initSQLite(context);
-    else if (context.db_type === "postgres" || context.POSTGRES_URL) await this.initPostgres(context);
+
+    if (context.db_type === "mongodb" || context.MONGO_URI) {
+      await this.initMongo(context);
+    } else if (context.db_type === "sqlite" || context.db_path) {
+      await this.initSQLite(context);
+    } else if (context.db_type === "postgres" || context.POSTGRES_URL) {
+      await this.initPostgres(context);
+    }
+
     this.initialized = true;
   }
 
@@ -63,8 +75,8 @@ class DatabaseAdapter {
 
   async queryMongo(collectionName, filter = {}, projection = {}) {
     if (!this.mongoClient) throw new Error("MongoDB client not initialized");
-    const db = this.mongoClient.db(process.env.DB_NAME || context.db_name || "olang");
-    return await db.collection(collectionName).find(filter, { projection }).toArray();
+    const db = this.mongoClient.db(process.env.DB_NAME || "olang");
+    return db.collection(collectionName).find(filter, { projection }).toArray();
   }
 
   async initPostgres(context) {
@@ -78,7 +90,7 @@ class DatabaseAdapter {
       database: context.DB_NAME || "olang",
     };
     Object.keys(poolConfig).forEach((k) => {
-      if (poolConfig[k] === undefined || poolConfig[k] === null) delete poolConfig[k];
+      if (poolConfig[k] == null) delete poolConfig[k];
     });
     this.postgresClient = new Pool(poolConfig);
   }
@@ -90,16 +102,24 @@ class DatabaseAdapter {
   }
 
   async queryDocuments(context) {
-    const { db_type, db_table = "documents", db_content_column = "content", db_id_column = "id" } = context;
+    const {
+      db_type,
+      db_table = "documents",
+      db_content_column = "content",
+      db_id_column = "id",
+    } = context;
+
     if (db_type === "mongodb" || context.MONGO_URI) {
       const { filter, projection } = this.buildMongoQuery(context);
       const results = await this.queryMongo(db_table, filter, projection);
       return results.map((doc) => ({
-        id: doc._id?.toString() || doc.id || doc[db_id_column],
-        content: doc[db_content_column] || doc.content || doc.text || "",
+        id: doc._id?.toString() || doc[db_id_column],
+        content: doc[db_content_column] || "",
         source: `mongodb:${db_table}`,
       }));
-    } else if (db_type === "sqlite" || context.db_path) {
+    }
+
+    if (db_type === "sqlite" || context.db_path) {
       const { sql, params } = this.buildSqlQuery(context);
       const results = await this.querySQLite(sql, params);
       return results.map((row) => ({
@@ -107,7 +127,9 @@ class DatabaseAdapter {
         content: row[db_content_column],
         source: `sqlite:${db_table}`,
       }));
-    } else if (db_type === "postgres" || context.POSTGRES_URL) {
+    }
+
+    if (db_type === "postgres" || context.POSTGRES_URL) {
       const { sql, params } = this.buildSqlQuery(context);
       const results = await this.queryPostgres(sql, params);
       return results.map((row) => ({
@@ -116,111 +138,131 @@ class DatabaseAdapter {
         source: `postgres:${db_table}`,
       }));
     }
+
     return [];
   }
 
   buildMongoQuery(context) {
-    const { doc_filter = {}, doc_projection = {} } = context;
     let filter = {};
-    if (typeof doc_filter === "string") {
-      try { filter = JSON.parse(doc_filter); } catch { filter = { $text: { $search: doc_filter } }; }
-    } else if (typeof doc_filter === "object" && Object.keys(doc_filter).length > 0) filter = doc_filter;
-    const projection = typeof doc_projection === "string" ? JSON.parse(doc_projection) : doc_projection;
-    return { filter, projection };
+    if (typeof context.doc_filter === "string") {
+      try {
+        filter = JSON.parse(context.doc_filter);
+      } catch {
+        filter = { $text: { $search: context.doc_filter } };
+      }
+    }
+    return { filter, projection: {} };
   }
 
   buildSqlQuery(context) {
-    const { db_content_column = "content", db_id_column = "id", doc_where = "1=1", doc_params = [] } = context;
-    let params = doc_params;
-    if (typeof doc_params === "string") {
-      try { params = JSON.parse(doc_params); } catch { params = [doc_params]; }
-    }
     const table = context.db_table || "documents";
-    const sql = `SELECT ${db_id_column}, ${db_content_column} FROM ${table} WHERE ${doc_where}`;
-    return { sql, params };
-  }
-
-  async close() {
-    if (this.sqliteClient) { try { this.sqliteClient.close(); } catch {} this.sqliteClient = null; }
-    if (this.mongoClient) { try { await this.mongoClient.close(); } catch {} this.mongoClient = null; }
-    if (this.postgresClient) { try { await this.postgresClient.end(); } catch {} this.postgresClient = null; }
-    this.initialized = false;
+    const where = context.doc_where || "1=1";
+    return {
+      sql: `SELECT * FROM ${table} WHERE ${where}`,
+      params: [],
+    };
   }
 }
 
 // ------------------- DOCUMENT LOADING -------------------
-async function loadDocumentsFromDatabase(context) {
-  if (!context.db_type && !context.db_path && !context.MONGO_URI && !context.POSTGRES_URL) return null;
-  const dbAdapter = new DatabaseAdapter();
-  try { await dbAdapter.initialize(context); return await dbAdapter.queryDocuments(context); } catch (e) { console.error("🗃️ [doc-search] Database load error:", e.message); return null; }
-}
-
 async function loadAllDocuments(context) {
-  const documents = [];
-  const dbDocs = await loadDocumentsFromDatabase(context);
-  if (dbDocs) documents.push(...dbDocs);
+  const docs = [];
+  const db = new DatabaseAdapter();
+
+  try {
+    await db.initialize(context);
+    docs.push(...(await db.queryDocuments(context)));
+  } catch {}
+
+  const baseDir = context.doc_root
+    ? safeResolve(process.cwd(), context.doc_root)
+    : path.join(process.cwd(), "docs");
 
-  const baseDir = context.doc_root ? safeResolve(process.cwd(), context.doc_root) : path.join(process.cwd(), "docs");
   if (fs.existsSync(baseDir)) {
     const files = fs.readdirSync(baseDir).filter((f) => f.endsWith(".txt") || f.endsWith(".md"));
-    for (const file of files) {
-      try {
-        const content = fs.readFileSync(path.join(baseDir, file), "utf8");
-        documents.push({ id: file, content, source: `file:${file}` });
-      } catch (e) { console.warn(`⚠️ [doc-search] Failed to read file ${file}: ${e.message}`); }
+    for (const f of files) {
+      const content = fs.readFileSync(path.join(baseDir, f), "utf8");
+      docs.push({ id: f, content, source: `file:${f}` });
     }
   }
-  return documents;
-}
 
-// ------------------- VECTOR MIGRATION -------------------
-async function checkPgVectorHasData(pgVectorAdapter) {
-  try { const result = await pgVectorAdapter.pool.query("SELECT COUNT(*) FROM doc_embeddings"); return parseInt(result.rows[0].count) > 0; } catch { return false; }
+  return docs;
 }
 
-async function migrateDocumentsToPgVector(docRoot, pgVectorAdapter, embedder) {
-  const baseDir = safeResolve(process.cwd(), docRoot);
-  if (!fs.existsSync(baseDir)) { console.log("📁 No docs directory found, skipping migration"); return; }
-  const files = fs.readdirSync(baseDir).filter((f) => f.endsWith(".txt") || f.endsWith(".md"));
-  console.log(`🔄 Migrating ${files.length} documents to pgvector...`);
-  for (const file of files) {
-    try {
-      const content = fs.readFileSync(path.join(baseDir, file), "utf8");
-      const vector = await embedder.embed(content);
-      await pgVectorAdapter.upsert({ id: file, vector, content, source: `file:${file}` });
-      console.log(`✅ Migrated ${file}`);
-    } catch (e) { console.warn(`⚠️ Failed to migrate ${file}: ${e.message}`); }
+// ------------------- HYBRID SEARCH (FIXED & DEFINED) -------------------
+async function performHybridDocQA(query, context = {}) {
+  const cache = loadCache();
+  const embedder = new LocalEmbedding({ dimension: 384 });
+
+  const router = VectorRouter.create({
+    backend: context.vectorBackend || "memory",
+    dimension: 384,
+    ...context,
+  });
+
+  const documents = await loadAllDocuments(context);
+  if (!documents.length) return { text: "", meta: {} };
+
+  for (const doc of documents) {
+    if (!cache[doc.id]) {
+      const chunks = chunkText(doc.content, 500);
+      cache[doc.id] = [];
+      for (const chunk of chunks) {
+        const vector = await embedder.embed(chunk);
+        await router.upsert({
+          id: `${doc.id}:${cache[doc.id].length}`,
+          vector,
+          text: chunk,
+          source: doc.source,
+        });
+        cache[doc.id].push(vector);
+      }
+    }
   }
+
+  saveCache(cache);
+
+  const queryVector = await embedder.embed(query);
+  const results = await router.search({ embedding: queryVector, topK: 5 });
+
+  return {
+    text: highlightMatches(
+      results.map((r) => r.text).join("\n\n"),
+      extractKeywords(query)
+    ),
+    meta: { matches: results.length },
+  };
 }
 
-// ------------------- VECTOR SEARCH (AUTO SWITCH) -------------------
-async function performVectorQA(query, context = {}) {
-  const postgresUrl = context.POSTGRES_URL || process.env.POSTGRES_URL;
-  const vectorBackend = context.vectorBackend;
-
-  if (postgresUrl) {
-    return await performPgVectorSearch(query, context);
-  } else if (vectorBackend) {
-    return await performVectorSearch(query, context);
-  } else {
-    return await performHybridDocQA(query, context);
-  }
+// ------------------- PGVECTOR SEARCH -------------------
+async function performPgVectorSearch(query, context = {}) {
+  const adapter = new PgVectorAdapter({ POSTGRES_URL: context.POSTGRES_URL });
+  const embedder = new LocalEmbedding({ dimension: 384 });
+  const vector = await embedder.embed(query);
+  const results = await adapter.search(vector, 5);
+  return {
+    text: results.map((r) => r.content).join("\n\n"),
+    meta: { matches: results.length },
+  };
 }
 
-// ------------------- HYBRID + VECTOR SEARCH FUNCTIONS -------------------
-// [Keep performPgVectorSearch, performHybridDocQA, loadAllDocuments, chunking, cache logic identical to previous full file]
+// ------------------- ROUTER -------------------
+async function performVectorQA(query, context = {}) {
+  if (context.POSTGRES_URL) return performPgVectorSearch(query, context);
+  return performHybridDocQA(query, context);
+}
 
 async function performDocQA(query, context = {}) {
-  return await performVectorQA(query, context);
+  return performVectorQA(query, context);
 }
 
+// ------------------- RESOLVER -------------------
 async function docSearchResolver(action, context) {
-  if (action.startsWith("Ask doc-search ")) {
+  if (action.startsWith("Ask doc-search")) {
     const match = action.match(/"(.*)"|'(.*)'/);
-    const query = match ? match[1] || match[2] : action.replace(/^Ask doc-search\s+/, "").trim();
-    return await performDocQA(query, context);
+    const query = match ? match[1] || match[2] : action.replace("Ask doc-search", "").trim();
+    return performDocQA(query, context);
   }
-  return undefined;
 }
 
 docSearchResolver.resolverName = "doc-search";