@o-lang/semantic-doc-search 1.0.20 → 1.0.22

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@o-lang/semantic-doc-search",
-  "version": "1.0.20",
+  "version": "1.0.22",
   "description": "O-lang Semantic Document Search Resolver with hybrid search, embeddings, rerank, and streaming.",
   "main": "src/index.js",
   "type": "commonjs",

package/src/index.js

@@ -1,4 +1,3 @@
-// doc-search.js
 const fs = require("fs");
 const path = require("path");
 const { createLLM } = require("./llm/router.js");
@@ -14,29 +13,45 @@ const CACHE_PATH = path.join(process.cwd(), "embeddings.json");
 
 function safeResolve(base, userPath) {
   const resolved = path.resolve(base, userPath);
-  if (!resolved.startsWith(path.resolve(base))) throw new Error("Path traversal detected");
+  if (!resolved.startsWith(path.resolve(base))) {
+    throw new Error("Path traversal detected");
+  }
   return resolved;
 }
 
 function loadCache() {
   try {
-    if (fs.existsSync(CACHE_PATH)) return JSON.parse(fs.readFileSync(CACHE_PATH, "utf8")) || {};
+    if (fs.existsSync(CACHE_PATH)) {
+      return JSON.parse(fs.readFileSync(CACHE_PATH, "utf8")) || {};
+    }
   } catch {}
   return {};
 }
 
 function saveCache(cache) {
-  try { fs.writeFileSync(CACHE_PATH, JSON.stringify(cache, null, 2)); } catch {}
+  try {
+    fs.writeFileSync(CACHE_PATH, JSON.stringify(cache, null, 2));
+  } catch {}
 }
 
-// ------------------- DATABASE ADAPTER -------------------
+/* ---------------- DATABASE ADAPTER ---------------- */
+
 class DatabaseAdapter {
-  constructor() { this.initialized = false; }
+  constructor() {
+    this.initialized = false;
+  }
+
   async initialize(context) {
     if (this.initialized) return;
-    if (context.db_type === "mongodb" || context.MONGO_URI) await this.initMongo(context);
-    else if (context.db_type === "sqlite" || context.db_path) await this.initSQLite(context);
-    else if (context.db_type === "postgres" || context.POSTGRES_URL) await this.initPostgres(context);
+
+    if (context.db_type === "mongodb" || context.MONGO_URI) {
+      await this.initMongo(context);
+    } else if (context.db_type === "sqlite" || context.db_path) {
+      await this.initSQLite(context);
+    } else if (context.db_type === "postgres" || context.POSTGRES_URL) {
+      await this.initPostgres(context);
+    }
+
     this.initialized = true;
   }
 
@@ -44,183 +59,202 @@ class DatabaseAdapter {
     const Database = require("better-sqlite3");
     const dbPath = context.db_path || "./database.db";
     const dbDir = path.dirname(path.resolve(dbPath));
-    if (!fs.existsSync(dbDir)) throw new Error(`SQLite database directory not found: ${dbDir}`);
+    if (!fs.existsSync(dbDir)) {
+      throw new Error(`SQLite database directory not found: ${dbDir}`);
+    }
     this.sqliteClient = new Database(dbPath, { readonly: true });
   }
 
   async querySQLite(query, params = []) {
-    if (!this.sqliteClient) throw new Error("SQLite client not initialized");
     const stmt = this.sqliteClient.prepare(query);
     return stmt.all(...params);
   }
 
   async initMongo(context) {
     const { MongoClient } = require("mongodb");
-    const uri = context.MONGO_URI || `mongodb://localhost:27017/${context.db_name || "olang"}`;
+    const uri =
+      context.MONGO_URI ||
+      `mongodb://localhost:27017/${context.db_name || "olang"}`;
     this.mongoClient = new MongoClient(uri);
     await this.mongoClient.connect();
   }
 
   async queryMongo(collectionName, filter = {}, projection = {}) {
-    if (!this.mongoClient) throw new Error("MongoDB client not initialized");
-    const db = this.mongoClient.db(process.env.DB_NAME || context.db_name || "olang");
-    return await db.collection(collectionName).find(filter, { projection }).toArray();
+    const db = this.mongoClient.db(process.env.DB_NAME || "olang");
+    return db.collection(collectionName).find(filter, { projection }).toArray();
   }
 
   async initPostgres(context) {
     const { Pool } = require("pg");
-    const poolConfig = {
+    this.postgresClient = new Pool({
       connectionString: context.POSTGRES_URL,
-      host: context.DB_HOST || "localhost",
-      port: parseInt(context.DB_PORT) || 5432,
-      user: context.DB_USER,
-      password: context.DB_PASSWORD,
-      database: context.DB_NAME || "olang",
-    };
-    Object.keys(poolConfig).forEach((k) => {
-      if (poolConfig[k] === undefined || poolConfig[k] === null) delete poolConfig[k];
     });
-    this.postgresClient = new Pool(poolConfig);
   }
 
   async queryPostgres(query, params = []) {
-    if (!this.postgresClient) throw new Error("PostgreSQL client not initialized");
     const result = await this.postgresClient.query(query, params);
     return result.rows;
   }
 
   async queryDocuments(context) {
-    const { db_type, db_table = "documents", db_content_column = "content", db_id_column = "id" } = context;
-    if (db_type === "mongodb" || context.MONGO_URI) {
-      const { filter, projection } = this.buildMongoQuery(context);
-      const results = await this.queryMongo(db_table, filter, projection);
-      return results.map((doc) => ({
-        id: doc._id?.toString() || doc.id || doc[db_id_column],
-        content: doc[db_content_column] || doc.content || doc.text || "",
-        source: `mongodb:${db_table}`,
-      }));
-    } else if (db_type === "sqlite" || context.db_path) {
-      const { sql, params } = this.buildSqlQuery(context);
-      const results = await this.querySQLite(sql, params);
-      return results.map((row) => ({
-        id: row[db_id_column],
-        content: row[db_content_column],
-        source: `sqlite:${db_table}`,
-      }));
-    } else if (db_type === "postgres" || context.POSTGRES_URL) {
-      const { sql, params } = this.buildSqlQuery(context);
-      const results = await this.queryPostgres(sql, params);
-      return results.map((row) => ({
-        id: row[db_id_column],
-        content: row[db_content_column],
-        source: `postgres:${db_table}`,
+    const table = context.db_table || "documents";
+    const contentCol = context.db_content_column || "content";
+    const idCol = context.db_id_column || "id";
+
+    if (context.MONGO_URI) {
+      const rows = await this.queryMongo(table);
+      return rows.map((r) => ({
+        id: r._id?.toString(),
+        content: r[contentCol] || "",
+        source: `mongodb:${table}`,
       }));
     }
-    return [];
-  }
 
-  buildMongoQuery(context) {
-    const { doc_filter = {}, doc_projection = {} } = context;
-    let filter = {};
-    if (typeof doc_filter === "string") {
-      try { filter = JSON.parse(doc_filter); } catch { filter = { $text: { $search: doc_filter } }; }
-    } else if (typeof doc_filter === "object" && Object.keys(doc_filter).length > 0) filter = doc_filter;
-    const projection = typeof doc_projection === "string" ? JSON.parse(doc_projection) : doc_projection;
-    return { filter, projection };
-  }
+    if (context.db_path) {
+      const rows = await this.querySQLite(
+        `SELECT ${idCol}, ${contentCol} FROM ${table}`
+      );
+      return rows.map((r) => ({
+        id: r[idCol],
+        content: r[contentCol],
+        source: `sqlite:${table}`,
+      }));
+    }
 
-  buildSqlQuery(context) {
-    const { db_content_column = "content", db_id_column = "id", doc_where = "1=1", doc_params = [] } = context;
-    let params = doc_params;
-    if (typeof doc_params === "string") {
-      try { params = JSON.parse(doc_params); } catch { params = [doc_params]; }
+    if (context.POSTGRES_URL) {
+      const rows = await this.queryPostgres(
+        `SELECT ${idCol}, ${contentCol} FROM ${table}`
+      );
+      return rows.map((r) => ({
+        id: r[idCol],
+        content: r[contentCol],
+        source: `postgres:${table}`,
+      }));
     }
-    const table = context.db_table || "documents";
-    const sql = `SELECT ${db_id_column}, ${db_content_column} FROM ${table} WHERE ${doc_where}`;
-    return { sql, params };
-  }
 
-  async close() {
-    if (this.sqliteClient) { try { this.sqliteClient.close(); } catch {} this.sqliteClient = null; }
-    if (this.mongoClient) { try { await this.mongoClient.close(); } catch {} this.mongoClient = null; }
-    if (this.postgresClient) { try { await this.postgresClient.end(); } catch {} this.postgresClient = null; }
-    this.initialized = false;
+    return [];
   }
 }
 
-// ------------------- DOCUMENT LOADING -------------------
-async function loadDocumentsFromDatabase(context) {
-  if (!context.db_type && !context.db_path && !context.MONGO_URI && !context.POSTGRES_URL) return null;
-  const dbAdapter = new DatabaseAdapter();
-  try { await dbAdapter.initialize(context); return await dbAdapter.queryDocuments(context); } catch (e) { console.error("🗃️ [doc-search] Database load error:", e.message); return null; }
-}
+/* ---------------- DOCUMENT LOADING ---------------- */
 
 async function loadAllDocuments(context) {
-  const documents = [];
-  const dbDocs = await loadDocumentsFromDatabase(context);
-  if (dbDocs) documents.push(...dbDocs);
+  const docs = [];
+  const db = new DatabaseAdapter();
+
+  try {
+    await db.initialize(context);
+    docs.push(...(await db.queryDocuments(context)));
+  } catch {}
+
+  const baseDir = context.doc_root
+    ? safeResolve(process.cwd(), context.doc_root)
+    : path.join(process.cwd(), "docs");
 
-  const baseDir = context.doc_root ? safeResolve(process.cwd(), context.doc_root) : path.join(process.cwd(), "docs");
   if (fs.existsSync(baseDir)) {
-    const files = fs.readdirSync(baseDir).filter((f) => f.endsWith(".txt") || f.endsWith(".md"));
-    for (const file of files) {
-      try {
-        const content = fs.readFileSync(path.join(baseDir, file), "utf8");
-        documents.push({ id: file, content, source: `file:${file}` });
-      } catch (e) { console.warn(`⚠️ [doc-search] Failed to read file ${file}: ${e.message}`); }
+    const files = fs
+      .readdirSync(baseDir)
+      .filter((f) => f.endsWith(".txt") || f.endsWith(".md"));
+
+    for (const f of files) {
+      docs.push({
+        id: f,
+        content: fs.readFileSync(path.join(baseDir, f), "utf8"),
+        source: `file:${f}`,
+      });
     }
   }
-  return documents;
-}
 
-// ------------------- VECTOR MIGRATION -------------------
-async function checkPgVectorHasData(pgVectorAdapter) {
-  try { const result = await pgVectorAdapter.pool.query("SELECT COUNT(*) FROM doc_embeddings"); return parseInt(result.rows[0].count) > 0; } catch { return false; }
+  return docs;
 }
 
-async function migrateDocumentsToPgVector(docRoot, pgVectorAdapter, embedder) {
-  const baseDir = safeResolve(process.cwd(), docRoot);
-  if (!fs.existsSync(baseDir)) { console.log("📁 No docs directory found, skipping migration"); return; }
-  const files = fs.readdirSync(baseDir).filter((f) => f.endsWith(".txt") || f.endsWith(".md"));
-  console.log(`🔄 Migrating ${files.length} documents to pgvector...`);
-  for (const file of files) {
-    try {
-      const content = fs.readFileSync(path.join(baseDir, file), "utf8");
-      const vector = await embedder.embed(content);
-      await pgVectorAdapter.upsert({ id: file, vector, content, source: `file:${file}` });
-      console.log(`✅ Migrated ${file}`);
-    } catch (e) { console.warn(`⚠️ Failed to migrate ${file}: ${e.message}`); }
+/* ---------------- HYBRID VECTOR SEARCH ---------------- */
+
+async function performHybridDocQA(query, context = {}) {
+  const cache = loadCache();
+  const embedder = new LocalEmbedding({ dimension: 384 });
+
+  const vectorStore = VectorRouter.create({
+    backend: context.vectorBackend || "memory",
+    dimension: 384,
+    ...context,
+  });
+
+  const documents = await loadAllDocuments(context);
+  if (!documents.length) {
+    return { text: "", meta: {} };
   }
-}
 
-// ------------------- VECTOR SEARCH (AUTO SWITCH) -------------------
-async function performVectorQA(query, context = {}) {
-  const postgresUrl = context.POSTGRES_URL || process.env.POSTGRES_URL;
-  const vectorBackend = context.vectorBackend;
-
-  if (postgresUrl) {
-    return await performPgVectorSearch(query, context);
-  } else if (vectorBackend) {
-    return await performVectorSearch(query, context);
-  } else {
-    return await performHybridDocQA(query, context);
+  for (const doc of documents) {
+    if (!cache[doc.id]) {
+      cache[doc.id] = true;
+      const chunks = chunkText(doc.content, 500);
+
+      for (let i = 0; i < chunks.length; i++) {
+        const vector = await embedder.embed(chunks[i]);
+        await vectorStore.upsert({
+          id: `${doc.id}:${i}`,
+          vector,
+          content: chunks[i],
+          source: doc.source,
+        });
+      }
+    }
   }
+
+  saveCache(cache);
+
+  const queryVector = await embedder.embed(query);
+  const results = await vectorStore.query(queryVector, 5);
+
+  return {
+    text: highlightMatches(
+      results.map((r) => r.content).join("\n\n"),
+      extractKeywords(query)
+    ),
+    meta: { matches: results.length },
+  };
+}
+
+/* ---------------- PGVECTOR SEARCH ---------------- */
+
+async function performPgVectorSearch(query, context = {}) {
+  const adapter = new PgVectorAdapter({
+    POSTGRES_URL: context.POSTGRES_URL,
+  });
+
+  const embedder = new LocalEmbedding({ dimension: 384 });
+  const vector = await embedder.embed(query);
+  const results = await adapter.query(vector, 5);
+
+  await adapter.close();
+
+  return {
+    text: results.map((r) => r.content).join("\n\n"),
+    meta: { matches: results.length },
+  };
 }
 
-// ------------------- HYBRID + VECTOR SEARCH FUNCTIONS -------------------
-// [Keep performPgVectorSearch, performHybridDocQA, loadAllDocuments, chunking, cache logic identical to previous full file]
+/* ---------------- ROUTER ---------------- */
 
 async function performDocQA(query, context = {}) {
-  return await performVectorQA(query, context);
+  if (context.POSTGRES_URL) {
+    return performPgVectorSearch(query, context);
+  }
+  return performHybridDocQA(query, context);
 }
 
+/* ---------------- O-LANG RESOLVER ---------------- */
+
 async function docSearchResolver(action, context) {
-  if (action.startsWith("Ask doc-search ")) {
+  if (action.startsWith("Ask doc-search")) {
     const match = action.match(/"(.*)"|'(.*)'/);
-    const query = match ? match[1] || match[2] : action.replace(/^Ask doc-search\s+/, "").trim();
-    return await performDocQA(query, context);
+    const query = match
+      ? match[1] || match[2]
+      : action.replace("Ask doc-search", "").trim();
+
+    return performDocQA(query, context);
   }
-  return undefined;
 }
 
 docSearchResolver.resolverName = "doc-search";