@o-lang/semantic-doc-search 1.0.21 → 1.0.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@o-lang/semantic-doc-search",
-  "version": "1.0.21",
+  "version": "1.0.23",
   "description": "O-lang Semantic Document Search Resolver with hybrid search, embeddings, rerank, and streaming.",
   "main": "src/index.js",
   "type": "commonjs",

package/src/index.js

@@ -1,19 +1,21 @@
 const fs = require("fs");
 const path = require("path");
-const { createLLM } = require("./llm/router.js");
 const { LocalEmbedding } = require("./embeddings/local.js");
 const { chunkText } = require("./utils/chunker.js");
 const { extractKeywords } = require("./utils/extractText.js");
-const { cosine } = require("./utils/similarity.js");
 const { highlightMatches } = require("./utils/highlight.js");
-const PgVectorAdapter = require("./adapters/pgvectorAdapter.js");
 const VectorRouter = require("./adapters/vectorRouter");
+const PgVectorAdapter = require("./adapters/pgvectorAdapter.js");
 
 const CACHE_PATH = path.join(process.cwd(), "embeddings.json");
 
+/* ---------------- UTIL ---------------- */
+
 function safeResolve(base, userPath) {
   const resolved = path.resolve(base, userPath);
-  if (!resolved.startsWith(path.resolve(base))) throw new Error("Path traversal detected");
+  if (!resolved.startsWith(path.resolve(base))) {
+    throw new Error("Path traversal detected");
+  }
   return resolved;
 }
 
@@ -32,7 +34,8 @@ function saveCache(cache) {
   } catch {}
 }
 
-// ------------------- DATABASE ADAPTER -------------------
+/* ---------------- DATABASE ADAPTER ---------------- */
+
 class DatabaseAdapter {
   constructor() {
     this.initialized = false;
@@ -41,13 +44,9 @@ class DatabaseAdapter {
   async initialize(context) {
     if (this.initialized) return;
 
-    if (context.db_type === "mongodb" || context.MONGO_URI) {
-      await this.initMongo(context);
-    } else if (context.db_type === "sqlite" || context.db_path) {
-      await this.initSQLite(context);
-    } else if (context.db_type === "postgres" || context.POSTGRES_URL) {
-      await this.initPostgres(context);
-    }
+    if (context.MONGO_URI) await this.initMongo(context);
+    else if (context.db_path) await this.initSQLite(context);
+    else if (context.POSTGRES_URL) await this.initPostgres(context);
 
     this.initialized = true;
   }
@@ -55,120 +54,67 @@ class DatabaseAdapter {
   async initSQLite(context) {
     const Database = require("better-sqlite3");
     const dbPath = context.db_path || "./database.db";
-    const dbDir = path.dirname(path.resolve(dbPath));
-    if (!fs.existsSync(dbDir)) throw new Error(`SQLite database directory not found: ${dbDir}`);
-    this.sqliteClient = new Database(dbPath, { readonly: true });
-  }
-
-  async querySQLite(query, params = []) {
-    if (!this.sqliteClient) throw new Error("SQLite client not initialized");
-    const stmt = this.sqliteClient.prepare(query);
-    return stmt.all(...params);
+    this.sqlite = new Database(dbPath, { readonly: true });
   }
 
   async initMongo(context) {
     const { MongoClient } = require("mongodb");
-    const uri = context.MONGO_URI || `mongodb://localhost:27017/${context.db_name || "olang"}`;
-    this.mongoClient = new MongoClient(uri);
-    await this.mongoClient.connect();
-  }
-
-  async queryMongo(collectionName, filter = {}, projection = {}) {
-    if (!this.mongoClient) throw new Error("MongoDB client not initialized");
-    const db = this.mongoClient.db(process.env.DB_NAME || "olang");
-    return db.collection(collectionName).find(filter, { projection }).toArray();
+    const uri = context.MONGO_URI;
+    this.mongo = new MongoClient(uri);
+    await this.mongo.connect();
   }
 
   async initPostgres(context) {
     const { Pool } = require("pg");
-    const poolConfig = {
-      connectionString: context.POSTGRES_URL,
-      host: context.DB_HOST || "localhost",
-      port: parseInt(context.DB_PORT) || 5432,
-      user: context.DB_USER,
-      password: context.DB_PASSWORD,
-      database: context.DB_NAME || "olang",
-    };
-    Object.keys(poolConfig).forEach((k) => {
-      if (poolConfig[k] == null) delete poolConfig[k];
-    });
-    this.postgresClient = new Pool(poolConfig);
-  }
-
-  async queryPostgres(query, params = []) {
-    if (!this.postgresClient) throw new Error("PostgreSQL client not initialized");
-    const result = await this.postgresClient.query(query, params);
-    return result.rows;
+    this.pg = new Pool({ connectionString: context.POSTGRES_URL });
   }
 
   async queryDocuments(context) {
-    const {
-      db_type,
-      db_table = "documents",
-      db_content_column = "content",
-      db_id_column = "id",
-    } = context;
-
-    if (db_type === "mongodb" || context.MONGO_URI) {
-      const { filter, projection } = this.buildMongoQuery(context);
-      const results = await this.queryMongo(db_table, filter, projection);
-      return results.map((doc) => ({
-        id: doc._id?.toString() || doc[db_id_column],
-        content: doc[db_content_column] || "",
-        source: `mongodb:${db_table}`,
+    const table = context.db_table || "documents";
+    const idCol = context.db_id_column || "id";
+    const contentCol = context.db_content_column || "content";
+
+    if (this.mongo) {
+      const rows = await this.mongo.db().collection(table).find({}).toArray();
+      return rows.map(r => ({
+        id: r._id.toString(),
+        content: r[contentCol] || "",
+        source: `mongodb:${table}`,
       }));
     }
 
-    if (db_type === "sqlite" || context.db_path) {
-      const { sql, params } = this.buildSqlQuery(context);
-      const results = await this.querySQLite(sql, params);
-      return results.map((row) => ({
-        id: row[db_id_column],
-        content: row[db_content_column],
-        source: `sqlite:${db_table}`,
+    if (this.sqlite) {
+      const rows = this.sqlite
+        .prepare(`SELECT ${idCol}, ${contentCol} FROM ${table}`)
+        .all();
+      return rows.map(r => ({
+        id: r[idCol],
+        content: r[contentCol],
+        source: `sqlite:${table}`,
       }));
     }
 
-    if (db_type === "postgres" || context.POSTGRES_URL) {
-      const { sql, params } = this.buildSqlQuery(context);
-      const results = await this.queryPostgres(sql, params);
-      return results.map((row) => ({
-        id: row[db_id_column],
-        content: row[db_content_column],
-        source: `postgres:${db_table}`,
+    if (this.pg) {
+      const res = await this.pg.query(
+        `SELECT ${idCol}, ${contentCol} FROM ${table}`
+      );
+      return res.rows.map(r => ({
+        id: r[idCol],
+        content: r[contentCol],
+        source: `postgres:${table}`,
       }));
     }
 
     return [];
   }
-
-  buildMongoQuery(context) {
-    let filter = {};
-    if (typeof context.doc_filter === "string") {
-      try {
-        filter = JSON.parse(context.doc_filter);
-      } catch {
-        filter = { $text: { $search: context.doc_filter } };
-      }
-    }
-    return { filter, projection: {} };
-  }
-
-  buildSqlQuery(context) {
-    const table = context.db_table || "documents";
-    const where = context.doc_where || "1=1";
-    return {
-      sql: `SELECT * FROM ${table} WHERE ${where}`,
-      params: [],
-    };
-  }
 }
 
-// ------------------- DOCUMENT LOADING -------------------
+/* ---------------- DOCUMENT LOADING ---------------- */
+
 async function loadAllDocuments(context) {
   const docs = [];
-  const db = new DatabaseAdapter();
 
+  const db = new DatabaseAdapter();
   try {
     await db.initialize(context);
     docs.push(...(await db.queryDocuments(context)));
@@ -179,22 +125,29 @@ async function loadAllDocuments(context) {
     : path.join(process.cwd(), "docs");
 
   if (fs.existsSync(baseDir)) {
-    const files = fs.readdirSync(baseDir).filter((f) => f.endsWith(".txt") || f.endsWith(".md"));
-    for (const f of files) {
-      const content = fs.readFileSync(path.join(baseDir, f), "utf8");
-      docs.push({ id: f, content, source: `file:${f}` });
+    const files = fs.readdirSync(baseDir).filter(f =>
+      f.endsWith(".txt") || f.endsWith(".md")
+    );
+
+    for (const file of files) {
+      docs.push({
+        id: file,
+        content: fs.readFileSync(path.join(baseDir, file), "utf8"),
+        source: `file:${file}`,
+      });
     }
   }
 
   return docs;
 }
 
-// ------------------- HYBRID SEARCH (FIXED & DEFINED) -------------------
-async function performHybridDocQA(query, context = {}) {
+/* ---------------- HYBRID VECTOR SEARCH ---------------- */
+
+async function performHybridDocQA(query, context) {
   const cache = loadCache();
   const embedder = new LocalEmbedding({ dimension: 384 });
 
-  const router = VectorRouter.create({
+  const store = VectorRouter.create({
     backend: context.vectorBackend || "memory",
     dimension: 384,
     ...context,
@@ -205,17 +158,17 @@ async function performHybridDocQA(query, context = {}) {
 
   for (const doc of documents) {
     if (!cache[doc.id]) {
+      cache[doc.id] = true;
       const chunks = chunkText(doc.content, 500);
-      cache[doc.id] = [];
-      for (const chunk of chunks) {
-        const vector = await embedder.embed(chunk);
-        await router.upsert({
-          id: `${doc.id}:${cache[doc.id].length}`,
+
+      for (let i = 0; i < chunks.length; i++) {
+        const vector = await embedder.embed(chunks[i]);
+        await store.upsert({
+          id: `${doc.id}:${i}`,
           vector,
-          text: chunk,
+          content: chunks[i],
           source: doc.source,
         });
-        cache[doc.id].push(vector);
       }
     }
   }
@@ -223,47 +176,58 @@ async function performHybridDocQA(query, context = {}) {
   saveCache(cache);
 
   const queryVector = await embedder.embed(query);
-  const results = await router.search({ embedding: queryVector, topK: 5 });
+  const results = await store.search({
+    embedding: queryVector,
+    topK: 5,
+  });
 
   return {
     text: highlightMatches(
-      results.map((r) => r.text).join("\n\n"),
+      results.map(r => r.content).join("\n\n"),
       extractKeywords(query)
     ),
     meta: { matches: results.length },
   };
 }
 
-// ------------------- PGVECTOR SEARCH -------------------
-async function performPgVectorSearch(query, context = {}) {
+/* ---------------- PGVECTOR SEARCH ---------------- */
+
+async function performPgVectorSearch(query, context) {
   const adapter = new PgVectorAdapter({ POSTGRES_URL: context.POSTGRES_URL });
   const embedder = new LocalEmbedding({ dimension: 384 });
+
   const vector = await embedder.embed(query);
   const results = await adapter.search(vector, 5);
+  await adapter.close();
+
   return {
-    text: results.map((r) => r.content).join("\n\n"),
+    text: results.map(r => r.content).join("\n\n"),
     meta: { matches: results.length },
   };
 }
 
-// ------------------- ROUTER -------------------
-async function performVectorQA(query, context = {}) {
-  if (context.POSTGRES_URL) return performPgVectorSearch(query, context);
+/* ---------------- ROUTER ---------------- */
+
+async function performDocQA(query, context) {
+  if (context.POSTGRES_URL) {
+    return performPgVectorSearch(query, context);
+  }
   return performHybridDocQA(query, context);
 }
 
-async function performDocQA(query, context = {}) {
-  return performVectorQA(query, context);
-}
+/* ---------------- O-LANG RESOLVER ---------------- */
 
-// ------------------- RESOLVER -------------------
 async function docSearchResolver(action, context) {
-  if (action.startsWith("Ask doc-search")) {
-    const match = action.match(/"(.*)"|'(.*)'/);
-    const query = match ? match[1] || match[2] : action.replace("Ask doc-search", "").trim();
-    return performDocQA(query, context);
-  }
+  if (!action.startsWith("Ask doc-search")) return;
+
+  const match = action.match(/"(.*)"|'(.*)'/);
+  const query = match
+    ? match[1] || match[2]
+    : action.replace("Ask doc-search", "").trim();
+
+  return performDocQA(query, context);
 }
 
 docSearchResolver.resolverName = "doc-search";
 module.exports = docSearchResolver;
+