@o-lang/semantic-doc-search 1.0.0 → 1.0.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@o-lang/semantic-doc-search",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "description": "O-lang Semantic Document Search Resolver with hybrid search, embeddings, rerank, and streaming.",
   "main": "src/index.js",
   "type": "module",

package/src/index.js

@@ -33,6 +33,236 @@ function saveCache(cache) {
   } catch {}
 }
 
+// ✅ UNIVERSAL DATABASE ADAPTER (NEW - Keep your existing imports!)
+class DatabaseAdapter {
+  constructor() {
+    this.initialized = false;
+  }
+
+  async initialize(context) {
+    if (this.initialized) return;
+    
+    // Initialize based on context configuration
+    if (context.db_type === 'mongodb' || context.MONGO_URI) {
+      await this.initMongo(context);
+    } else if (context.db_type === 'sqlite' || context.db_path) {
+      await this.initSQLite(context);
+    } else if (context.db_type === 'postgres' || context.POSTGRES_URL) {
+      await this.initPostgres(context);
+    }
+    this.initialized = true;
+  }
+
+  // SQLite Support
+  async initSQLite(context) {
+    const Database = require('better-sqlite3');
+    const dbPath = context.db_path || './database.db';
+    const dbDir = path.dirname(path.resolve(dbPath));
+    if (!fs.existsSync(dbDir)) {
+      throw new Error(`SQLite database directory not found: ${dbDir}`);
+    }
+    this.sqliteClient = new Database(dbPath, { readonly: true });
+  }
+
+  async querySQLite(query, params = []) {
+    if (!this.sqliteClient) throw new Error('SQLite client not initialized');
+    const stmt = this.sqliteClient.prepare(query);
+    return stmt.all(...params);
+  }
+
+  // MongoDB Support
+  async initMongo(context) {
+    const { MongoClient } = require('mongodb');
+    const uri = context.MONGO_URI || `mongodb://localhost:27017/${context.db_name || 'olang'}`;
+    this.mongoClient = new MongoClient(uri);
+    await this.mongoClient.connect();
+  }
+
+  async queryMongo(collectionName, filter = {}, projection = {}) {
+    if (!this.mongoClient) throw new Error('MongoDB client not initialized');
+    const db = this.mongoClient.db(process.env.DB_NAME || context.db_name || 'olang');
+    return await db.collection(collectionName).find(filter, { projection }).toArray();
+  }
+
+  // PostgreSQL Support  
+  async initPostgres(context) {
+    const { Pool } = require('pg');
+    const poolConfig = {
+      connectionString: context.POSTGRES_URL,
+      host: context.DB_HOST || 'localhost',
+      port: parseInt(context.DB_PORT) || 5432,
+      user: context.DB_USER,
+      password: context.DB_PASSWORD,
+      database: context.DB_NAME || 'olang'
+    };
+    // Remove undefined/null values
+    Object.keys(poolConfig).forEach(key => {
+      if (poolConfig[key] === undefined || poolConfig[key] === null) {
+        delete poolConfig[key];
+      }
+    });
+    this.postgresClient = new Pool(poolConfig);
+  }
+
+  async queryPostgres(query, params = []) {
+    if (!this.postgresClient) throw new Error('PostgreSQL client not initialized');
+    const result = await this.postgresClient.query(query, params);
+    return result.rows;
+  }
+
+  // Universal Query Method
+  async queryDocuments(context) {
+    const { 
+      db_type, 
+      db_table = 'documents',
+      db_content_column = 'content', 
+      db_id_column = 'id' 
+    } = context;
+    
+    if (db_type === 'mongodb' || context.MONGO_URI) {
+      const mongoQuery = this.buildMongoQuery(context);
+      const results = await this.queryMongo(db_table, mongoQuery.filter, mongoQuery.projection);
+      return results.map(doc => ({
+        id: doc._id?.toString() || doc.id || doc[db_id_column],
+        content: doc[db_content_column] || doc.content || doc.text || '',
+        source: `mongodb:${db_table}`
+      }));
+    } 
+    else if (db_type === 'sqlite' || context.db_path) {
+      const sqliteQuery = this.buildSqlQuery(context, 'sqlite');
+      const results = await this.querySQLite(sqliteQuery.sql, sqliteQuery.params);
+      return results.map(row => ({
+        id: row[db_id_column],
+        content: row[db_content_column],
+        source: `sqlite:${db_table}`
+      }));
+    }
+    else if (db_type === 'postgres' || context.POSTGRES_URL) {
+      const postgresQuery = this.buildSqlQuery(context, 'postgres');
+      const results = await this.queryPostgres(postgresQuery.sql, postgresQuery.params);
+      return results.map(row => ({
+        id: row[db_id_column],
+        content: row[db_content_column],
+        source: `postgres:${db_table}`
+      }));
+    }
+    
+    return [];
+  }
+
+  buildMongoQuery(context) {
+    const { doc_filter = {}, doc_projection = {} } = context;
+    
+    let filter = {};
+    if (typeof doc_filter === 'string') {
+      try {
+        filter = JSON.parse(doc_filter);
+      } catch {
+        // Text search fallback
+        filter = { $text: { $search: doc_filter } };
+      }
+    } else if (typeof doc_filter === 'object' && Object.keys(doc_filter).length > 0) {
+      filter = doc_filter;
+    }
+    
+    const projection = typeof doc_projection === 'string' 
+      ? JSON.parse(doc_projection) 
+      : doc_projection;
+      
+    return { filter, projection };
+  }
+
+  buildSqlQuery(context, dialect) {
+    const { 
+      db_content_column = 'content', 
+      db_id_column = 'id',
+      doc_where = '1=1',
+      doc_params = []
+    } = context;
+    
+    // Parse doc_params from string if needed
+    let params = doc_params;
+    if (typeof doc_params === 'string') {
+      try {
+        params = JSON.parse(doc_params);
+      } catch {
+        params = [doc_params];
+      }
+    }
+    
+    const table = context.db_table || 'documents';
+    const sql = `SELECT ${db_id_column}, ${db_content_column} FROM ${table} WHERE ${doc_where}`;
+    return { sql, params };
+  }
+
+  async close() {
+    if (this.sqliteClient) {
+      try { this.sqliteClient.close(); } catch {}
+      this.sqliteClient = null;
+    }
+    if (this.mongoClient) {
+      try { await this.mongoClient.close(); } catch {}
+      this.mongoClient = null;
+    }
+    if (this.postgresClient) {
+      try { await this.postgresClient.end(); } catch {}
+      this.postgresClient = null;
+    }
+    this.initialized = false;
+  }
+}
+
+// ✅ LOAD DOCUMENTS FROM DATABASE (if configured)
+async function loadDocumentsFromDatabase(context) {
+  if (!context.db_type && !context.db_path && !context.MONGO_URI && !context.POSTGRES_URL) {
+    return null; // No database configured
+  }
+  
+  const dbAdapter = new DatabaseAdapter();
+  try {
+    await dbAdapter.initialize(context);
+    return await dbAdapter.queryDocuments(context);
+  } catch (error) {
+    console.error('🗃️ [doc-search] Database load error:', error.message);
+    return null;
+  }
+}
+
+// ✅ LOAD ALL DOCUMENTS (Database + Files)
+async function loadAllDocuments(context) {
+  const documents = [];
+  
+  // 1. Load from database first (if configured)
+  const dbDocs = await loadDocumentsFromDatabase(context);
+  if (dbDocs) {
+    documents.push(...dbDocs);
+  }
+  
+  // 2. Load from file system (existing behavior)
+  const baseDir = context.doc_root 
+    ? safeResolve(process.cwd(), context.doc_root)
+    : path.join(process.cwd(), "docs");
+  
+  if (fs.existsSync(baseDir)) {
+    const files = fs.readdirSync(baseDir).filter(f => f.endsWith(".txt") || f.endsWith(".md"));
+    for (const file of files) {
+      try {
+        const content = fs.readFileSync(path.join(baseDir, file), "utf8");
+        documents.push({
+          id: file,
+          content: content,
+          source: `file:${file}`
+        });
+      } catch (error) {
+        console.warn(`⚠️ [doc-search] Failed to read file ${file}: ${error.message}`);
+      }
+    }
+  }
+  
+  return documents;
+}
+
+// ✅ MAIN SEARCH FUNCTION (Your existing logic + universal docs)
 async function performDocQA(query, context = {}) {
   const { doc_root, stream = false } = context;
   const options = context.options || {};
@@ -46,25 +276,20 @@ async function performDocQA(query, context = {}) {
     return { text: "Missing required input: query" };
   }
 
-  const baseDir = doc_root
-    ? safeResolve(process.cwd(), doc_root)
-    : path.join(process.cwd(), "docs");
-
-  if (!fs.existsSync(baseDir)) {
-    return { text: `Document directory not found: ${baseDir}` };
-  }
-
-  const files = fs.readdirSync(baseDir).filter(f => f.endsWith(".txt") || f.endsWith(".md"));
-  if (!files.length) {
+  // Load documents from both database and files
+  const allDocs = await loadAllDocuments(context);
+  if (!allDocs || !allDocs.length) {
     return { text: "No documents available." };
   }
 
   const qLower = query.toLowerCase().trim();
-  const exact = files.find(f => path.basename(f, path.extname(f)).toLowerCase() === qLower);
-  if (exact) {
+  const exactMatch = allDocs.find(doc => 
+    path.basename(doc.id || '', path.extname(doc.id || '')).toLowerCase() === qLower
+  );
+  if (exactMatch) {
     return {
-      text: fs.readFileSync(path.join(baseDir, exact), "utf8"),
-      meta: { file: exact, method: "exact-filename" }
+      text: exactMatch.content,
+      meta: { file: exactMatch.id, method: "exact-filename" }
     };
   }
 
@@ -72,13 +297,12 @@ async function performDocQA(query, context = {}) {
   const docs = [];
   const localEmbedder = new LocalEmbedding();
 
-  for (const file of files) {
-    const raw = fs.readFileSync(path.join(baseDir, file), "utf8");
-    const chunks = chunkText(raw, CHUNK_SIZE, OVERLAP);
+  for (const doc of allDocs) {
+    const chunks = chunkText(doc.content, CHUNK_SIZE, OVERLAP);
     const chunkObjs = [];
 
     for (let i = 0; i < chunks.length; i++) {
-      const key = `${file}::chunk::${i}`;
+      const key = `${doc.id}::chunk::${i}`;
       let emb = cache[key];
       if (!emb) {
         try {
@@ -91,7 +315,7 @@ async function performDocQA(query, context = {}) {
       }
       chunkObjs.push({ index: i, text: chunks[i], emb });
     }
-    docs.push({ file, raw, chunks: chunkObjs });
+    docs.push({ file: doc.id, raw: doc.content, chunks: chunkObjs, source: doc.source });
   }
 
   let queryEmb = null;
@@ -117,20 +341,20 @@ async function performDocQA(query, context = {}) {
         bestChunk = { ...ch, semScore, lexScore, hybrid };
       }
     }
-    return { file: doc.file, score: bestHybrid, bestChunk };
+    return { file: doc.file, score: bestHybrid, bestChunk, source: doc.source };
   });
 
   fileScores.sort((a, b) => b.score - a.score);
   const best = fileScores[0];
 
   if (!best || best.score < MIN_SCORE) {
-    for (const file of files) {
-      const text = fs.readFileSync(path.join(baseDir, file), "utf8").toLowerCase();
+    for (const doc of allDocs) {
+      const text = doc.content.toLowerCase();
       if (keywords.some(k => text.includes(k))) {
         const snippetIndex = text.indexOf(keywords.find(k => text.includes(k)));
         const start = Math.max(0, snippetIndex - 200);
         const snippet = text.slice(start, Math.min(text.length, snippetIndex + 400));
-        return { text: snippet, meta: { file, method: "lexical-fallback" } };
+        return { text: snippet, meta: { file: doc.id, method: "lexical-fallback", source: doc.source } };
       }
     }
     return { text: `No document found matching: "${query}"` };
@@ -150,24 +374,29 @@ async function performDocQA(query, context = {}) {
       await llm.stream({ prompt: snippet, model, onToken: context.onToken });
       return {
         text: snippet,
-        meta: { file: best.file, chunkIndex: best.bestChunk.index, method: "hybrid-semantic-stream" }
+        meta: { file: best.file, chunkIndex: best.bestChunk.index, method: "hybrid-semantic-stream", source: best.source }
       };
     } else {
       const resp = await llm.generate({ prompt: snippet, model });
       return {
         text: resp.text,
-        meta: { file: best.file, chunkIndex: best.bestChunk.index, method: "hybrid-semantic" }
+        meta: { file: best.file, chunkIndex: best.bestChunk.index, method: "hybrid-semantic", source: best.source }
       };
     }
   }
 
   return {
     text: snippet,
-    meta: { file: best.file, chunkIndex: best.bestChunk.index, method: "hybrid-semantic" }
+    meta: { 
+      file: best.file, 
+      chunkIndex: best.bestChunk.index, 
+      method: "hybrid-semantic",
+      source: best.source
+    }
   };
 }
 
-// ✅ O-Lang Resolver Interface
+// ✅ O-Lang Resolver Interface (Your existing interface)
 export default async function docSearchResolver(action, context) {
   if (action.startsWith('Ask doc-search ')) {
     const match = action.match(/"(.*)"|'(.*)'/);