@retrivora-ai/rag-engine 1.8.1 → 1.8.2

package/dist/{chunk-R3RGUMHE.mjs → chunk-OZFBG4BA.mjs}

@@ -11,7 +11,7 @@ import {
 import { Pool } from "pg";
 var MultiTablePostgresProvider = class extends BaseVectorProvider {
   constructor(config) {
-    var _a, _b, _c, _d, _e;
+    var _a, _b, _c;
     super(config);
     const opts = config.options || {};
     if (!opts.connectionString) {
@@ -19,37 +19,37 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
     }
     this.connectionString = opts.connectionString;
     this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
-    const rawTables = (_c = (_b = opts.tables) != null ? _b : process.env.VECTOR_DB_TABLES) != null ? _c : "";
-    this.tables = typeof rawTables === "string" ? rawTables.split(",").map((t) => t.trim()).filter(Boolean) : rawTables;
-    if (this.tables.length === 0) {
-      console.warn(
-        "[MultiTablePostgresProvider] No tables configured. Set VECTOR_DB_TABLES as a comma-separated list of table names or pass options.tables."
-      );
-    }
-    const rawSearchFields = (_e = (_d = opts.searchFields) != null ? _d : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _e : ["name", "product_name", "productname", "title"];
+    this.tables = [];
+    const rawSearchFields = (_c = (_b = opts.searchFields) != null ? _b : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _c : ["name", "product_name", "productname", "title"];
     this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
+    this.uploadTable = opts.uploadTable || "document_chunks";
   }
   async initialize() {
     this.pool = new Pool({ connectionString: this.connectionString });
     const client = await this.pool.connect();
     try {
+      await client.query("CREATE EXTENSION IF NOT EXISTS vector");
+      await client.query(`
+        CREATE TABLE IF NOT EXISTS ${this.uploadTable} (
+          id          TEXT PRIMARY KEY,
+          namespace   TEXT NOT NULL DEFAULT '',
+          content     TEXT NOT NULL,
+          metadata    JSONB,
+          embedding   VECTOR(${this.dimensions})
+        )
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
+        ON ${this.uploadTable}
+        USING hnsw (embedding vector_cosine_ops)
+      `);
       const res = await client.query(`
         SELECT table_name 
         FROM information_schema.columns 
         WHERE column_name = 'embedding' 
         AND table_schema = 'public'
       `);
-      const discoveredTables = res.rows.map((r) => r.table_name);
-      if (this.tables.length === 0) {
-        this.tables = discoveredTables;
-      } else {
-        const staticTables = [...this.tables];
-        this.tables = staticTables.filter((t) => discoveredTables.includes(t));
-        if (this.tables.length < staticTables.length) {
-          const missing = staticTables.filter((t) => !discoveredTables.includes(t));
-          console.warn(`[MultiTablePostgresProvider] Some configured tables were skipped (missing 'embedding' column): ${missing.join(", ")}`);
-        }
-      }
+      this.tables = res.rows.map((r) => r.table_name);
       if (this.tables.length === 0) {
         console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
       } else {
@@ -62,27 +62,106 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
     }
   }
   /**
-   * Upsert is not supported for MultiTablePostgresProvider as it's designed for 
-   * searching across pre-existing tables with varying schemas.
+   * Upsert a document by dynamically provisioning a table and columns.
    */
-  async upsert(_doc, _namespace) {
-    throw new Error(
-      "[MultiTablePostgresProvider] upsert() is not supported in multi-table mode. Please use the standard PostgreSQLProvider for single-table managed indices."
-    );
+  async upsert(doc, namespace = "") {
+    await this.batchUpsert([doc], namespace);
   }
   /**
-   * Batch upsert is not supported for MultiTablePostgresProvider.
+   * Batch upsert documents by dynamically provisioning tables and columns based on CSV headers.
    */
-  async batchUpsert(_docs, _namespace) {
-    throw new Error(
-      "[MultiTablePostgresProvider] batchUpsert() is not supported in multi-table mode."
-    );
+  async batchUpsert(docs, namespace = "") {
+    var _a;
+    if (docs.length === 0) return;
+    const docsByFile = {};
+    for (const doc of docs) {
+      const fileName = ((_a = doc.metadata) == null ? void 0 : _a.fileName) || this.uploadTable;
+      if (!docsByFile[fileName]) docsByFile[fileName] = [];
+      docsByFile[fileName].push(doc);
+    }
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      for (const [fileName, fileDocs] of Object.entries(docsByFile)) {
+        let tableName = fileName;
+        if (tableName.toLowerCase().endsWith(".csv")) {
+          tableName = tableName.slice(0, -4);
+        }
+        tableName = tableName.replace(/[^a-z0-9_]/gi, "_").toLowerCase() || this.uploadTable;
+        const firstMeta = fileDocs[0].metadata || {};
+        const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding"];
+        const csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+        const columnDefs = csvHeaders.map((h) => `"${h}" TEXT`).join(",\n          ");
+        const createTableSql = `
+          CREATE TABLE IF NOT EXISTS "${tableName}" (
+            id          TEXT PRIMARY KEY,
+            ${columnDefs ? columnDefs + "," : ""}
+            namespace   TEXT NOT NULL DEFAULT '',
+            content     TEXT NOT NULL,
+            metadata    JSONB,
+            embedding   VECTOR(${this.dimensions})
+          )
+        `;
+        await client.query(createTableSql);
+        await client.query(`
+          CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
+          ON "${tableName}"
+          USING hnsw (embedding vector_cosine_ops)
+        `);
+        if (!this.tables.includes(tableName)) {
+          this.tables.push(tableName);
+        }
+        const BATCH_SIZE = 50;
+        for (let i = 0; i < fileDocs.length; i += BATCH_SIZE) {
+          const batch = fileDocs.slice(i, i + BATCH_SIZE);
+          const values = [];
+          const headerColumns = csvHeaders.map((h) => `"${h}"`).join(", ");
+          const allColumns = `id, ${headerColumns ? headerColumns + ", " : ""}namespace, content, metadata, embedding`;
+          const valuePlaceholders = batch.map((doc, idx) => {
+            var _a2, _b;
+            const offset = idx * (5 + csvHeaders.length);
+            values.push(doc.id);
+            for (const h of csvHeaders) {
+              values.push(String(((_a2 = doc.metadata) == null ? void 0 : _a2[h]) || ""));
+            }
+            values.push(namespace, doc.content, JSON.stringify((_b = doc.metadata) != null ? _b : {}), `[${doc.vector.join(",")}]`);
+            const docPlaceholders = [];
+            for (let j = 1; j <= 5 + csvHeaders.length; j++) {
+              if (j === 5 + csvHeaders.length) {
+                docPlaceholders.push(`$${offset + j}::vector`);
+              } else {
+                docPlaceholders.push(`$${offset + j}`);
+              }
+            }
+            return `(${docPlaceholders.join(", ")})`;
+          }).join(", ");
+          const conflictUpdates = csvHeaders.map((h) => `"${h}" = EXCLUDED."${h}"`).join(",\n              ");
+          const query = `
+            INSERT INTO "${tableName}" (${allColumns})
+            VALUES ${valuePlaceholders}
+            ON CONFLICT (id) DO UPDATE
+              SET ${conflictUpdates ? conflictUpdates + "," : ""}
+                  namespace = EXCLUDED.namespace,
+                  content   = EXCLUDED.content,
+                  metadata  = EXCLUDED.metadata,
+                  embedding = EXCLUDED.embedding
+          `;
+          await client.query(query, values);
+        }
+      }
+      await client.query("COMMIT");
+    } catch (error) {
+      await client.query("ROLLBACK");
+      throw error;
+    } finally {
+      client.release();
+    }
   }
   /**
    * Query all configured tables and merge results, sorted by cosine similarity score.
    */
   async query(vector, topK, _namespace, _filter) {
-    var _a, _b, _c;
+    var _a;
     if (!this.pool) {
       throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
     }
@@ -174,22 +253,16 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
     for (const tableResults of resultsArray) {
       allResults.push(...tableResults);
     }
-    const resultsByTable = {};
-    for (const res of allResults) {
-      const table = (_a = res.metadata) == null ? void 0 : _a.source_table;
-      if (!resultsByTable[table]) resultsByTable[table] = [];
-      resultsByTable[table].push(res);
-    }
-    const balancedResults = [];
-    const tables = Object.keys(resultsByTable);
-    for (const table of tables) {
-      balancedResults.push(...resultsByTable[table].slice(0, 3));
-    }
-    const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
-    if (finalSorted.length > 0) {
-      console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
-      console.log(`[MultiTablePostgresProvider] Final top match from "${(_c = (_b = finalSorted[0].metadata) == null ? void 0 : _b.source_table) != null ? _c : "unknown"}" with score ${finalSorted[0].score.toFixed(4)}`);
-    }
+    allResults.sort((a, b) => b.score - a.score);
+    if (allResults.length === 0) return [];
+    const bestMatchTable = (_a = allResults[0].metadata) == null ? void 0 : _a.source_table;
+    const finalSorted = allResults.filter((res) => {
+      var _a2;
+      return ((_a2 = res.metadata) == null ? void 0 : _a2.source_table) === bestMatchTable;
+    });
+    console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
+    console.log(`[MultiTablePostgresProvider] Final top match from "${bestMatchTable}" with score ${finalSorted[0].score.toFixed(4)}`);
+    console.log(`[MultiTablePostgresProvider] Restricting recommendations strictly to table: "${bestMatchTable}"`);
     return finalSorted.slice(0, Math.max(topK, 15));
   }
   async delete(_id, _namespace) {

package/dist/handlers/index.d.mts

@@ -1,3 +1,3 @@
-import '../ILLMProvider-BfRgI1Xh.mjs';
-export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame, l as sseUIFrame } from '../index-BV0z5mb6.mjs';
+import '../ILLMProvider-BOJFz3Na.mjs';
+export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, l as sseObservabilityFrame, j as sseTextFrame, m as sseUIFrame } from '../index-D3V9Et2M.mjs';
 import 'next/server';

package/dist/handlers/index.d.ts

@@ -1,3 +1,3 @@
-import '../ILLMProvider-BfRgI1Xh.js';
-export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame, l as sseUIFrame } from '../index-1Z4GuYBi.js';
+import '../ILLMProvider-BOJFz3Na.js';
+export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, l as sseObservabilityFrame, j as sseTextFrame, m as sseUIFrame } from '../index-BwpcaziY.js';
 import 'next/server';