@retrivora-ai/rag-engine 1.8.0 → 1.8.2

package/dist/chunk-OZFBG4BA.mjs

@@ -0,0 +1,291 @@
+import {
+  BaseVectorProvider
+} from "./chunk-IMP6FUCY.mjs";
+import {
+  __objRest,
+  __spreadProps,
+  __spreadValues
+} from "./chunk-X4TOT24V.mjs";
+
+// src/providers/vectordb/MultiTablePostgresProvider.ts
+import { Pool } from "pg";
+var MultiTablePostgresProvider = class extends BaseVectorProvider {
+  constructor(config) {
+    var _a, _b, _c;
+    super(config);
+    const opts = config.options || {};
+    if (!opts.connectionString) {
+      throw new Error("[MultiTablePostgresProvider] options.connectionString is required");
+    }
+    this.connectionString = opts.connectionString;
+    this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
+    this.tables = [];
+    const rawSearchFields = (_c = (_b = opts.searchFields) != null ? _b : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _c : ["name", "product_name", "productname", "title"];
+    this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
+    this.uploadTable = opts.uploadTable || "document_chunks";
+  }
+  async initialize() {
+    this.pool = new Pool({ connectionString: this.connectionString });
+    const client = await this.pool.connect();
+    try {
+      await client.query("CREATE EXTENSION IF NOT EXISTS vector");
+      await client.query(`
+        CREATE TABLE IF NOT EXISTS ${this.uploadTable} (
+          id          TEXT PRIMARY KEY,
+          namespace   TEXT NOT NULL DEFAULT '',
+          content     TEXT NOT NULL,
+          metadata    JSONB,
+          embedding   VECTOR(${this.dimensions})
+        )
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
+        ON ${this.uploadTable}
+        USING hnsw (embedding vector_cosine_ops)
+      `);
+      const res = await client.query(`
+        SELECT table_name 
+        FROM information_schema.columns 
+        WHERE column_name = 'embedding' 
+        AND table_schema = 'public'
+      `);
+      this.tables = res.rows.map((r) => r.table_name);
+      if (this.tables.length === 0) {
+        console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
+      } else {
+        console.log(
+          `[MultiTablePostgresProvider] Connected. Searching across ${this.tables.length} table(s): ${this.tables.join(", ")}`
+        );
+      }
+    } finally {
+      client.release();
+    }
+  }
+  /**
+   * Upsert a document by dynamically provisioning a table and columns.
+   */
+  async upsert(doc, namespace = "") {
+    await this.batchUpsert([doc], namespace);
+  }
+  /**
+   * Batch upsert documents by dynamically provisioning tables and columns based on CSV headers.
+   */
+  async batchUpsert(docs, namespace = "") {
+    var _a;
+    if (docs.length === 0) return;
+    const docsByFile = {};
+    for (const doc of docs) {
+      const fileName = ((_a = doc.metadata) == null ? void 0 : _a.fileName) || this.uploadTable;
+      if (!docsByFile[fileName]) docsByFile[fileName] = [];
+      docsByFile[fileName].push(doc);
+    }
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      for (const [fileName, fileDocs] of Object.entries(docsByFile)) {
+        let tableName = fileName;
+        if (tableName.toLowerCase().endsWith(".csv")) {
+          tableName = tableName.slice(0, -4);
+        }
+        tableName = tableName.replace(/[^a-z0-9_]/gi, "_").toLowerCase() || this.uploadTable;
+        const firstMeta = fileDocs[0].metadata || {};
+        const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding"];
+        const csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+        const columnDefs = csvHeaders.map((h) => `"${h}" TEXT`).join(",\n          ");
+        const createTableSql = `
+          CREATE TABLE IF NOT EXISTS "${tableName}" (
+            id          TEXT PRIMARY KEY,
+            ${columnDefs ? columnDefs + "," : ""}
+            namespace   TEXT NOT NULL DEFAULT '',
+            content     TEXT NOT NULL,
+            metadata    JSONB,
+            embedding   VECTOR(${this.dimensions})
+          )
+        `;
+        await client.query(createTableSql);
+        await client.query(`
+          CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
+          ON "${tableName}"
+          USING hnsw (embedding vector_cosine_ops)
+        `);
+        if (!this.tables.includes(tableName)) {
+          this.tables.push(tableName);
+        }
+        const BATCH_SIZE = 50;
+        for (let i = 0; i < fileDocs.length; i += BATCH_SIZE) {
+          const batch = fileDocs.slice(i, i + BATCH_SIZE);
+          const values = [];
+          const headerColumns = csvHeaders.map((h) => `"${h}"`).join(", ");
+          const allColumns = `id, ${headerColumns ? headerColumns + ", " : ""}namespace, content, metadata, embedding`;
+          const valuePlaceholders = batch.map((doc, idx) => {
+            var _a2, _b;
+            const offset = idx * (5 + csvHeaders.length);
+            values.push(doc.id);
+            for (const h of csvHeaders) {
+              values.push(String(((_a2 = doc.metadata) == null ? void 0 : _a2[h]) || ""));
+            }
+            values.push(namespace, doc.content, JSON.stringify((_b = doc.metadata) != null ? _b : {}), `[${doc.vector.join(",")}]`);
+            const docPlaceholders = [];
+            for (let j = 1; j <= 5 + csvHeaders.length; j++) {
+              if (j === 5 + csvHeaders.length) {
+                docPlaceholders.push(`$${offset + j}::vector`);
+              } else {
+                docPlaceholders.push(`$${offset + j}`);
+              }
+            }
+            return `(${docPlaceholders.join(", ")})`;
+          }).join(", ");
+          const conflictUpdates = csvHeaders.map((h) => `"${h}" = EXCLUDED."${h}"`).join(",\n              ");
+          const query = `
+            INSERT INTO "${tableName}" (${allColumns})
+            VALUES ${valuePlaceholders}
+            ON CONFLICT (id) DO UPDATE
+              SET ${conflictUpdates ? conflictUpdates + "," : ""}
+                  namespace = EXCLUDED.namespace,
+                  content   = EXCLUDED.content,
+                  metadata  = EXCLUDED.metadata,
+                  embedding = EXCLUDED.embedding
+          `;
+          await client.query(query, values);
+        }
+      }
+      await client.query("COMMIT");
+    } catch (error) {
+      await client.query("ROLLBACK");
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+  /**
+   * Query all configured tables and merge results, sorted by cosine similarity score.
+   */
+  async query(vector, topK, _namespace, _filter) {
+    var _a;
+    if (!this.pool) {
+      throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
+    }
+    const vectorLiteral = `[${vector.join(",")}]`;
+    const allResults = [];
+    console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
+    const queryText = _filter == null ? void 0 : _filter.queryText;
+    const entityHints = Array.isArray(_filter == null ? void 0 : _filter.__entityHints) ? _filter.__entityHints.filter(
+      (hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
+    ).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
+    console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
+    console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
+    const getDynamicKeywordQuery = () => {
+      if (entityHints.length > 0) {
+        return entityHints.map((h) => h.includes(" ") ? `"${h}"` : h).join(" & ");
+      }
+      if (queryText) {
+        return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
+      }
+      return "";
+    };
+    const dynamicKeywordQuery = getDynamicKeywordQuery();
+    const queryPromises = this.tables.map(async (table) => {
+      try {
+        let sqlQuery = "";
+        let params = [];
+        if (queryText) {
+          const hasEntityHints = entityHints.length > 0;
+          const exactNameScoreExpr = hasEntityHints ? `+ (
+                 SELECT COALESCE(MAX(
+                   CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(", ")})
+                   THEN 1.0 ELSE 0.0 END
+                 ), 0)
+                 FROM jsonb_each_text(to_jsonb(t)) AS kv(key, val)
+                 WHERE key IN (${this.searchFields.map((f) => `'${f}'`).join(", ")})
+               ) * 3.0` : "";
+          sqlQuery = `
+            SELECT *, 
+             (1 - (embedding <=> $1::vector)) AS vector_score,
+             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
+             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
+            FROM "${table}" t
+            ORDER BY hybrid_score DESC
+            LIMIT 50
+          `;
+          params = [vectorLiteral, dynamicKeywordQuery];
+        } else {
+          sqlQuery = `
+            SELECT *, 
+             (1 - (embedding <=> $1::vector)) AS hybrid_score
+            FROM "${table}" t
+            ORDER BY hybrid_score DESC
+            LIMIT 50
+          `;
+          params = [vectorLiteral];
+        }
+        const result = await this.pool.query(sqlQuery, params);
+        if (result.rowCount && result.rowCount > 0) {
+          console.log(`   \u2705 Table "${table}": Found ${result.rowCount} potential matches. Top score: ${result.rows[0].hybrid_score.toFixed(4)}`);
+        } else {
+          console.log(`   \u26AA Table "${table}": No relevant matches found.`);
+        }
+        const tableResults = [];
+        for (const row of result.rows) {
+          const _a2 = row, { hybrid_score, id } = _a2, rest = __objRest(_a2, ["hybrid_score", "id"]);
+          delete rest.embedding;
+          delete rest.vector_score;
+          delete rest.keyword_score;
+          delete rest.exact_name_score;
+          const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
+` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
+          tableResults.push({
+            id: `${table}-${id}`,
+            score: parseFloat(String(hybrid_score)),
+            content,
+            metadata: __spreadProps(__spreadValues({}, rest), { source_table: table })
+          });
+        }
+        return tableResults;
+      } catch (err) {
+        console.error(
+          `[MultiTablePostgresProvider] Error querying table "${table}":`,
+          err.message
+        );
+        return [];
+      }
+    });
+    const resultsArray = await Promise.all(queryPromises);
+    for (const tableResults of resultsArray) {
+      allResults.push(...tableResults);
+    }
+    allResults.sort((a, b) => b.score - a.score);
+    if (allResults.length === 0) return [];
+    const bestMatchTable = (_a = allResults[0].metadata) == null ? void 0 : _a.source_table;
+    const finalSorted = allResults.filter((res) => {
+      var _a2;
+      return ((_a2 = res.metadata) == null ? void 0 : _a2.source_table) === bestMatchTable;
+    });
+    console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
+    console.log(`[MultiTablePostgresProvider] Final top match from "${bestMatchTable}" with score ${finalSorted[0].score.toFixed(4)}`);
+    console.log(`[MultiTablePostgresProvider] Restricting recommendations strictly to table: "${bestMatchTable}"`);
+    return finalSorted.slice(0, Math.max(topK, 15));
+  }
+  async delete(_id, _namespace) {
+    console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
+  }
+  async deleteNamespace(_namespace) {
+    console.warn("[MultiTablePostgresProvider] deleteNamespace() is a no-op for multi-table mode.");
+  }
+  async ping() {
+    try {
+      await this.pool.query("SELECT 1");
+      return true;
+    } catch (e) {
+      return false;
+    }
+  }
+  async disconnect() {
+    if (this.pool) {
+      await this.pool.end();
+    }
+  }
+};
+
+export {
+  MultiTablePostgresProvider
+};

package/dist/handlers/index.d.mts

@@ -1,3 +1,3 @@
-import '../index-DPsQodME.mjs';
-export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame, l as sseUIFrame } from '../index-Bb2yEopi.mjs';
+import '../ILLMProvider-BOJFz3Na.mjs';
+export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, l as sseObservabilityFrame, j as sseTextFrame, m as sseUIFrame } from '../index-D3V9Et2M.mjs';
 import 'next/server';

package/dist/handlers/index.d.ts

@@ -1,3 +1,3 @@
-import '../index-DPsQodME.js';
-export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame, l as sseUIFrame } from '../index-CkbTzj9J.js';
+import '../ILLMProvider-BOJFz3Na.js';
+export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, l as sseObservabilityFrame, j as sseTextFrame, m as sseUIFrame } from '../index-BwpcaziY.js';
 import 'next/server';