@retrivora-ai/rag-engine 2.5.6 → 2.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/ChromaDBProvider-4RPJN7Y3.mjs +8 -0
  2. package/dist/MilvusProvider-WJLLZ6TW.mjs +8 -0
  3. package/dist/MongoDBProvider-QSERELZX.mjs +8 -0
  4. package/dist/MultiTablePostgresProvider-JF2HEUDZ.mjs +9 -0
  5. package/dist/PineconeProvider-HLZL7HKU.mjs +10 -0
  6. package/dist/PostgreSQLProvider-JQMCNHRO.mjs +8 -0
  7. package/dist/QdrantProvider-6MNFPEVC.mjs +8 -0
  8. package/dist/RedisProvider-ERQL6AEB.mjs +8 -0
  9. package/dist/SimpleGraphProvider-OP3HAJJM.mjs +70 -0
  10. package/dist/UniversalVectorProvider-INM3KH5F.mjs +9 -0
  11. package/dist/WeaviateProvider-VHWWICCC.mjs +8 -0
  12. package/dist/chunk-366EFBZC.mjs +165 -0
  13. package/dist/chunk-6MGVQTU4.mjs +8883 -0
  14. package/dist/chunk-B4AXIUM7.mjs +49 -0
  15. package/dist/chunk-BEPLYQBS.mjs +89 -0
  16. package/dist/chunk-BNOPQCZW.mjs +238 -0
  17. package/dist/chunk-FJ43J4FW.mjs +654 -0
  18. package/dist/chunk-GP2AJQRQ.mjs +107 -0
  19. package/dist/chunk-IMP6FUCY.mjs +30 -0
  20. package/dist/chunk-JRQWSJNF.mjs +202 -0
  21. package/dist/chunk-LYHCKUNF.mjs +112 -0
  22. package/dist/chunk-MFIWWUC7.mjs +152 -0
  23. package/dist/chunk-MWICNCB2.mjs +157 -0
  24. package/dist/chunk-N54YTFBN.mjs +76 -0
  25. package/dist/chunk-POOWU2HR.mjs +211 -0
  26. package/dist/chunk-QGDGPOZH.mjs +217 -0
  27. package/dist/chunk-QI3FTN6G.mjs +281 -0
  28. package/dist/chunk-QJIDGVG6.mjs +334 -0
  29. package/dist/chunk-W2DKVD3G.mjs +81 -0
  30. package/dist/chunk-XHSDHPKI.mjs +336 -0
  31. package/dist/handlers/index.js +1 -1
  32. package/dist/handlers/index.mjs +24 -12052
  33. package/dist/{index-Byv5MaYi.d.mts → index-9h_ENWQn.d.mts} +16 -0
  34. package/dist/{index-DP3_OpuB.d.ts → index-Xci6GfpI.d.ts} +16 -0
  35. package/dist/index.css +112 -0
  36. package/dist/index.d.mts +176 -18
  37. package/dist/index.d.ts +176 -18
  38. package/dist/index.js +602 -104
  39. package/dist/index.min.css +1 -1
  40. package/dist/index.mjs +648 -1534
  41. package/dist/server.d.mts +2 -2
  42. package/dist/server.d.ts +2 -2
  43. package/dist/server.js +1 -1
  44. package/dist/server.mjs +372 -12608
  45. package/package.json +2 -2
@@ -0,0 +1,336 @@
1
+ import {
2
+ FIELD_SYNONYMS
3
+ } from "./chunk-BEPLYQBS.mjs";
4
+ import {
5
+ BaseVectorProvider
6
+ } from "./chunk-IMP6FUCY.mjs";
7
+ import {
8
+ __objRest,
9
+ __spreadProps,
10
+ __spreadValues
11
+ } from "./chunk-N54YTFBN.mjs";
12
+
13
+ // src/providers/vectordb/MultiTablePostgresProvider.ts
14
+ import { Pool } from "pg";
15
+ var MultiTablePostgresProvider = class extends BaseVectorProvider {
16
+ constructor(config) {
17
+ var _a, _b, _c;
18
+ super(config);
19
+ const opts = config.options || {};
20
+ if (!opts.connectionString) {
21
+ throw new Error("[MultiTablePostgresProvider] options.connectionString is required");
22
+ }
23
+ this.connectionString = opts.connectionString;
24
+ this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
25
+ this.tables = [];
26
+ const rawSearchFields = (_c = (_b = opts.searchFields) != null ? _b : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _c : ["name", "product_name", "productname", "title"];
27
+ this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
28
+ this.uploadTable = opts.uploadTable || "document_chunks";
29
+ }
30
+ async initialize() {
31
+ this.pool = new Pool({ connectionString: this.connectionString });
32
+ const client = await this.pool.connect();
33
+ try {
34
+ await client.query("CREATE EXTENSION IF NOT EXISTS vector");
35
+ await client.query(`
36
+ CREATE TABLE IF NOT EXISTS ${this.uploadTable} (
37
+ id TEXT PRIMARY KEY,
38
+ namespace TEXT NOT NULL DEFAULT '',
39
+ content TEXT NOT NULL,
40
+ metadata JSONB,
41
+ embedding VECTOR(${this.dimensions})
42
+ )
43
+ `);
44
+ await client.query(`
45
+ CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
46
+ ON ${this.uploadTable}
47
+ USING hnsw (embedding vector_cosine_ops)
48
+ `);
49
+ await client.query(`
50
+ CREATE INDEX IF NOT EXISTS "${this.uploadTable}_fts_idx"
51
+ ON "${this.uploadTable}"
52
+ USING gin (to_tsvector('english', content))
53
+ `);
54
+ const res = await client.query(`
55
+ SELECT table_name
56
+ FROM information_schema.columns
57
+ WHERE column_name = 'embedding'
58
+ AND table_schema = 'public'
59
+ `);
60
+ this.tables = res.rows.map((r) => r.table_name);
61
+ if (this.tables.length === 0) {
62
+ console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
63
+ }
64
+ } finally {
65
+ client.release();
66
+ }
67
+ }
68
+ /**
69
+ * Upsert a document by dynamically provisioning a table and columns.
70
+ */
71
+ async upsert(doc, namespace = "") {
72
+ await this.batchUpsert([doc], namespace);
73
+ }
74
+ /**
75
+ * Batch upsert documents by dynamically provisioning tables and columns based on CSV headers.
76
+ */
77
+ async batchUpsert(docs, namespace = "") {
78
+ var _a;
79
+ if (docs.length === 0) return;
80
+ const docsByFile = {};
81
+ for (const doc of docs) {
82
+ const fileName = ((_a = doc.metadata) == null ? void 0 : _a.fileName) || this.uploadTable;
83
+ if (!docsByFile[fileName]) docsByFile[fileName] = [];
84
+ docsByFile[fileName].push(doc);
85
+ }
86
+ const client = await this.pool.connect();
87
+ try {
88
+ await client.query("BEGIN");
89
+ for (const [fileName, fileDocs] of Object.entries(docsByFile)) {
90
+ let tableName = fileName;
91
+ if (tableName.toLowerCase().endsWith(".csv")) {
92
+ tableName = tableName.slice(0, -4);
93
+ }
94
+ tableName = tableName.replace(/[^a-z0-9_]/gi, "_").toLowerCase() || this.uploadTable;
95
+ const firstMeta = fileDocs[0].metadata || {};
96
+ let csvHeaders = [];
97
+ if (Array.isArray(firstMeta.csvHeaders)) {
98
+ csvHeaders = firstMeta.csvHeaders;
99
+ } else {
100
+ const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding", "docId", "docid", "chunkId", "chunkid", "csvHeaders", "csvheaders"];
101
+ csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
102
+ }
103
+ const columnDefs = csvHeaders.map((h) => `"${h}" TEXT`).join(",\n ");
104
+ const createTableSql = `
105
+ CREATE TABLE IF NOT EXISTS "${tableName}" (
106
+ id TEXT PRIMARY KEY,
107
+ ${columnDefs ? columnDefs + "," : ""}
108
+ namespace TEXT NOT NULL DEFAULT '',
109
+ content TEXT NOT NULL,
110
+ metadata JSONB,
111
+ embedding VECTOR(${this.dimensions})
112
+ )
113
+ `;
114
+ await client.query(createTableSql);
115
+ await client.query(`
116
+ CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
117
+ ON "${tableName}"
118
+ USING hnsw (embedding vector_cosine_ops)
119
+ `);
120
+ await client.query(`
121
+ CREATE INDEX IF NOT EXISTS "${tableName}_fts_idx"
122
+ ON "${tableName}"
123
+ USING gin (to_tsvector('english', content))
124
+ `);
125
+ if (!this.tables.includes(tableName)) {
126
+ this.tables.push(tableName);
127
+ }
128
+ const BATCH_SIZE = 50;
129
+ for (let i = 0; i < fileDocs.length; i += BATCH_SIZE) {
130
+ const batch = fileDocs.slice(i, i + BATCH_SIZE);
131
+ const values = [];
132
+ const headerColumns = csvHeaders.map((h) => `"${h}"`).join(", ");
133
+ const allColumns = `id, ${headerColumns ? headerColumns + ", " : ""}namespace, content, metadata, embedding`;
134
+ const valuePlaceholders = batch.map((doc, idx) => {
135
+ var _a2, _b;
136
+ const offset = idx * (5 + csvHeaders.length);
137
+ values.push(doc.id);
138
+ for (const h of csvHeaders) {
139
+ values.push(String(((_a2 = doc.metadata) == null ? void 0 : _a2[h]) || ""));
140
+ }
141
+ values.push(namespace, doc.content, JSON.stringify((_b = doc.metadata) != null ? _b : {}), `[${doc.vector.join(",")}]`);
142
+ const docPlaceholders = [];
143
+ for (let j = 1; j <= 5 + csvHeaders.length; j++) {
144
+ if (j === 5 + csvHeaders.length) {
145
+ docPlaceholders.push(`$${offset + j}::vector`);
146
+ } else {
147
+ docPlaceholders.push(`$${offset + j}`);
148
+ }
149
+ }
150
+ return `(${docPlaceholders.join(", ")})`;
151
+ }).join(", ");
152
+ const conflictUpdates = csvHeaders.map((h) => `"${h}" = EXCLUDED."${h}"`).join(",\n ");
153
+ const query = `
154
+ INSERT INTO "${tableName}" (${allColumns})
155
+ VALUES ${valuePlaceholders}
156
+ ON CONFLICT (id) DO UPDATE
157
+ SET ${conflictUpdates ? conflictUpdates + "," : ""}
158
+ namespace = EXCLUDED.namespace,
159
+ content = EXCLUDED.content,
160
+ metadata = EXCLUDED.metadata,
161
+ embedding = EXCLUDED.embedding
162
+ `;
163
+ await client.query(query, values);
164
+ }
165
+ }
166
+ await client.query("COMMIT");
167
+ } catch (error) {
168
+ await client.query("ROLLBACK");
169
+ throw error;
170
+ } finally {
171
+ client.release();
172
+ }
173
+ }
174
+ /**
175
+ * Query all configured tables and merge results, sorted by cosine similarity score.
176
+ */
177
+ async query(vector, topK, _namespace, _filter) {
178
+ var _a;
179
+ if (!this.pool) {
180
+ throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
181
+ }
182
+ const vectorLiteral = `[${vector.join(",")}]`;
183
+ const allResults = [];
184
+ const queryText = _filter == null ? void 0 : _filter.queryText;
185
+ const entityHints = Array.isArray(_filter == null ? void 0 : _filter.keywords) ? _filter.keywords.map((k) => k.trim().toLowerCase()).filter(Boolean) : [];
186
+ const getDynamicKeywordQuery = () => {
187
+ if (entityHints.length > 0) {
188
+ return entityHints.map((h) => h.replace(/\s+/g, " & ")).join(" & ");
189
+ }
190
+ if (queryText) {
191
+ return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|organization[s]?|comp(?:any|anies)|of|category|find|list|browse|what|are|display|all|whose|that|which|have|has|having|greater|than|more|less|equal|equals|above|below|over|under|at|least|most|for|in|with|about|get|details|information|info)\b/g, "").replace(/\b\d+(?:\.\d+)?\b/g, "").trim().replace(/\s+/g, " & ");
192
+ }
193
+ return "";
194
+ };
195
+ const dynamicKeywordQuery = getDynamicKeywordQuery();
196
+ const tableLimit = Math.max(topK, 50);
197
+ const metadataFilters = _filter == null ? void 0 : _filter.metadata;
198
+ const queryPromises = this.tables.map(async (table) => {
199
+ try {
200
+ let sqlQuery = "";
201
+ let params = [];
202
+ let whereClause = "";
203
+ const filterParams = [];
204
+ if (metadataFilters && Object.keys(metadataFilters).length > 0) {
205
+ const conditions = Object.entries(metadataFilters).map(([key, val]) => {
206
+ var _a3;
207
+ filterParams.push(String(val));
208
+ const baseOffset = queryText && dynamicKeywordQuery ? 2 : 1;
209
+ const paramIdx = baseOffset + filterParams.length;
210
+ const synonyms = (_a3 = FIELD_SYNONYMS[key]) != null ? _a3 : [];
211
+ const keysToCheck = [key, ...synonyms];
212
+ const coalesceExprs = keysToCheck.flatMap((k) => [
213
+ `metadata->>'${k}'`,
214
+ `to_jsonb(t)->>'${k}'`
215
+ ]);
216
+ return `COALESCE(${coalesceExprs.map((expr) => `LOWER(${expr})`).join(", ")}) = LOWER($${paramIdx})`;
217
+ });
218
+ whereClause = `WHERE ${conditions.join(" AND ")}`;
219
+ }
220
+ if (queryText && dynamicKeywordQuery) {
221
+ const hasEntityHints = entityHints.length > 0;
222
+ const exactNameScoreExpr = hasEntityHints ? `+ (
223
+ SELECT COALESCE(MAX(
224
+ CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(", ")})
225
+ THEN 1.0 ELSE 0.0 END
226
+ ), 0)
227
+ FROM jsonb_each_text(COALESCE(v.metadata, k.metadata)) AS kv(key, val)
228
+ WHERE key IN (${this.searchFields.map((f) => `'${f}'`).join(", ")})
229
+ ) * 3.0` : "";
230
+ sqlQuery = `
231
+ WITH vector_search AS (
232
+ SELECT *, (1 - (embedding <=> $1::vector)) AS vector_score
233
+ FROM "${table}" t
234
+ ${whereClause}
235
+ ORDER BY embedding <=> $1::vector ASC
236
+ LIMIT ${tableLimit}
237
+ ),
238
+ keyword_search AS (
239
+ SELECT *, COALESCE(ts_rank(to_tsvector('english', content), to_tsquery('english', $2)), 0) AS keyword_score
240
+ FROM "${table}" t
241
+ WHERE ${whereClause ? whereClause.replace("WHERE", "") : "TRUE"}
242
+ AND to_tsvector('english', content) @@ to_tsquery('english', $2)
243
+ ORDER BY keyword_score DESC
244
+ LIMIT ${tableLimit}
245
+ )
246
+ SELECT
247
+ COALESCE(v.id, k.id) AS id,
248
+ COALESCE(v.namespace, k.namespace) AS namespace,
249
+ COALESCE(v.content, k.content) AS content,
250
+ COALESCE(v.metadata, k.metadata) AS metadata,
251
+ COALESCE(v.vector_score, 0) AS vector_score,
252
+ COALESCE(k.keyword_score, 0) AS keyword_score,
253
+ (COALESCE(v.vector_score, 0) + COALESCE(k.keyword_score, 0) * 2.0 ${exactNameScoreExpr}) AS hybrid_score
254
+ FROM vector_search v
255
+ FULL OUTER JOIN keyword_search k ON v.id = k.id
256
+ ORDER BY hybrid_score DESC
257
+ LIMIT ${tableLimit}
258
+ `;
259
+ params = [vectorLiteral, dynamicKeywordQuery, ...filterParams];
260
+ } else {
261
+ sqlQuery = `
262
+ SELECT *,
263
+ (1 - (embedding <=> $1::vector)) AS hybrid_score
264
+ FROM "${table}" t
265
+ ${whereClause}
266
+ ORDER BY embedding <=> $1::vector ASC
267
+ LIMIT ${tableLimit}
268
+ `;
269
+ params = [vectorLiteral, ...filterParams];
270
+ }
271
+ const result = await this.pool.query(sqlQuery, params);
272
+ if (result.rowCount && result.rowCount > 0) {
273
+ } else {
274
+ }
275
+ const tableResults = [];
276
+ for (const row of result.rows) {
277
+ const _a2 = row, { hybrid_score, id } = _a2, rest = __objRest(_a2, ["hybrid_score", "id"]);
278
+ delete rest.embedding;
279
+ delete rest.vector_score;
280
+ delete rest.keyword_score;
281
+ delete rest.exact_name_score;
282
+ const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
283
+ ` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
284
+ tableResults.push({
285
+ id: `${table}-${id}`,
286
+ score: parseFloat(String(hybrid_score)),
287
+ content,
288
+ metadata: __spreadProps(__spreadValues({}, rest), { source_table: table })
289
+ });
290
+ }
291
+ return tableResults;
292
+ } catch (err) {
293
+ console.error(
294
+ `[MultiTablePostgresProvider] Error querying table "${table}":`,
295
+ err.message
296
+ );
297
+ return [];
298
+ }
299
+ });
300
+ const resultsArray = await Promise.all(queryPromises);
301
+ for (const tableResults of resultsArray) {
302
+ allResults.push(...tableResults);
303
+ }
304
+ allResults.sort((a, b) => b.score - a.score);
305
+ if (allResults.length === 0) return [];
306
+ const bestMatchTable = (_a = allResults[0].metadata) == null ? void 0 : _a.source_table;
307
+ const finalSorted = allResults.filter((res) => {
308
+ var _a2;
309
+ return ((_a2 = res.metadata) == null ? void 0 : _a2.source_table) === bestMatchTable;
310
+ });
311
+ return finalSorted.slice(0, Math.max(topK, 15));
312
+ }
313
+ async delete(_id, _namespace) {
314
+ console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
315
+ }
316
+ async deleteNamespace(_namespace) {
317
+ console.warn("[MultiTablePostgresProvider] deleteNamespace() is a no-op for multi-table mode.");
318
+ }
319
+ async ping() {
320
+ try {
321
+ await this.pool.query("SELECT 1");
322
+ return true;
323
+ } catch (e) {
324
+ return false;
325
+ }
326
+ }
327
+ async disconnect() {
328
+ if (this.pool) {
329
+ await this.pool.end();
330
+ }
331
+ }
332
+ };
333
+
334
+ export {
335
+ MultiTablePostgresProvider
336
+ };
@@ -318,7 +318,7 @@ var init_package = __esm({
318
318
  "package.json"() {
319
319
  package_default = {
320
320
  name: "@retrivora-ai/rag-engine",
321
- version: "2.5.6",
321
+ version: "2.5.7",
322
322
  description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
323
323
  author: "Abhinav Alkuchi",
324
324
  license: "UNLICENSED",