@retrivora-ai/rag-engine 2.5.7 → 2.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/{ILLMProvider-DwFj0h2u.d.mts → ILLMProvider-B2N3lMBc.d.mts} +10 -1
  2. package/dist/{ILLMProvider-DwFj0h2u.d.ts → ILLMProvider-B2N3lMBc.d.ts} +10 -1
  3. package/dist/handlers/index.d.mts +2 -2
  4. package/dist/handlers/index.d.ts +2 -2
  5. package/dist/handlers/index.js +961 -192
  6. package/dist/handlers/index.mjs +12821 -24
  7. package/dist/{index-Cb0GsT16.d.ts → index-37V810a-.d.mts} +3 -3
  8. package/dist/{index-D1jN8KSF.d.mts → index-BxzDv2jd.d.ts} +3 -3
  9. package/dist/{index-9h_ENWQn.d.mts → index-CAGGPyI7.d.mts} +6 -2
  10. package/dist/{index-Xci6GfpI.d.ts → index-DdQxoIpu.d.ts} +6 -2
  11. package/dist/index.css +76 -4
  12. package/dist/index.d.mts +6 -6
  13. package/dist/index.d.ts +6 -6
  14. package/dist/index.js +136 -94
  15. package/dist/index.min.css +1 -1
  16. package/dist/index.mjs +1553 -129
  17. package/dist/server.d.mts +100 -9
  18. package/dist/server.d.ts +100 -9
  19. package/dist/server.js +963 -192
  20. package/dist/server.mjs +13378 -372
  21. package/package.json +5 -2
  22. package/dist/ChromaDBProvider-4RPJN7Y3.mjs +0 -8
  23. package/dist/MilvusProvider-WJLLZ6TW.mjs +0 -8
  24. package/dist/MongoDBProvider-QSERELZX.mjs +0 -8
  25. package/dist/MultiTablePostgresProvider-JF2HEUDZ.mjs +0 -9
  26. package/dist/PineconeProvider-HLZL7HKU.mjs +0 -10
  27. package/dist/PostgreSQLProvider-JQMCNHRO.mjs +0 -8
  28. package/dist/QdrantProvider-6MNFPEVC.mjs +0 -8
  29. package/dist/RedisProvider-ERQL6AEB.mjs +0 -8
  30. package/dist/SimpleGraphProvider-OP3HAJJM.mjs +0 -70
  31. package/dist/UniversalVectorProvider-INM3KH5F.mjs +0 -9
  32. package/dist/WeaviateProvider-VHWWICCC.mjs +0 -8
  33. package/dist/chunk-366EFBZC.mjs +0 -165
  34. package/dist/chunk-6MGVQTU4.mjs +0 -8883
  35. package/dist/chunk-B4AXIUM7.mjs +0 -49
  36. package/dist/chunk-BEPLYQBS.mjs +0 -89
  37. package/dist/chunk-BNOPQCZW.mjs +0 -238
  38. package/dist/chunk-FJ43J4FW.mjs +0 -654
  39. package/dist/chunk-GP2AJQRQ.mjs +0 -107
  40. package/dist/chunk-IMP6FUCY.mjs +0 -30
  41. package/dist/chunk-JRQWSJNF.mjs +0 -202
  42. package/dist/chunk-LYHCKUNF.mjs +0 -112
  43. package/dist/chunk-MFIWWUC7.mjs +0 -152
  44. package/dist/chunk-MWICNCB2.mjs +0 -157
  45. package/dist/chunk-N54YTFBN.mjs +0 -76
  46. package/dist/chunk-POOWU2HR.mjs +0 -211
  47. package/dist/chunk-QGDGPOZH.mjs +0 -217
  48. package/dist/chunk-QI3FTN6G.mjs +0 -281
  49. package/dist/chunk-QJIDGVG6.mjs +0 -334
  50. package/dist/chunk-W2DKVD3G.mjs +0 -81
  51. package/dist/chunk-XHSDHPKI.mjs +0 -336
@@ -1,336 +0,0 @@
1
- import {
2
- FIELD_SYNONYMS
3
- } from "./chunk-BEPLYQBS.mjs";
4
- import {
5
- BaseVectorProvider
6
- } from "./chunk-IMP6FUCY.mjs";
7
- import {
8
- __objRest,
9
- __spreadProps,
10
- __spreadValues
11
- } from "./chunk-N54YTFBN.mjs";
12
-
13
- // src/providers/vectordb/MultiTablePostgresProvider.ts
14
- import { Pool } from "pg";
15
- var MultiTablePostgresProvider = class extends BaseVectorProvider {
16
- constructor(config) {
17
- var _a, _b, _c;
18
- super(config);
19
- const opts = config.options || {};
20
- if (!opts.connectionString) {
21
- throw new Error("[MultiTablePostgresProvider] options.connectionString is required");
22
- }
23
- this.connectionString = opts.connectionString;
24
- this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
25
- this.tables = [];
26
- const rawSearchFields = (_c = (_b = opts.searchFields) != null ? _b : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _c : ["name", "product_name", "productname", "title"];
27
- this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
28
- this.uploadTable = opts.uploadTable || "document_chunks";
29
- }
30
- async initialize() {
31
- this.pool = new Pool({ connectionString: this.connectionString });
32
- const client = await this.pool.connect();
33
- try {
34
- await client.query("CREATE EXTENSION IF NOT EXISTS vector");
35
- await client.query(`
36
- CREATE TABLE IF NOT EXISTS ${this.uploadTable} (
37
- id TEXT PRIMARY KEY,
38
- namespace TEXT NOT NULL DEFAULT '',
39
- content TEXT NOT NULL,
40
- metadata JSONB,
41
- embedding VECTOR(${this.dimensions})
42
- )
43
- `);
44
- await client.query(`
45
- CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
46
- ON ${this.uploadTable}
47
- USING hnsw (embedding vector_cosine_ops)
48
- `);
49
- await client.query(`
50
- CREATE INDEX IF NOT EXISTS "${this.uploadTable}_fts_idx"
51
- ON "${this.uploadTable}"
52
- USING gin (to_tsvector('english', content))
53
- `);
54
- const res = await client.query(`
55
- SELECT table_name
56
- FROM information_schema.columns
57
- WHERE column_name = 'embedding'
58
- AND table_schema = 'public'
59
- `);
60
- this.tables = res.rows.map((r) => r.table_name);
61
- if (this.tables.length === 0) {
62
- console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
63
- }
64
- } finally {
65
- client.release();
66
- }
67
- }
68
- /**
69
- * Upsert a document by dynamically provisioning a table and columns.
70
- */
71
- async upsert(doc, namespace = "") {
72
- await this.batchUpsert([doc], namespace);
73
- }
74
- /**
75
- * Batch upsert documents by dynamically provisioning tables and columns based on CSV headers.
76
- */
77
- async batchUpsert(docs, namespace = "") {
78
- var _a;
79
- if (docs.length === 0) return;
80
- const docsByFile = {};
81
- for (const doc of docs) {
82
- const fileName = ((_a = doc.metadata) == null ? void 0 : _a.fileName) || this.uploadTable;
83
- if (!docsByFile[fileName]) docsByFile[fileName] = [];
84
- docsByFile[fileName].push(doc);
85
- }
86
- const client = await this.pool.connect();
87
- try {
88
- await client.query("BEGIN");
89
- for (const [fileName, fileDocs] of Object.entries(docsByFile)) {
90
- let tableName = fileName;
91
- if (tableName.toLowerCase().endsWith(".csv")) {
92
- tableName = tableName.slice(0, -4);
93
- }
94
- tableName = tableName.replace(/[^a-z0-9_]/gi, "_").toLowerCase() || this.uploadTable;
95
- const firstMeta = fileDocs[0].metadata || {};
96
- let csvHeaders = [];
97
- if (Array.isArray(firstMeta.csvHeaders)) {
98
- csvHeaders = firstMeta.csvHeaders;
99
- } else {
100
- const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding", "docId", "docid", "chunkId", "chunkid", "csvHeaders", "csvheaders"];
101
- csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
102
- }
103
- const columnDefs = csvHeaders.map((h) => `"${h}" TEXT`).join(",\n ");
104
- const createTableSql = `
105
- CREATE TABLE IF NOT EXISTS "${tableName}" (
106
- id TEXT PRIMARY KEY,
107
- ${columnDefs ? columnDefs + "," : ""}
108
- namespace TEXT NOT NULL DEFAULT '',
109
- content TEXT NOT NULL,
110
- metadata JSONB,
111
- embedding VECTOR(${this.dimensions})
112
- )
113
- `;
114
- await client.query(createTableSql);
115
- await client.query(`
116
- CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
117
- ON "${tableName}"
118
- USING hnsw (embedding vector_cosine_ops)
119
- `);
120
- await client.query(`
121
- CREATE INDEX IF NOT EXISTS "${tableName}_fts_idx"
122
- ON "${tableName}"
123
- USING gin (to_tsvector('english', content))
124
- `);
125
- if (!this.tables.includes(tableName)) {
126
- this.tables.push(tableName);
127
- }
128
- const BATCH_SIZE = 50;
129
- for (let i = 0; i < fileDocs.length; i += BATCH_SIZE) {
130
- const batch = fileDocs.slice(i, i + BATCH_SIZE);
131
- const values = [];
132
- const headerColumns = csvHeaders.map((h) => `"${h}"`).join(", ");
133
- const allColumns = `id, ${headerColumns ? headerColumns + ", " : ""}namespace, content, metadata, embedding`;
134
- const valuePlaceholders = batch.map((doc, idx) => {
135
- var _a2, _b;
136
- const offset = idx * (5 + csvHeaders.length);
137
- values.push(doc.id);
138
- for (const h of csvHeaders) {
139
- values.push(String(((_a2 = doc.metadata) == null ? void 0 : _a2[h]) || ""));
140
- }
141
- values.push(namespace, doc.content, JSON.stringify((_b = doc.metadata) != null ? _b : {}), `[${doc.vector.join(",")}]`);
142
- const docPlaceholders = [];
143
- for (let j = 1; j <= 5 + csvHeaders.length; j++) {
144
- if (j === 5 + csvHeaders.length) {
145
- docPlaceholders.push(`$${offset + j}::vector`);
146
- } else {
147
- docPlaceholders.push(`$${offset + j}`);
148
- }
149
- }
150
- return `(${docPlaceholders.join(", ")})`;
151
- }).join(", ");
152
- const conflictUpdates = csvHeaders.map((h) => `"${h}" = EXCLUDED."${h}"`).join(",\n ");
153
- const query = `
154
- INSERT INTO "${tableName}" (${allColumns})
155
- VALUES ${valuePlaceholders}
156
- ON CONFLICT (id) DO UPDATE
157
- SET ${conflictUpdates ? conflictUpdates + "," : ""}
158
- namespace = EXCLUDED.namespace,
159
- content = EXCLUDED.content,
160
- metadata = EXCLUDED.metadata,
161
- embedding = EXCLUDED.embedding
162
- `;
163
- await client.query(query, values);
164
- }
165
- }
166
- await client.query("COMMIT");
167
- } catch (error) {
168
- await client.query("ROLLBACK");
169
- throw error;
170
- } finally {
171
- client.release();
172
- }
173
- }
174
- /**
175
- * Query all configured tables and merge results, sorted by cosine similarity score.
176
- */
177
- async query(vector, topK, _namespace, _filter) {
178
- var _a;
179
- if (!this.pool) {
180
- throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
181
- }
182
- const vectorLiteral = `[${vector.join(",")}]`;
183
- const allResults = [];
184
- const queryText = _filter == null ? void 0 : _filter.queryText;
185
- const entityHints = Array.isArray(_filter == null ? void 0 : _filter.keywords) ? _filter.keywords.map((k) => k.trim().toLowerCase()).filter(Boolean) : [];
186
- const getDynamicKeywordQuery = () => {
187
- if (entityHints.length > 0) {
188
- return entityHints.map((h) => h.replace(/\s+/g, " & ")).join(" & ");
189
- }
190
- if (queryText) {
191
- return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|organization[s]?|comp(?:any|anies)|of|category|find|list|browse|what|are|display|all|whose|that|which|have|has|having|greater|than|more|less|equal|equals|above|below|over|under|at|least|most|for|in|with|about|get|details|information|info)\b/g, "").replace(/\b\d+(?:\.\d+)?\b/g, "").trim().replace(/\s+/g, " & ");
192
- }
193
- return "";
194
- };
195
- const dynamicKeywordQuery = getDynamicKeywordQuery();
196
- const tableLimit = Math.max(topK, 50);
197
- const metadataFilters = _filter == null ? void 0 : _filter.metadata;
198
- const queryPromises = this.tables.map(async (table) => {
199
- try {
200
- let sqlQuery = "";
201
- let params = [];
202
- let whereClause = "";
203
- const filterParams = [];
204
- if (metadataFilters && Object.keys(metadataFilters).length > 0) {
205
- const conditions = Object.entries(metadataFilters).map(([key, val]) => {
206
- var _a3;
207
- filterParams.push(String(val));
208
- const baseOffset = queryText && dynamicKeywordQuery ? 2 : 1;
209
- const paramIdx = baseOffset + filterParams.length;
210
- const synonyms = (_a3 = FIELD_SYNONYMS[key]) != null ? _a3 : [];
211
- const keysToCheck = [key, ...synonyms];
212
- const coalesceExprs = keysToCheck.flatMap((k) => [
213
- `metadata->>'${k}'`,
214
- `to_jsonb(t)->>'${k}'`
215
- ]);
216
- return `COALESCE(${coalesceExprs.map((expr) => `LOWER(${expr})`).join(", ")}) = LOWER($${paramIdx})`;
217
- });
218
- whereClause = `WHERE ${conditions.join(" AND ")}`;
219
- }
220
- if (queryText && dynamicKeywordQuery) {
221
- const hasEntityHints = entityHints.length > 0;
222
- const exactNameScoreExpr = hasEntityHints ? `+ (
223
- SELECT COALESCE(MAX(
224
- CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(", ")})
225
- THEN 1.0 ELSE 0.0 END
226
- ), 0)
227
- FROM jsonb_each_text(COALESCE(v.metadata, k.metadata)) AS kv(key, val)
228
- WHERE key IN (${this.searchFields.map((f) => `'${f}'`).join(", ")})
229
- ) * 3.0` : "";
230
- sqlQuery = `
231
- WITH vector_search AS (
232
- SELECT *, (1 - (embedding <=> $1::vector)) AS vector_score
233
- FROM "${table}" t
234
- ${whereClause}
235
- ORDER BY embedding <=> $1::vector ASC
236
- LIMIT ${tableLimit}
237
- ),
238
- keyword_search AS (
239
- SELECT *, COALESCE(ts_rank(to_tsvector('english', content), to_tsquery('english', $2)), 0) AS keyword_score
240
- FROM "${table}" t
241
- WHERE ${whereClause ? whereClause.replace("WHERE", "") : "TRUE"}
242
- AND to_tsvector('english', content) @@ to_tsquery('english', $2)
243
- ORDER BY keyword_score DESC
244
- LIMIT ${tableLimit}
245
- )
246
- SELECT
247
- COALESCE(v.id, k.id) AS id,
248
- COALESCE(v.namespace, k.namespace) AS namespace,
249
- COALESCE(v.content, k.content) AS content,
250
- COALESCE(v.metadata, k.metadata) AS metadata,
251
- COALESCE(v.vector_score, 0) AS vector_score,
252
- COALESCE(k.keyword_score, 0) AS keyword_score,
253
- (COALESCE(v.vector_score, 0) + COALESCE(k.keyword_score, 0) * 2.0 ${exactNameScoreExpr}) AS hybrid_score
254
- FROM vector_search v
255
- FULL OUTER JOIN keyword_search k ON v.id = k.id
256
- ORDER BY hybrid_score DESC
257
- LIMIT ${tableLimit}
258
- `;
259
- params = [vectorLiteral, dynamicKeywordQuery, ...filterParams];
260
- } else {
261
- sqlQuery = `
262
- SELECT *,
263
- (1 - (embedding <=> $1::vector)) AS hybrid_score
264
- FROM "${table}" t
265
- ${whereClause}
266
- ORDER BY embedding <=> $1::vector ASC
267
- LIMIT ${tableLimit}
268
- `;
269
- params = [vectorLiteral, ...filterParams];
270
- }
271
- const result = await this.pool.query(sqlQuery, params);
272
- if (result.rowCount && result.rowCount > 0) {
273
- } else {
274
- }
275
- const tableResults = [];
276
- for (const row of result.rows) {
277
- const _a2 = row, { hybrid_score, id } = _a2, rest = __objRest(_a2, ["hybrid_score", "id"]);
278
- delete rest.embedding;
279
- delete rest.vector_score;
280
- delete rest.keyword_score;
281
- delete rest.exact_name_score;
282
- const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
283
- ` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
284
- tableResults.push({
285
- id: `${table}-${id}`,
286
- score: parseFloat(String(hybrid_score)),
287
- content,
288
- metadata: __spreadProps(__spreadValues({}, rest), { source_table: table })
289
- });
290
- }
291
- return tableResults;
292
- } catch (err) {
293
- console.error(
294
- `[MultiTablePostgresProvider] Error querying table "${table}":`,
295
- err.message
296
- );
297
- return [];
298
- }
299
- });
300
- const resultsArray = await Promise.all(queryPromises);
301
- for (const tableResults of resultsArray) {
302
- allResults.push(...tableResults);
303
- }
304
- allResults.sort((a, b) => b.score - a.score);
305
- if (allResults.length === 0) return [];
306
- const bestMatchTable = (_a = allResults[0].metadata) == null ? void 0 : _a.source_table;
307
- const finalSorted = allResults.filter((res) => {
308
- var _a2;
309
- return ((_a2 = res.metadata) == null ? void 0 : _a2.source_table) === bestMatchTable;
310
- });
311
- return finalSorted.slice(0, Math.max(topK, 15));
312
- }
313
- async delete(_id, _namespace) {
314
- console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
315
- }
316
- async deleteNamespace(_namespace) {
317
- console.warn("[MultiTablePostgresProvider] deleteNamespace() is a no-op for multi-table mode.");
318
- }
319
- async ping() {
320
- try {
321
- await this.pool.query("SELECT 1");
322
- return true;
323
- } catch (e) {
324
- return false;
325
- }
326
- }
327
- async disconnect() {
328
- if (this.pool) {
329
- await this.pool.end();
330
- }
331
- }
332
- };
333
-
334
- export {
335
- MultiTablePostgresProvider
336
- };