@retrivora-ai/rag-engine 2.6.2 → 2.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ChromaDBProvider-NDA2IBOH.mjs +8 -0
- package/dist/MilvusProvider-H6GGL5FQ.mjs +8 -0
- package/dist/MongoDBProvider-KC3GF3FH.mjs +8 -0
- package/dist/MultiTablePostgresProvider-6LJBFZL2.mjs +9 -0
- package/dist/PineconeProvider-3PXQFHSO.mjs +10 -0
- package/dist/PostgreSQLProvider-VWFL4OIP.mjs +8 -0
- package/dist/QdrantProvider-UU7RGXBI.mjs +8 -0
- package/dist/RedisProvider-MIIPSAAE.mjs +8 -0
- package/dist/SimpleGraphProvider-HJIS4S7Y.mjs +70 -0
- package/dist/UniversalVectorProvider-NO3L255G.mjs +9 -0
- package/dist/WeaviateProvider-YRZDZFAC.mjs +8 -0
- package/dist/chunk-23PFNEBF.mjs +107 -0
- package/dist/chunk-3SQ4DODU.mjs +238 -0
- package/dist/chunk-6ZZOZFK6.mjs +334 -0
- package/dist/chunk-AHMMUZ4W.mjs +152 -0
- package/dist/chunk-ASBNNEVI.mjs +81 -0
- package/dist/chunk-BEPLYQBS.mjs +89 -0
- package/dist/chunk-BPIATXJ4.mjs +281 -0
- package/dist/chunk-FMQN2K2D.mjs +168 -0
- package/dist/chunk-G7MJRFCO.mjs +340 -0
- package/dist/chunk-IHWKVAUR.mjs +112 -0
- package/dist/chunk-IMP6FUCY.mjs +30 -0
- package/dist/chunk-MAXMVO7B.mjs +211 -0
- package/dist/chunk-MGJ6V2IT.mjs +203 -0
- package/dist/chunk-OMQM7AN6.mjs +219 -0
- package/dist/chunk-SG6XYLQE.mjs +53 -0
- package/dist/chunk-T7XSD76K.mjs +10851 -0
- package/dist/chunk-UT6SZZEC.mjs +69 -0
- package/dist/chunk-VW3U4MMX.mjs +157 -0
- package/dist/chunk-XYBKDJDW.mjs +696 -0
- package/dist/handlers/index.d.mts +1 -1
- package/dist/handlers/index.d.ts +1 -1
- package/dist/handlers/index.js +56 -23
- package/dist/handlers/index.mjs +24 -14027
- package/dist/{index-D7FuQRSb.d.mts → index-BBbt5A2y.d.mts} +3 -1
- package/dist/{index-CFl4KrKq.d.ts → index-BZ_rR6V8.d.ts} +3 -1
- package/dist/index.css +13 -0
- package/dist/index.js +22 -5
- package/dist/index.min.css +1 -1
- package/dist/index.mjs +56 -1462
- package/dist/server.d.mts +2 -2
- package/dist/server.d.ts +2 -2
- package/dist/server.js +56 -23
- package/dist/server.mjs +377 -14573
- package/package.json +1 -1
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
import {
|
|
2
|
+
FIELD_SYNONYMS
|
|
3
|
+
} from "./chunk-BEPLYQBS.mjs";
|
|
4
|
+
import {
|
|
5
|
+
BaseVectorProvider
|
|
6
|
+
} from "./chunk-IMP6FUCY.mjs";
|
|
7
|
+
import {
|
|
8
|
+
__objRest,
|
|
9
|
+
__spreadProps,
|
|
10
|
+
__spreadValues
|
|
11
|
+
} from "./chunk-UT6SZZEC.mjs";
|
|
12
|
+
|
|
13
|
+
// src/providers/vectordb/MultiTablePostgresProvider.ts
|
|
14
|
+
import { Pool } from "pg";
|
|
15
|
+
var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
16
|
+
constructor(config) {
|
|
17
|
+
var _a, _b, _c;
|
|
18
|
+
super(config);
|
|
19
|
+
const opts = config.options || {};
|
|
20
|
+
if (!opts.connectionString) {
|
|
21
|
+
throw new Error("[MultiTablePostgresProvider] options.connectionString is required");
|
|
22
|
+
}
|
|
23
|
+
this.connectionString = opts.connectionString;
|
|
24
|
+
this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
|
|
25
|
+
this.tables = [];
|
|
26
|
+
const rawSearchFields = (_c = (_b = opts.searchFields) != null ? _b : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _c : ["name", "product_name", "productname", "title"];
|
|
27
|
+
this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
|
|
28
|
+
const rawUploadTable = opts.uploadTable || "document_chunks";
|
|
29
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]{0,62}$/.test(rawUploadTable)) {
|
|
30
|
+
throw new Error(`Invalid uploadTable name: ${rawUploadTable}`);
|
|
31
|
+
}
|
|
32
|
+
this.uploadTable = rawUploadTable;
|
|
33
|
+
}
|
|
34
|
+
async initialize() {
|
|
35
|
+
this.pool = new Pool({ connectionString: this.connectionString });
|
|
36
|
+
const client = await this.pool.connect();
|
|
37
|
+
try {
|
|
38
|
+
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
39
|
+
await client.query(`
|
|
40
|
+
CREATE TABLE IF NOT EXISTS ${this.uploadTable} (
|
|
41
|
+
id TEXT PRIMARY KEY,
|
|
42
|
+
namespace TEXT NOT NULL DEFAULT '',
|
|
43
|
+
content TEXT NOT NULL,
|
|
44
|
+
metadata JSONB,
|
|
45
|
+
embedding VECTOR(${this.dimensions})
|
|
46
|
+
)
|
|
47
|
+
`);
|
|
48
|
+
await client.query(`
|
|
49
|
+
CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
|
|
50
|
+
ON ${this.uploadTable}
|
|
51
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
52
|
+
`);
|
|
53
|
+
await client.query(`
|
|
54
|
+
CREATE INDEX IF NOT EXISTS "${this.uploadTable}_fts_idx"
|
|
55
|
+
ON "${this.uploadTable}"
|
|
56
|
+
USING gin (to_tsvector('english', content))
|
|
57
|
+
`);
|
|
58
|
+
const res = await client.query(`
|
|
59
|
+
SELECT table_name
|
|
60
|
+
FROM information_schema.columns
|
|
61
|
+
WHERE column_name = 'embedding'
|
|
62
|
+
AND table_schema = 'public'
|
|
63
|
+
`);
|
|
64
|
+
this.tables = res.rows.map((r) => r.table_name);
|
|
65
|
+
if (this.tables.length === 0) {
|
|
66
|
+
console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
|
|
67
|
+
}
|
|
68
|
+
} finally {
|
|
69
|
+
client.release();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Upsert a document by dynamically provisioning a table and columns.
|
|
74
|
+
*/
|
|
75
|
+
async upsert(doc, namespace = "") {
|
|
76
|
+
await this.batchUpsert([doc], namespace);
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Batch upsert documents by dynamically provisioning tables and columns based on CSV headers.
|
|
80
|
+
*/
|
|
81
|
+
async batchUpsert(docs, namespace = "") {
|
|
82
|
+
var _a;
|
|
83
|
+
if (docs.length === 0) return;
|
|
84
|
+
const docsByFile = {};
|
|
85
|
+
for (const doc of docs) {
|
|
86
|
+
const fileName = ((_a = doc.metadata) == null ? void 0 : _a.fileName) || this.uploadTable;
|
|
87
|
+
if (!docsByFile[fileName]) docsByFile[fileName] = [];
|
|
88
|
+
docsByFile[fileName].push(doc);
|
|
89
|
+
}
|
|
90
|
+
const client = await this.pool.connect();
|
|
91
|
+
try {
|
|
92
|
+
await client.query("BEGIN");
|
|
93
|
+
for (const [fileName, fileDocs] of Object.entries(docsByFile)) {
|
|
94
|
+
let tableName = fileName;
|
|
95
|
+
if (tableName.toLowerCase().endsWith(".csv")) {
|
|
96
|
+
tableName = tableName.slice(0, -4);
|
|
97
|
+
}
|
|
98
|
+
tableName = tableName.replace(/[^a-z0-9_]/gi, "_").toLowerCase() || this.uploadTable;
|
|
99
|
+
const firstMeta = fileDocs[0].metadata || {};
|
|
100
|
+
let csvHeaders = [];
|
|
101
|
+
if (Array.isArray(firstMeta.csvHeaders)) {
|
|
102
|
+
csvHeaders = firstMeta.csvHeaders;
|
|
103
|
+
} else {
|
|
104
|
+
const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding", "docId", "docid", "chunkId", "chunkid", "csvHeaders", "csvheaders"];
|
|
105
|
+
csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
|
|
106
|
+
}
|
|
107
|
+
const columnDefs = csvHeaders.map((h) => `"${h}" TEXT`).join(",\n ");
|
|
108
|
+
const createTableSql = `
|
|
109
|
+
CREATE TABLE IF NOT EXISTS "${tableName}" (
|
|
110
|
+
id TEXT PRIMARY KEY,
|
|
111
|
+
${columnDefs ? columnDefs + "," : ""}
|
|
112
|
+
namespace TEXT NOT NULL DEFAULT '',
|
|
113
|
+
content TEXT NOT NULL,
|
|
114
|
+
metadata JSONB,
|
|
115
|
+
embedding VECTOR(${this.dimensions})
|
|
116
|
+
)
|
|
117
|
+
`;
|
|
118
|
+
await client.query(createTableSql);
|
|
119
|
+
await client.query(`
|
|
120
|
+
CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
|
|
121
|
+
ON "${tableName}"
|
|
122
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
123
|
+
`);
|
|
124
|
+
await client.query(`
|
|
125
|
+
CREATE INDEX IF NOT EXISTS "${tableName}_fts_idx"
|
|
126
|
+
ON "${tableName}"
|
|
127
|
+
USING gin (to_tsvector('english', content))
|
|
128
|
+
`);
|
|
129
|
+
if (!this.tables.includes(tableName)) {
|
|
130
|
+
this.tables.push(tableName);
|
|
131
|
+
}
|
|
132
|
+
const BATCH_SIZE = 50;
|
|
133
|
+
for (let i = 0; i < fileDocs.length; i += BATCH_SIZE) {
|
|
134
|
+
const batch = fileDocs.slice(i, i + BATCH_SIZE);
|
|
135
|
+
const values = [];
|
|
136
|
+
const headerColumns = csvHeaders.map((h) => `"${h}"`).join(", ");
|
|
137
|
+
const allColumns = `id, ${headerColumns ? headerColumns + ", " : ""}namespace, content, metadata, embedding`;
|
|
138
|
+
const valuePlaceholders = batch.map((doc, idx) => {
|
|
139
|
+
var _a2, _b;
|
|
140
|
+
const offset = idx * (5 + csvHeaders.length);
|
|
141
|
+
values.push(doc.id);
|
|
142
|
+
for (const h of csvHeaders) {
|
|
143
|
+
values.push(String(((_a2 = doc.metadata) == null ? void 0 : _a2[h]) || ""));
|
|
144
|
+
}
|
|
145
|
+
values.push(namespace, doc.content, JSON.stringify((_b = doc.metadata) != null ? _b : {}), `[${doc.vector.join(",")}]`);
|
|
146
|
+
const docPlaceholders = [];
|
|
147
|
+
for (let j = 1; j <= 5 + csvHeaders.length; j++) {
|
|
148
|
+
if (j === 5 + csvHeaders.length) {
|
|
149
|
+
docPlaceholders.push(`$${offset + j}::vector`);
|
|
150
|
+
} else {
|
|
151
|
+
docPlaceholders.push(`$${offset + j}`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return `(${docPlaceholders.join(", ")})`;
|
|
155
|
+
}).join(", ");
|
|
156
|
+
const conflictUpdates = csvHeaders.map((h) => `"${h}" = EXCLUDED."${h}"`).join(",\n ");
|
|
157
|
+
const query = `
|
|
158
|
+
INSERT INTO "${tableName}" (${allColumns})
|
|
159
|
+
VALUES ${valuePlaceholders}
|
|
160
|
+
ON CONFLICT (id) DO UPDATE
|
|
161
|
+
SET ${conflictUpdates ? conflictUpdates + "," : ""}
|
|
162
|
+
namespace = EXCLUDED.namespace,
|
|
163
|
+
content = EXCLUDED.content,
|
|
164
|
+
metadata = EXCLUDED.metadata,
|
|
165
|
+
embedding = EXCLUDED.embedding
|
|
166
|
+
`;
|
|
167
|
+
await client.query(query, values);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
await client.query("COMMIT");
|
|
171
|
+
} catch (error) {
|
|
172
|
+
await client.query("ROLLBACK");
|
|
173
|
+
throw error;
|
|
174
|
+
} finally {
|
|
175
|
+
client.release();
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Query all configured tables and merge results, sorted by cosine similarity score.
|
|
180
|
+
*/
|
|
181
|
+
async query(vector, topK, _namespace, _filter) {
|
|
182
|
+
var _a;
|
|
183
|
+
if (!this.pool) {
|
|
184
|
+
throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
|
|
185
|
+
}
|
|
186
|
+
const vectorLiteral = `[${vector.join(",")}]`;
|
|
187
|
+
const allResults = [];
|
|
188
|
+
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
189
|
+
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.keywords) ? _filter.keywords.map((k) => k.trim().toLowerCase()).filter(Boolean) : [];
|
|
190
|
+
const getDynamicKeywordQuery = () => {
|
|
191
|
+
if (entityHints.length > 0) {
|
|
192
|
+
return entityHints.map((h) => h.replace(/\s+/g, " & ")).join(" & ");
|
|
193
|
+
}
|
|
194
|
+
if (queryText) {
|
|
195
|
+
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|organization[s]?|comp(?:any|anies)|of|category|find|list|browse|what|are|display|all|whose|that|which|have|has|having|greater|than|more|less|equal|equals|above|below|over|under|at|least|most|for|in|with|about|get|details|information|info)\b/g, "").replace(/\b\d+(?:\.\d+)?\b/g, "").trim().replace(/\s+/g, " & ");
|
|
196
|
+
}
|
|
197
|
+
return "";
|
|
198
|
+
};
|
|
199
|
+
const dynamicKeywordQuery = getDynamicKeywordQuery();
|
|
200
|
+
const tableLimit = Math.max(topK, 50);
|
|
201
|
+
const metadataFilters = _filter == null ? void 0 : _filter.metadata;
|
|
202
|
+
const queryPromises = this.tables.map(async (table) => {
|
|
203
|
+
try {
|
|
204
|
+
let sqlQuery = "";
|
|
205
|
+
let params = [];
|
|
206
|
+
let whereClause = "";
|
|
207
|
+
const filterParams = [];
|
|
208
|
+
if (metadataFilters && Object.keys(metadataFilters).length > 0) {
|
|
209
|
+
const conditions = Object.entries(metadataFilters).map(([key, val]) => {
|
|
210
|
+
var _a3;
|
|
211
|
+
filterParams.push(String(val));
|
|
212
|
+
const baseOffset = queryText && dynamicKeywordQuery ? 2 : 1;
|
|
213
|
+
const paramIdx = baseOffset + filterParams.length;
|
|
214
|
+
const synonyms = (_a3 = FIELD_SYNONYMS[key]) != null ? _a3 : [];
|
|
215
|
+
const keysToCheck = [key, ...synonyms].filter((k) => /^[a-zA-Z_][a-zA-Z0-9_]{0,62}$/.test(k));
|
|
216
|
+
const coalesceExprs = keysToCheck.flatMap((k) => [
|
|
217
|
+
`metadata->>'${k}'`,
|
|
218
|
+
`to_jsonb(t)->>'${k}'`
|
|
219
|
+
]);
|
|
220
|
+
return `COALESCE(${coalesceExprs.map((expr) => `LOWER(${expr})`).join(", ")}) = LOWER($${paramIdx})`;
|
|
221
|
+
});
|
|
222
|
+
whereClause = `WHERE ${conditions.join(" AND ")}`;
|
|
223
|
+
}
|
|
224
|
+
if (queryText && dynamicKeywordQuery) {
|
|
225
|
+
const hasEntityHints = entityHints.length > 0;
|
|
226
|
+
const exactNameScoreExpr = hasEntityHints ? `+ (
|
|
227
|
+
SELECT COALESCE(MAX(
|
|
228
|
+
CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(", ")})
|
|
229
|
+
THEN 1.0 ELSE 0.0 END
|
|
230
|
+
), 0)
|
|
231
|
+
FROM jsonb_each_text(COALESCE(v.metadata, k.metadata)) AS kv(key, val)
|
|
232
|
+
WHERE key IN (${this.searchFields.map((f) => `'${f.replace(/'/g, "''")}'`).join(", ")})
|
|
233
|
+
) * 3.0` : "";
|
|
234
|
+
sqlQuery = `
|
|
235
|
+
WITH vector_search AS (
|
|
236
|
+
SELECT *, (1 - (embedding <=> $1::vector)) AS vector_score
|
|
237
|
+
FROM "${table}" t
|
|
238
|
+
${whereClause}
|
|
239
|
+
ORDER BY embedding <=> $1::vector ASC
|
|
240
|
+
LIMIT ${tableLimit}
|
|
241
|
+
),
|
|
242
|
+
keyword_search AS (
|
|
243
|
+
SELECT *, COALESCE(ts_rank(to_tsvector('english', content), to_tsquery('english', $2)), 0) AS keyword_score
|
|
244
|
+
FROM "${table}" t
|
|
245
|
+
WHERE ${whereClause ? whereClause.replace("WHERE", "") : "TRUE"}
|
|
246
|
+
AND to_tsvector('english', content) @@ to_tsquery('english', $2)
|
|
247
|
+
ORDER BY keyword_score DESC
|
|
248
|
+
LIMIT ${tableLimit}
|
|
249
|
+
)
|
|
250
|
+
SELECT
|
|
251
|
+
COALESCE(v.id, k.id) AS id,
|
|
252
|
+
COALESCE(v.namespace, k.namespace) AS namespace,
|
|
253
|
+
COALESCE(v.content, k.content) AS content,
|
|
254
|
+
COALESCE(v.metadata, k.metadata) AS metadata,
|
|
255
|
+
COALESCE(v.vector_score, 0) AS vector_score,
|
|
256
|
+
COALESCE(k.keyword_score, 0) AS keyword_score,
|
|
257
|
+
(COALESCE(v.vector_score, 0) + COALESCE(k.keyword_score, 0) * 2.0 ${exactNameScoreExpr}) AS hybrid_score
|
|
258
|
+
FROM vector_search v
|
|
259
|
+
FULL OUTER JOIN keyword_search k ON v.id = k.id
|
|
260
|
+
ORDER BY hybrid_score DESC
|
|
261
|
+
LIMIT ${tableLimit}
|
|
262
|
+
`;
|
|
263
|
+
params = [vectorLiteral, dynamicKeywordQuery, ...filterParams];
|
|
264
|
+
} else {
|
|
265
|
+
sqlQuery = `
|
|
266
|
+
SELECT *,
|
|
267
|
+
(1 - (embedding <=> $1::vector)) AS hybrid_score
|
|
268
|
+
FROM "${table}" t
|
|
269
|
+
${whereClause}
|
|
270
|
+
ORDER BY embedding <=> $1::vector ASC
|
|
271
|
+
LIMIT ${tableLimit}
|
|
272
|
+
`;
|
|
273
|
+
params = [vectorLiteral, ...filterParams];
|
|
274
|
+
}
|
|
275
|
+
const result = await this.pool.query(sqlQuery, params);
|
|
276
|
+
if (result.rowCount && result.rowCount > 0) {
|
|
277
|
+
} else {
|
|
278
|
+
}
|
|
279
|
+
const tableResults = [];
|
|
280
|
+
for (const row of result.rows) {
|
|
281
|
+
const _a2 = row, { hybrid_score, id } = _a2, rest = __objRest(_a2, ["hybrid_score", "id"]);
|
|
282
|
+
delete rest.embedding;
|
|
283
|
+
delete rest.vector_score;
|
|
284
|
+
delete rest.keyword_score;
|
|
285
|
+
delete rest.exact_name_score;
|
|
286
|
+
const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
|
|
287
|
+
` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
|
|
288
|
+
tableResults.push({
|
|
289
|
+
id: `${table}-${id}`,
|
|
290
|
+
score: parseFloat(String(hybrid_score)),
|
|
291
|
+
content,
|
|
292
|
+
metadata: __spreadProps(__spreadValues({}, rest), { source_table: table })
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
return tableResults;
|
|
296
|
+
} catch (err) {
|
|
297
|
+
console.error(
|
|
298
|
+
`[MultiTablePostgresProvider] Error querying table "${table}":`,
|
|
299
|
+
err.message
|
|
300
|
+
);
|
|
301
|
+
return [];
|
|
302
|
+
}
|
|
303
|
+
});
|
|
304
|
+
const resultsArray = await Promise.all(queryPromises);
|
|
305
|
+
for (const tableResults of resultsArray) {
|
|
306
|
+
allResults.push(...tableResults);
|
|
307
|
+
}
|
|
308
|
+
allResults.sort((a, b) => b.score - a.score);
|
|
309
|
+
if (allResults.length === 0) return [];
|
|
310
|
+
const bestMatchTable = (_a = allResults[0].metadata) == null ? void 0 : _a.source_table;
|
|
311
|
+
const finalSorted = allResults.filter((res) => {
|
|
312
|
+
var _a2;
|
|
313
|
+
return ((_a2 = res.metadata) == null ? void 0 : _a2.source_table) === bestMatchTable;
|
|
314
|
+
});
|
|
315
|
+
return finalSorted.slice(0, Math.max(topK, 15));
|
|
316
|
+
}
|
|
317
|
+
async delete(_id, _namespace) {
|
|
318
|
+
console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
|
|
319
|
+
}
|
|
320
|
+
async deleteNamespace(_namespace) {
|
|
321
|
+
console.warn("[MultiTablePostgresProvider] deleteNamespace() is a no-op for multi-table mode.");
|
|
322
|
+
}
|
|
323
|
+
async ping() {
|
|
324
|
+
try {
|
|
325
|
+
await this.pool.query("SELECT 1");
|
|
326
|
+
return true;
|
|
327
|
+
} catch (e) {
|
|
328
|
+
return false;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
async disconnect() {
|
|
332
|
+
if (this.pool) {
|
|
333
|
+
await this.pool.end();
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
};
|
|
337
|
+
|
|
338
|
+
export {
|
|
339
|
+
MultiTablePostgresProvider
|
|
340
|
+
};
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BaseVectorProvider
|
|
3
|
+
} from "./chunk-IMP6FUCY.mjs";
|
|
4
|
+
import {
|
|
5
|
+
__spreadValues
|
|
6
|
+
} from "./chunk-UT6SZZEC.mjs";
|
|
7
|
+
|
|
8
|
+
// src/providers/vectordb/MilvusProvider.ts
|
|
9
|
+
import axios from "axios";
|
|
10
|
+
var MilvusProvider = class extends BaseVectorProvider {
|
|
11
|
+
constructor(config) {
|
|
12
|
+
super(config);
|
|
13
|
+
const opts = config.options;
|
|
14
|
+
const baseUrl = opts.baseUrl || opts.uri || (opts.host ? `http://${opts.host}:${opts.port || 19530}` : void 0);
|
|
15
|
+
if (!baseUrl) throw new Error("[MilvusProvider] options.baseUrl (or uri / host+port) is required");
|
|
16
|
+
this.http = axios.create({
|
|
17
|
+
baseURL: baseUrl,
|
|
18
|
+
headers: __spreadValues(__spreadValues({
|
|
19
|
+
"Content-Type": "application/json"
|
|
20
|
+
}, opts.headers || {}), opts.apiKey ? { Authorization: `Bearer ${opts.apiKey}` } : {})
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
async initialize() {
|
|
24
|
+
await this.ping();
|
|
25
|
+
}
|
|
26
|
+
async upsert(doc, namespace) {
|
|
27
|
+
const payload = {
|
|
28
|
+
collectionName: this.indexName,
|
|
29
|
+
data: [__spreadValues({
|
|
30
|
+
vector: doc.vector,
|
|
31
|
+
content: doc.content,
|
|
32
|
+
metadata: doc.metadata || {}
|
|
33
|
+
}, namespace ? { namespace } : {})]
|
|
34
|
+
};
|
|
35
|
+
await this.http.post("/v1/vector/upsert", payload);
|
|
36
|
+
}
|
|
37
|
+
async batchUpsert(docs, namespace) {
|
|
38
|
+
const batchSize = 100;
|
|
39
|
+
for (let i = 0; i < docs.length; i += batchSize) {
|
|
40
|
+
const batch = docs.slice(i, i + batchSize);
|
|
41
|
+
const payload = {
|
|
42
|
+
collectionName: this.indexName,
|
|
43
|
+
data: batch.map((doc) => __spreadValues({
|
|
44
|
+
vector: doc.vector,
|
|
45
|
+
content: doc.content,
|
|
46
|
+
metadata: doc.metadata || {}
|
|
47
|
+
}, namespace ? { namespace } : {}))
|
|
48
|
+
};
|
|
49
|
+
await this.http.post("/v1/vector/upsert", payload);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async query(vector, topK, namespace, _filter) {
|
|
53
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
54
|
+
const filterParts = [];
|
|
55
|
+
if (namespace) {
|
|
56
|
+
filterParts.push(`namespace == "${namespace}"`);
|
|
57
|
+
}
|
|
58
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
59
|
+
if (key === "queryText" || key === "keywords") continue;
|
|
60
|
+
if (typeof value === "string") {
|
|
61
|
+
const escaped = value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
|
62
|
+
filterParts.push(`metadata["${key}"] == "${escaped}"`);
|
|
63
|
+
} else if (typeof value === "number") {
|
|
64
|
+
filterParts.push(`metadata["${key}"] == ${value}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
const payload = {
|
|
68
|
+
collectionName: this.indexName,
|
|
69
|
+
vector,
|
|
70
|
+
limit: topK,
|
|
71
|
+
filter: filterParts.length > 0 ? filterParts.join(" && ") : void 0,
|
|
72
|
+
outputFields: ["content", "metadata"],
|
|
73
|
+
searchParams: {
|
|
74
|
+
nprobe: this.config.options.nprobe || 16,
|
|
75
|
+
ef: this.config.options.efSearch || Math.max(topK * 10, 64)
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
const { data } = await this.http.post("/v1/vector/search", payload);
|
|
79
|
+
return (data.data || []).map((res) => ({
|
|
80
|
+
id: String(res["id"]),
|
|
81
|
+
score: res["distance"],
|
|
82
|
+
content: res["content"],
|
|
83
|
+
metadata: res["metadata"] || {}
|
|
84
|
+
}));
|
|
85
|
+
}
|
|
86
|
+
async delete(id, namespace) {
|
|
87
|
+
await this.http.post("/v1/vector/delete", {
|
|
88
|
+
collectionName: this.indexName,
|
|
89
|
+
id
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
async deleteNamespace(namespace) {
|
|
93
|
+
await this.http.post("/v1/vector/delete", {
|
|
94
|
+
collectionName: this.indexName,
|
|
95
|
+
filter: `namespace == "${namespace}"`
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
async ping() {
|
|
99
|
+
try {
|
|
100
|
+
await this.http.get("/v1/health");
|
|
101
|
+
return true;
|
|
102
|
+
} catch (e) {
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async disconnect() {
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
export {
|
|
111
|
+
MilvusProvider
|
|
112
|
+
};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// src/providers/vectordb/BaseVectorProvider.ts
|
|
2
|
+
var BaseVectorProvider = class {
|
|
3
|
+
constructor(config) {
|
|
4
|
+
this.config = config;
|
|
5
|
+
this.indexName = config.indexName || "default";
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Remove internal keys (starting with __) and queryText from a filter object
|
|
9
|
+
* before passing it to an external provider that might not support them.
|
|
10
|
+
*/
|
|
11
|
+
sanitizeFilter(filter) {
|
|
12
|
+
if (!filter) return {};
|
|
13
|
+
const result = {};
|
|
14
|
+
for (const [key, value] of Object.entries(filter)) {
|
|
15
|
+
if (key.startsWith("__") || key === "queryText" || key === "keywords") {
|
|
16
|
+
continue;
|
|
17
|
+
}
|
|
18
|
+
if (key === "metadata" && typeof value === "object" && value !== null) {
|
|
19
|
+
Object.assign(result, value);
|
|
20
|
+
} else {
|
|
21
|
+
result[key] = value;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return result;
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export {
|
|
29
|
+
BaseVectorProvider
|
|
30
|
+
};
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BaseVectorProvider
|
|
3
|
+
} from "./chunk-IMP6FUCY.mjs";
|
|
4
|
+
import {
|
|
5
|
+
__spreadValues
|
|
6
|
+
} from "./chunk-UT6SZZEC.mjs";
|
|
7
|
+
|
|
8
|
+
// src/providers/vectordb/MongoDBProvider.ts
|
|
9
|
+
import { MongoClient } from "mongodb";
|
|
10
|
+
var MongoDBProvider = class extends BaseVectorProvider {
|
|
11
|
+
constructor(config) {
|
|
12
|
+
super(config);
|
|
13
|
+
this.initialized = false;
|
|
14
|
+
const opts = config.options;
|
|
15
|
+
if (!opts.uri || !opts.database || !opts.collection) {
|
|
16
|
+
throw new Error("[MongoDBProvider] options uri, database, and collection are required.");
|
|
17
|
+
}
|
|
18
|
+
this.client = new MongoClient(opts.uri, { serverSelectionTimeoutMS: 5e3 });
|
|
19
|
+
this.dbName = opts.database;
|
|
20
|
+
this.collectionName = opts.collection;
|
|
21
|
+
this.embeddingKey = opts.embeddingKey || "embedding";
|
|
22
|
+
this.contentKey = opts.contentKey || "content";
|
|
23
|
+
this.metadataKey = opts.metadataKey || "metadata";
|
|
24
|
+
}
|
|
25
|
+
static getValidator() {
|
|
26
|
+
return {
|
|
27
|
+
validate(config) {
|
|
28
|
+
const errors = [];
|
|
29
|
+
const opts = config.options || {};
|
|
30
|
+
if (!opts.uri) {
|
|
31
|
+
errors.push({
|
|
32
|
+
field: "vectorDb.options.uri",
|
|
33
|
+
message: "MongoDB connection URI is required",
|
|
34
|
+
suggestion: "Set MONGODB_URI environment variable",
|
|
35
|
+
severity: "error"
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
if (!opts.database) {
|
|
39
|
+
errors.push({
|
|
40
|
+
field: "vectorDb.options.database",
|
|
41
|
+
message: "MongoDB database name is required",
|
|
42
|
+
severity: "error"
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
if (!opts.collection) {
|
|
46
|
+
errors.push({
|
|
47
|
+
field: "vectorDb.options.collection",
|
|
48
|
+
message: "MongoDB collection name is required",
|
|
49
|
+
severity: "error"
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
return errors;
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
static getHealthChecker() {
|
|
57
|
+
return {
|
|
58
|
+
async check(config) {
|
|
59
|
+
const opts = config.options || {};
|
|
60
|
+
const timestamp = Date.now();
|
|
61
|
+
try {
|
|
62
|
+
const { MongoClient: MongoClient2 } = await import("mongodb");
|
|
63
|
+
const client = new MongoClient2(opts.uri);
|
|
64
|
+
await client.connect();
|
|
65
|
+
const db = client.db(opts.database);
|
|
66
|
+
await db.command({ ping: 1 });
|
|
67
|
+
const collections = await db.listCollections({ name: opts.collection }).toArray();
|
|
68
|
+
await client.close();
|
|
69
|
+
return {
|
|
70
|
+
healthy: true,
|
|
71
|
+
provider: "mongodb",
|
|
72
|
+
capabilities: {
|
|
73
|
+
database: opts.database,
|
|
74
|
+
collection: opts.collection,
|
|
75
|
+
exists: collections.length > 0
|
|
76
|
+
},
|
|
77
|
+
timestamp
|
|
78
|
+
};
|
|
79
|
+
} catch (error) {
|
|
80
|
+
return {
|
|
81
|
+
healthy: false,
|
|
82
|
+
provider: "mongodb",
|
|
83
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
84
|
+
timestamp
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
async initialize() {
|
|
91
|
+
if (this.initialized) return;
|
|
92
|
+
await this.client.connect();
|
|
93
|
+
this.db = this.client.db(this.dbName);
|
|
94
|
+
this.collection = this.db.collection(this.collectionName);
|
|
95
|
+
this.initialized = true;
|
|
96
|
+
}
|
|
97
|
+
async upsert(doc, namespace) {
|
|
98
|
+
const document = __spreadValues({
|
|
99
|
+
_id: doc.id,
|
|
100
|
+
[this.embeddingKey]: doc.vector,
|
|
101
|
+
[this.contentKey]: doc.content,
|
|
102
|
+
[this.metadataKey]: doc.metadata || {}
|
|
103
|
+
}, namespace ? { namespace } : {});
|
|
104
|
+
await this.collection.updateOne({ _id: doc.id }, { $set: document }, { upsert: true });
|
|
105
|
+
}
|
|
106
|
+
async batchUpsert(docs, namespace) {
|
|
107
|
+
const operations = docs.map((doc) => ({
|
|
108
|
+
updateOne: {
|
|
109
|
+
filter: { _id: doc.id },
|
|
110
|
+
update: {
|
|
111
|
+
$set: __spreadValues({
|
|
112
|
+
[this.embeddingKey]: doc.vector,
|
|
113
|
+
[this.contentKey]: doc.content,
|
|
114
|
+
[this.metadataKey]: doc.metadata || {}
|
|
115
|
+
}, namespace ? { namespace } : {})
|
|
116
|
+
},
|
|
117
|
+
upsert: true
|
|
118
|
+
}
|
|
119
|
+
}));
|
|
120
|
+
await this.collection.bulkWrite(operations);
|
|
121
|
+
}
|
|
122
|
+
async query(vector, topK, namespace, filter) {
|
|
123
|
+
const publicFilter = this.sanitizeFilter(filter);
|
|
124
|
+
const vectorSearchFilter = {};
|
|
125
|
+
const matchFilter = {};
|
|
126
|
+
if (namespace) {
|
|
127
|
+
vectorSearchFilter.namespace = namespace;
|
|
128
|
+
}
|
|
129
|
+
for (const [key, value] of Object.entries(publicFilter)) {
|
|
130
|
+
if (key === "namespace") {
|
|
131
|
+
vectorSearchFilter.namespace = value;
|
|
132
|
+
} else {
|
|
133
|
+
if (typeof value === "string") {
|
|
134
|
+
matchFilter[key] = { $regex: new RegExp(`^${value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`, "i") };
|
|
135
|
+
} else {
|
|
136
|
+
matchFilter[key] = value;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
const pipeline = [
|
|
141
|
+
{
|
|
142
|
+
$vectorSearch: __spreadValues({
|
|
143
|
+
index: this.config.indexName || "vector_index",
|
|
144
|
+
path: this.embeddingKey,
|
|
145
|
+
queryVector: vector,
|
|
146
|
+
numCandidates: this.config.options.numCandidates || Math.max(topK * 20, 200),
|
|
147
|
+
limit: topK * 2
|
|
148
|
+
}, Object.keys(vectorSearchFilter).length > 0 ? { filter: vectorSearchFilter } : {})
|
|
149
|
+
}
|
|
150
|
+
];
|
|
151
|
+
if (Object.keys(matchFilter).length > 0) {
|
|
152
|
+
pipeline.push({ $match: matchFilter });
|
|
153
|
+
}
|
|
154
|
+
pipeline.push(
|
|
155
|
+
{ $limit: topK },
|
|
156
|
+
{
|
|
157
|
+
$project: {
|
|
158
|
+
_id: 1,
|
|
159
|
+
[this.contentKey]: 1,
|
|
160
|
+
[this.metadataKey]: 1,
|
|
161
|
+
namespace: 1,
|
|
162
|
+
score: { $meta: "vectorSearchScore" }
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
);
|
|
166
|
+
const results = await this.collection.aggregate(pipeline).toArray();
|
|
167
|
+
return results.map((res) => {
|
|
168
|
+
const normalizedScore = res.score * 2 - 1;
|
|
169
|
+
return {
|
|
170
|
+
id: res._id,
|
|
171
|
+
content: res[this.contentKey],
|
|
172
|
+
metadata: res[this.metadataKey] || {},
|
|
173
|
+
score: normalizedScore
|
|
174
|
+
};
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
async delete(id, namespace) {
|
|
178
|
+
await this.collection.deleteOne(__spreadValues({ _id: id }, namespace ? { namespace } : {}));
|
|
179
|
+
}
|
|
180
|
+
async deleteNamespace(namespace) {
|
|
181
|
+
await this.collection.deleteMany({ namespace });
|
|
182
|
+
}
|
|
183
|
+
sanitizeFilter(filter) {
|
|
184
|
+
const sanitized = super.sanitizeFilter(filter);
|
|
185
|
+
const mongoFilter = {};
|
|
186
|
+
for (const [key, value] of Object.entries(sanitized)) {
|
|
187
|
+
if (key === "namespace") {
|
|
188
|
+
mongoFilter[key] = value;
|
|
189
|
+
} else {
|
|
190
|
+
mongoFilter[`${this.metadataKey}.${key}`] = value;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
return mongoFilter;
|
|
194
|
+
}
|
|
195
|
+
async ping() {
|
|
196
|
+
try {
|
|
197
|
+
await this.db.command({ ping: 1 });
|
|
198
|
+
return true;
|
|
199
|
+
} catch (e) {
|
|
200
|
+
return false;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
async disconnect() {
|
|
204
|
+
await this.client.close();
|
|
205
|
+
this.initialized = false;
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
export {
|
|
210
|
+
MongoDBProvider
|
|
211
|
+
};
|