@retrivora-ai/rag-engine 2.5.6 → 2.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ChromaDBProvider-4RPJN7Y3.mjs +8 -0
- package/dist/MilvusProvider-WJLLZ6TW.mjs +8 -0
- package/dist/MongoDBProvider-QSERELZX.mjs +8 -0
- package/dist/MultiTablePostgresProvider-JF2HEUDZ.mjs +9 -0
- package/dist/PineconeProvider-HLZL7HKU.mjs +10 -0
- package/dist/PostgreSQLProvider-JQMCNHRO.mjs +8 -0
- package/dist/QdrantProvider-6MNFPEVC.mjs +8 -0
- package/dist/RedisProvider-ERQL6AEB.mjs +8 -0
- package/dist/SimpleGraphProvider-OP3HAJJM.mjs +70 -0
- package/dist/UniversalVectorProvider-INM3KH5F.mjs +9 -0
- package/dist/WeaviateProvider-VHWWICCC.mjs +8 -0
- package/dist/chunk-366EFBZC.mjs +165 -0
- package/dist/chunk-6MGVQTU4.mjs +8883 -0
- package/dist/chunk-B4AXIUM7.mjs +49 -0
- package/dist/chunk-BEPLYQBS.mjs +89 -0
- package/dist/chunk-BNOPQCZW.mjs +238 -0
- package/dist/chunk-FJ43J4FW.mjs +654 -0
- package/dist/chunk-GP2AJQRQ.mjs +107 -0
- package/dist/chunk-IMP6FUCY.mjs +30 -0
- package/dist/chunk-JRQWSJNF.mjs +202 -0
- package/dist/chunk-LYHCKUNF.mjs +112 -0
- package/dist/chunk-MFIWWUC7.mjs +152 -0
- package/dist/chunk-MWICNCB2.mjs +157 -0
- package/dist/chunk-N54YTFBN.mjs +76 -0
- package/dist/chunk-POOWU2HR.mjs +211 -0
- package/dist/chunk-QGDGPOZH.mjs +217 -0
- package/dist/chunk-QI3FTN6G.mjs +281 -0
- package/dist/chunk-QJIDGVG6.mjs +334 -0
- package/dist/chunk-W2DKVD3G.mjs +81 -0
- package/dist/chunk-XHSDHPKI.mjs +336 -0
- package/dist/handlers/index.js +1 -1
- package/dist/handlers/index.mjs +24 -12052
- package/dist/{index-Byv5MaYi.d.mts → index-9h_ENWQn.d.mts} +16 -0
- package/dist/{index-DP3_OpuB.d.ts → index-Xci6GfpI.d.ts} +16 -0
- package/dist/index.css +112 -0
- package/dist/index.d.mts +176 -18
- package/dist/index.d.ts +176 -18
- package/dist/index.js +602 -104
- package/dist/index.min.css +1 -1
- package/dist/index.mjs +648 -1534
- package/dist/server.d.mts +2 -2
- package/dist/server.d.ts +2 -2
- package/dist/server.js +1 -1
- package/dist/server.mjs +372 -12608
- package/package.json +2 -2
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BaseVectorProvider
|
|
3
|
+
} from "./chunk-IMP6FUCY.mjs";
|
|
4
|
+
import {
|
|
5
|
+
__spreadValues
|
|
6
|
+
} from "./chunk-N54YTFBN.mjs";
|
|
7
|
+
|
|
8
|
+
// src/providers/vectordb/ChromaDBProvider.ts
|
|
9
|
+
import axios from "axios";
|
|
10
|
+
var ChromaDBProvider = class extends BaseVectorProvider {
|
|
11
|
+
constructor(config) {
|
|
12
|
+
super(config);
|
|
13
|
+
this.collectionId = "";
|
|
14
|
+
const opts = config.options;
|
|
15
|
+
const baseUrl = opts.baseUrl || (opts.host ? `http://${opts.host}:${opts.port || 8e3}` : void 0);
|
|
16
|
+
if (!baseUrl) throw new Error("[ChromaDBProvider] options.baseUrl is required");
|
|
17
|
+
this.http = axios.create({
|
|
18
|
+
baseURL: baseUrl,
|
|
19
|
+
headers: { "Content-Type": "application/json" }
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Get or create the ChromaDB collection.
|
|
24
|
+
*/
|
|
25
|
+
async initialize() {
|
|
26
|
+
var _a;
|
|
27
|
+
try {
|
|
28
|
+
const { data } = await this.http.get(`/api/v1/collections/${this.indexName}`);
|
|
29
|
+
this.collectionId = data.id;
|
|
30
|
+
} catch (err) {
|
|
31
|
+
if (axios.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
|
|
32
|
+
const { data } = await this.http.post("/api/v1/collections", {
|
|
33
|
+
name: this.indexName
|
|
34
|
+
});
|
|
35
|
+
this.collectionId = data.id;
|
|
36
|
+
} else {
|
|
37
|
+
throw err;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
async upsert(doc, namespace) {
|
|
42
|
+
await this.batchUpsert([doc], namespace);
|
|
43
|
+
}
|
|
44
|
+
async batchUpsert(docs, namespace) {
|
|
45
|
+
const payload = {
|
|
46
|
+
ids: docs.map((d) => String(d.id)),
|
|
47
|
+
embeddings: docs.map((d) => d.vector),
|
|
48
|
+
documents: docs.map((d) => d.content),
|
|
49
|
+
metadatas: docs.map((d) => __spreadValues(__spreadValues({}, d.metadata || {}), namespace ? { namespace } : {}))
|
|
50
|
+
};
|
|
51
|
+
await this.http.post(`/api/v1/collections/${this.collectionId}/upsert`, payload);
|
|
52
|
+
}
|
|
53
|
+
async query(vector, topK, namespace, _filter) {
|
|
54
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
55
|
+
const whereClauses = [];
|
|
56
|
+
if (namespace) whereClauses.push({ namespace: { $eq: namespace } });
|
|
57
|
+
Object.entries(sanitizedFilter).forEach(([key, value]) => {
|
|
58
|
+
if (key === "namespace") return;
|
|
59
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
60
|
+
whereClauses.push({ [key]: { $eq: value } });
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
const payload = {
|
|
64
|
+
query_embeddings: [vector],
|
|
65
|
+
n_results: topK,
|
|
66
|
+
where: whereClauses.length > 1 ? { $and: whereClauses } : whereClauses[0]
|
|
67
|
+
};
|
|
68
|
+
const { data } = await this.http.post(`/api/v1/collections/${this.collectionId}/query`, payload);
|
|
69
|
+
const matches = [];
|
|
70
|
+
if (data.ids && data.ids[0]) {
|
|
71
|
+
for (let i = 0; i < data.ids[0].length; i++) {
|
|
72
|
+
matches.push({
|
|
73
|
+
id: data.ids[0][i],
|
|
74
|
+
score: data.distances ? 1 - data.distances[0][i] : 0,
|
|
75
|
+
content: data.documents ? data.documents[0][i] : "",
|
|
76
|
+
metadata: data.metadatas ? data.metadatas[0][i] : {}
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return matches;
|
|
81
|
+
}
|
|
82
|
+
async delete(id, namespace) {
|
|
83
|
+
await this.http.post(`/api/v1/collections/${this.collectionId}/delete`, {
|
|
84
|
+
ids: [id],
|
|
85
|
+
where: namespace ? { namespace: { $eq: namespace } } : void 0
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
async deleteNamespace(namespace) {
|
|
89
|
+
await this.http.post(`/api/v1/collections/${this.collectionId}/delete`, {
|
|
90
|
+
where: { namespace: { $eq: namespace } }
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
async ping() {
|
|
94
|
+
try {
|
|
95
|
+
await this.http.get("/api/v1/heartbeat");
|
|
96
|
+
return true;
|
|
97
|
+
} catch (e) {
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
async disconnect() {
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
export {
|
|
106
|
+
ChromaDBProvider
|
|
107
|
+
};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
// src/providers/vectordb/BaseVectorProvider.ts
|
|
2
|
+
var BaseVectorProvider = class {
|
|
3
|
+
constructor(config) {
|
|
4
|
+
this.config = config;
|
|
5
|
+
this.indexName = config.indexName || "default";
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Remove internal keys (starting with __) and queryText from a filter object
|
|
9
|
+
* before passing it to an external provider that might not support them.
|
|
10
|
+
*/
|
|
11
|
+
sanitizeFilter(filter) {
|
|
12
|
+
if (!filter) return {};
|
|
13
|
+
const result = {};
|
|
14
|
+
for (const [key, value] of Object.entries(filter)) {
|
|
15
|
+
if (key.startsWith("__") || key === "queryText" || key === "keywords") {
|
|
16
|
+
continue;
|
|
17
|
+
}
|
|
18
|
+
if (key === "metadata" && typeof value === "object" && value !== null) {
|
|
19
|
+
Object.assign(result, value);
|
|
20
|
+
} else {
|
|
21
|
+
result[key] = value;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return result;
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export {
|
|
29
|
+
BaseVectorProvider
|
|
30
|
+
};
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BaseVectorProvider
|
|
3
|
+
} from "./chunk-IMP6FUCY.mjs";
|
|
4
|
+
|
|
5
|
+
// src/providers/vectordb/PostgreSQLProvider.ts
|
|
6
|
+
import { Pool } from "pg";
|
|
7
|
+
var PostgreSQLProvider = class extends BaseVectorProvider {
|
|
8
|
+
constructor(config) {
|
|
9
|
+
var _a;
|
|
10
|
+
super(config);
|
|
11
|
+
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, "_");
|
|
12
|
+
const opts = config.options;
|
|
13
|
+
if (!opts.connectionString) throw new Error("[PostgreSQLProvider] options.connectionString is required");
|
|
14
|
+
this.connectionString = opts.connectionString;
|
|
15
|
+
this.dimensions = (_a = opts.dimensions) != null ? _a : 1536;
|
|
16
|
+
}
|
|
17
|
+
static getValidator() {
|
|
18
|
+
return {
|
|
19
|
+
validate(config) {
|
|
20
|
+
const errors = [];
|
|
21
|
+
const opts = config.options || {};
|
|
22
|
+
if (!opts.connectionString) {
|
|
23
|
+
errors.push({
|
|
24
|
+
field: "vectorDb.options.connectionString",
|
|
25
|
+
message: "PostgreSQL connection string is required",
|
|
26
|
+
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
27
|
+
severity: "error"
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
31
|
+
errors.push({
|
|
32
|
+
field: "vectorDb.options.tables",
|
|
33
|
+
message: "PostgreSQL tables must be a string or a string array",
|
|
34
|
+
severity: "error"
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
return errors;
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
static getHealthChecker() {
|
|
42
|
+
return {
|
|
43
|
+
async check(config) {
|
|
44
|
+
const opts = config.options || {};
|
|
45
|
+
const timestamp = Date.now();
|
|
46
|
+
try {
|
|
47
|
+
const { Client } = await import("pg");
|
|
48
|
+
const client = new Client({ connectionString: opts.connectionString });
|
|
49
|
+
await client.connect();
|
|
50
|
+
const result = await client.query(`
|
|
51
|
+
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
52
|
+
`);
|
|
53
|
+
const hasVector = result.rows[0].exists;
|
|
54
|
+
await client.end();
|
|
55
|
+
return {
|
|
56
|
+
healthy: true,
|
|
57
|
+
provider: "postgresql",
|
|
58
|
+
capabilities: { pgvectorInstalled: hasVector },
|
|
59
|
+
timestamp
|
|
60
|
+
};
|
|
61
|
+
} catch (error) {
|
|
62
|
+
return {
|
|
63
|
+
healthy: false,
|
|
64
|
+
provider: "postgresql",
|
|
65
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
66
|
+
timestamp
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
async initialize() {
|
|
73
|
+
this.pool = new Pool({ connectionString: this.connectionString });
|
|
74
|
+
const client = await this.pool.connect();
|
|
75
|
+
try {
|
|
76
|
+
await client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
77
|
+
await client.query(`
|
|
78
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
79
|
+
id TEXT PRIMARY KEY,
|
|
80
|
+
namespace TEXT NOT NULL DEFAULT '',
|
|
81
|
+
content TEXT NOT NULL,
|
|
82
|
+
metadata JSONB,
|
|
83
|
+
embedding VECTOR(${this.dimensions})
|
|
84
|
+
)
|
|
85
|
+
`);
|
|
86
|
+
await client.query(`
|
|
87
|
+
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
88
|
+
ON ${this.tableName}
|
|
89
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
90
|
+
`);
|
|
91
|
+
} finally {
|
|
92
|
+
client.release();
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
async upsert(doc, namespace = "") {
|
|
96
|
+
var _a;
|
|
97
|
+
const vectorLiteral = `[${doc.vector.join(",")}]`;
|
|
98
|
+
await this.pool.query(
|
|
99
|
+
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
100
|
+
VALUES ($1, $2, $3, $4, $5::vector)
|
|
101
|
+
ON CONFLICT (id) DO UPDATE
|
|
102
|
+
SET namespace = EXCLUDED.namespace,
|
|
103
|
+
content = EXCLUDED.content,
|
|
104
|
+
metadata = EXCLUDED.metadata,
|
|
105
|
+
embedding = EXCLUDED.embedding`,
|
|
106
|
+
[doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), vectorLiteral]
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
async batchUpsert(docs, namespace = "") {
|
|
110
|
+
if (docs.length === 0) return;
|
|
111
|
+
const client = await this.pool.connect();
|
|
112
|
+
try {
|
|
113
|
+
await client.query("BEGIN");
|
|
114
|
+
const BATCH_SIZE = 50;
|
|
115
|
+
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
116
|
+
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
117
|
+
const values = [];
|
|
118
|
+
const valuePlaceholders = batch.map((doc, idx) => {
|
|
119
|
+
var _a;
|
|
120
|
+
const offset = idx * 5;
|
|
121
|
+
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
122
|
+
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
123
|
+
}).join(", ");
|
|
124
|
+
const query = `
|
|
125
|
+
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
126
|
+
VALUES ${valuePlaceholders}
|
|
127
|
+
ON CONFLICT (id) DO UPDATE
|
|
128
|
+
SET namespace = EXCLUDED.namespace,
|
|
129
|
+
content = EXCLUDED.content,
|
|
130
|
+
metadata = EXCLUDED.metadata,
|
|
131
|
+
embedding = EXCLUDED.embedding
|
|
132
|
+
`;
|
|
133
|
+
await client.query(query, values);
|
|
134
|
+
}
|
|
135
|
+
await client.query("COMMIT");
|
|
136
|
+
} catch (error) {
|
|
137
|
+
await client.query("ROLLBACK");
|
|
138
|
+
throw error;
|
|
139
|
+
} finally {
|
|
140
|
+
client.release();
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
async query(vector, topK, namespace, filter) {
|
|
144
|
+
const vectorLiteral = `[${vector.join(",")}]`;
|
|
145
|
+
let whereClause = namespace ? `WHERE namespace = $3` : "";
|
|
146
|
+
const params = [vectorLiteral, topK];
|
|
147
|
+
if (namespace) params.push(namespace);
|
|
148
|
+
const publicFilter = this.sanitizeFilter(filter);
|
|
149
|
+
if (Object.keys(publicFilter).length > 0) {
|
|
150
|
+
const filterConditions = Object.entries(publicFilter).map(([key, val]) => {
|
|
151
|
+
const paramIdx = params.length + 1;
|
|
152
|
+
params.push(String(val));
|
|
153
|
+
return `metadata->>'${key}' = $${paramIdx}`;
|
|
154
|
+
}).join(" AND ");
|
|
155
|
+
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
156
|
+
}
|
|
157
|
+
const client = await this.pool.connect();
|
|
158
|
+
try {
|
|
159
|
+
const efSearch = this.config.options.efSearch || Math.max(topK * 10, 40);
|
|
160
|
+
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
161
|
+
const result = await client.query(
|
|
162
|
+
`SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
|
|
163
|
+
FROM ${this.tableName}
|
|
164
|
+
${whereClause}
|
|
165
|
+
ORDER BY embedding <=> $1::vector
|
|
166
|
+
LIMIT $2`,
|
|
167
|
+
params
|
|
168
|
+
);
|
|
169
|
+
return result.rows.map((row) => ({
|
|
170
|
+
id: String(row["id"]),
|
|
171
|
+
score: parseFloat(String(row["score"])),
|
|
172
|
+
content: String(row["content"]),
|
|
173
|
+
metadata: row["metadata"]
|
|
174
|
+
}));
|
|
175
|
+
} finally {
|
|
176
|
+
client.release();
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
async delete(id, namespace) {
|
|
180
|
+
const where = namespace ? "WHERE id = $1 AND namespace = $2" : "WHERE id = $1";
|
|
181
|
+
const params = namespace ? [id, namespace] : [id];
|
|
182
|
+
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
183
|
+
}
|
|
184
|
+
async deleteNamespace(namespace) {
|
|
185
|
+
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
186
|
+
}
|
|
187
|
+
async ping() {
|
|
188
|
+
try {
|
|
189
|
+
await this.pool.query("SELECT 1");
|
|
190
|
+
return true;
|
|
191
|
+
} catch (e) {
|
|
192
|
+
return false;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
async disconnect() {
|
|
196
|
+
await this.pool.end();
|
|
197
|
+
}
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
export {
|
|
201
|
+
PostgreSQLProvider
|
|
202
|
+
};
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BaseVectorProvider
|
|
3
|
+
} from "./chunk-IMP6FUCY.mjs";
|
|
4
|
+
import {
|
|
5
|
+
__spreadValues
|
|
6
|
+
} from "./chunk-N54YTFBN.mjs";
|
|
7
|
+
|
|
8
|
+
// src/providers/vectordb/MilvusProvider.ts
|
|
9
|
+
import axios from "axios";
|
|
10
|
+
var MilvusProvider = class extends BaseVectorProvider {
|
|
11
|
+
constructor(config) {
|
|
12
|
+
super(config);
|
|
13
|
+
const opts = config.options;
|
|
14
|
+
const baseUrl = opts.baseUrl || opts.uri || (opts.host ? `http://${opts.host}:${opts.port || 19530}` : void 0);
|
|
15
|
+
if (!baseUrl) throw new Error("[MilvusProvider] options.baseUrl (or uri / host+port) is required");
|
|
16
|
+
this.http = axios.create({
|
|
17
|
+
baseURL: baseUrl,
|
|
18
|
+
headers: __spreadValues(__spreadValues({
|
|
19
|
+
"Content-Type": "application/json"
|
|
20
|
+
}, opts.headers || {}), opts.apiKey ? { Authorization: `Bearer ${opts.apiKey}` } : {})
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
async initialize() {
|
|
24
|
+
await this.ping();
|
|
25
|
+
}
|
|
26
|
+
async upsert(doc, namespace) {
|
|
27
|
+
const payload = {
|
|
28
|
+
collectionName: this.indexName,
|
|
29
|
+
data: [__spreadValues({
|
|
30
|
+
vector: doc.vector,
|
|
31
|
+
content: doc.content,
|
|
32
|
+
metadata: doc.metadata || {}
|
|
33
|
+
}, namespace ? { namespace } : {})]
|
|
34
|
+
};
|
|
35
|
+
await this.http.post("/v1/vector/upsert", payload);
|
|
36
|
+
}
|
|
37
|
+
async batchUpsert(docs, namespace) {
|
|
38
|
+
const batchSize = 100;
|
|
39
|
+
for (let i = 0; i < docs.length; i += batchSize) {
|
|
40
|
+
const batch = docs.slice(i, i + batchSize);
|
|
41
|
+
const payload = {
|
|
42
|
+
collectionName: this.indexName,
|
|
43
|
+
data: batch.map((doc) => __spreadValues({
|
|
44
|
+
vector: doc.vector,
|
|
45
|
+
content: doc.content,
|
|
46
|
+
metadata: doc.metadata || {}
|
|
47
|
+
}, namespace ? { namespace } : {}))
|
|
48
|
+
};
|
|
49
|
+
await this.http.post("/v1/vector/upsert", payload);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async query(vector, topK, namespace, _filter) {
|
|
53
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
54
|
+
const filterParts = [];
|
|
55
|
+
if (namespace) {
|
|
56
|
+
filterParts.push(`namespace == "${namespace}"`);
|
|
57
|
+
}
|
|
58
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
59
|
+
if (key === "queryText" || key === "keywords") continue;
|
|
60
|
+
if (typeof value === "string") {
|
|
61
|
+
const escaped = value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
|
62
|
+
filterParts.push(`metadata["${key}"] == "${escaped}"`);
|
|
63
|
+
} else if (typeof value === "number") {
|
|
64
|
+
filterParts.push(`metadata["${key}"] == ${value}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
const payload = {
|
|
68
|
+
collectionName: this.indexName,
|
|
69
|
+
vector,
|
|
70
|
+
limit: topK,
|
|
71
|
+
filter: filterParts.length > 0 ? filterParts.join(" && ") : void 0,
|
|
72
|
+
outputFields: ["content", "metadata"],
|
|
73
|
+
searchParams: {
|
|
74
|
+
nprobe: this.config.options.nprobe || 16,
|
|
75
|
+
ef: this.config.options.efSearch || Math.max(topK * 10, 64)
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
const { data } = await this.http.post("/v1/vector/search", payload);
|
|
79
|
+
return (data.data || []).map((res) => ({
|
|
80
|
+
id: String(res["id"]),
|
|
81
|
+
score: res["distance"],
|
|
82
|
+
content: res["content"],
|
|
83
|
+
metadata: res["metadata"] || {}
|
|
84
|
+
}));
|
|
85
|
+
}
|
|
86
|
+
async delete(id, namespace) {
|
|
87
|
+
await this.http.post("/v1/vector/delete", {
|
|
88
|
+
collectionName: this.indexName,
|
|
89
|
+
id
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
async deleteNamespace(namespace) {
|
|
93
|
+
await this.http.post("/v1/vector/delete", {
|
|
94
|
+
collectionName: this.indexName,
|
|
95
|
+
filter: `namespace == "${namespace}"`
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
async ping() {
|
|
99
|
+
try {
|
|
100
|
+
await this.http.get("/v1/health");
|
|
101
|
+
return true;
|
|
102
|
+
} catch (e) {
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async disconnect() {
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
export {
|
|
111
|
+
MilvusProvider
|
|
112
|
+
};
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BaseVectorProvider
|
|
3
|
+
} from "./chunk-IMP6FUCY.mjs";
|
|
4
|
+
import {
|
|
5
|
+
__spreadProps,
|
|
6
|
+
__spreadValues
|
|
7
|
+
} from "./chunk-N54YTFBN.mjs";
|
|
8
|
+
|
|
9
|
+
// src/providers/vectordb/WeaviateProvider.ts
|
|
10
|
+
import axios from "axios";
|
|
11
|
+
var WeaviateProvider = class extends BaseVectorProvider {
|
|
12
|
+
constructor(config) {
|
|
13
|
+
super(config);
|
|
14
|
+
const opts = config.options;
|
|
15
|
+
const baseUrl = opts.baseUrl || opts.url;
|
|
16
|
+
if (!baseUrl) throw new Error("[WeaviateProvider] options.baseUrl is required");
|
|
17
|
+
this.http = axios.create({
|
|
18
|
+
baseURL: baseUrl,
|
|
19
|
+
headers: __spreadValues(__spreadValues({
|
|
20
|
+
"Content-Type": "application/json"
|
|
21
|
+
}, opts.apiKey ? {
|
|
22
|
+
"Authorization": `Bearer ${opts.apiKey}`
|
|
23
|
+
} : {}), opts.openaiApiKey ? {
|
|
24
|
+
"X-OpenAI-Api-Key": opts.openaiApiKey
|
|
25
|
+
} : {})
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
async initialize() {
|
|
29
|
+
await this.ping();
|
|
30
|
+
}
|
|
31
|
+
async upsert(doc, namespace) {
|
|
32
|
+
const primitiveMetadata = this.extractPrimitiveMetadata(doc.metadata);
|
|
33
|
+
const payload = {
|
|
34
|
+
class: this.indexName,
|
|
35
|
+
id: doc.id,
|
|
36
|
+
vector: doc.vector,
|
|
37
|
+
properties: __spreadProps(__spreadValues({
|
|
38
|
+
content: doc.content,
|
|
39
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
40
|
+
}, primitiveMetadata), {
|
|
41
|
+
namespace: namespace || ""
|
|
42
|
+
})
|
|
43
|
+
};
|
|
44
|
+
await this.http.post("/v1/objects", payload);
|
|
45
|
+
}
|
|
46
|
+
async batchUpsert(docs, namespace) {
|
|
47
|
+
const payload = {
|
|
48
|
+
objects: docs.map((doc) => ({
|
|
49
|
+
class: this.indexName,
|
|
50
|
+
id: doc.id,
|
|
51
|
+
vector: doc.vector,
|
|
52
|
+
properties: __spreadProps(__spreadValues({
|
|
53
|
+
content: doc.content,
|
|
54
|
+
metadata: JSON.stringify(doc.metadata || {})
|
|
55
|
+
}, this.extractPrimitiveMetadata(doc.metadata)), {
|
|
56
|
+
namespace: namespace || ""
|
|
57
|
+
})
|
|
58
|
+
}))
|
|
59
|
+
};
|
|
60
|
+
await this.http.post("/v1/batch/objects", payload);
|
|
61
|
+
}
|
|
62
|
+
async query(vector, topK, namespace, _filter) {
|
|
63
|
+
var _a, _b;
|
|
64
|
+
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
65
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
66
|
+
const searchParams = queryText ? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }` : `nearVector: { vector: ${JSON.stringify(vector)} }`;
|
|
67
|
+
const where = this.buildWhereFilter(namespace, sanitizedFilter);
|
|
68
|
+
const graphqlQuery = {
|
|
69
|
+
query: `
|
|
70
|
+
{
|
|
71
|
+
Get {
|
|
72
|
+
${this.indexName}(
|
|
73
|
+
${searchParams}
|
|
74
|
+
limit: ${topK}
|
|
75
|
+
${where ? `where: ${where}` : ""}
|
|
76
|
+
) {
|
|
77
|
+
content
|
|
78
|
+
metadata
|
|
79
|
+
_additional {
|
|
80
|
+
id
|
|
81
|
+
distance
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
`
|
|
87
|
+
};
|
|
88
|
+
const { data } = await this.http.post("/v1/graphql", graphqlQuery);
|
|
89
|
+
const results = ((_b = (_a = data.data) == null ? void 0 : _a.Get) == null ? void 0 : _b[this.indexName]) || [];
|
|
90
|
+
return results.map((res) => ({
|
|
91
|
+
id: res["_additional"].id,
|
|
92
|
+
score: 1 - res["_additional"].distance,
|
|
93
|
+
content: res["content"],
|
|
94
|
+
metadata: res["metadata"] ? JSON.parse(res["metadata"]) : {}
|
|
95
|
+
}));
|
|
96
|
+
}
|
|
97
|
+
async delete(id, _namespace) {
|
|
98
|
+
await this.http.delete(`/v1/objects/${this.indexName}/${id}`);
|
|
99
|
+
}
|
|
100
|
+
async deleteNamespace(namespace) {
|
|
101
|
+
await this.http.delete("/v1/batch/objects", {
|
|
102
|
+
data: {
|
|
103
|
+
match: {
|
|
104
|
+
class: this.indexName,
|
|
105
|
+
where: { path: ["namespace"], operator: "Equal", valueString: namespace }
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
async ping() {
|
|
111
|
+
try {
|
|
112
|
+
await this.http.get("/v1/.well-known/live");
|
|
113
|
+
return true;
|
|
114
|
+
} catch (e) {
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
async disconnect() {
|
|
119
|
+
}
|
|
120
|
+
extractPrimitiveMetadata(metadata) {
|
|
121
|
+
const result = {};
|
|
122
|
+
Object.entries(metadata || {}).forEach(([key, value]) => {
|
|
123
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
124
|
+
result[key] = value;
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
return result;
|
|
128
|
+
}
|
|
129
|
+
buildWhereFilter(namespace, filter) {
|
|
130
|
+
const operands = [];
|
|
131
|
+
if (namespace) operands.push(this.weaviateOperand("namespace", namespace));
|
|
132
|
+
Object.entries(filter || {}).forEach(([key, value]) => {
|
|
133
|
+
if (key === "namespace") return;
|
|
134
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
135
|
+
operands.push(this.weaviateOperand(key, value));
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
if (operands.length === 0) return void 0;
|
|
139
|
+
if (operands.length === 1) return operands[0];
|
|
140
|
+
return `{ operator: And, operands: [${operands.join(", ")}] }`;
|
|
141
|
+
}
|
|
142
|
+
weaviateOperand(key, value) {
|
|
143
|
+
const path = `path: [${JSON.stringify(key)}], operator: Equal`;
|
|
144
|
+
if (typeof value === "number") return `{ ${path}, valueNumber: ${value} }`;
|
|
145
|
+
if (typeof value === "boolean") return `{ ${path}, valueBoolean: ${value} }`;
|
|
146
|
+
return `{ ${path}, valueString: ${JSON.stringify(value)} }`;
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
export {
|
|
151
|
+
WeaviateProvider
|
|
152
|
+
};
|