@retrivora-ai/rag-engine 1.0.8 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{QdrantProvider-YNUNEOZH.mjs → QdrantProvider-WWXFX2XF.mjs} +1 -1
- package/dist/chunk-65S4BQL2.mjs +238 -0
- package/dist/{chunk-3GKA3PJF.mjs → chunk-G2LVK46T.mjs} +2 -1
- package/dist/handlers/index.js +121 -41
- package/dist/handlers/index.mjs +1 -1
- package/dist/server.d.mts +9 -6
- package/dist/server.d.ts +9 -6
- package/dist/server.js +121 -41
- package/dist/server.mjs +2 -2
- package/package.json +1 -1
- package/src/core/Pipeline.ts +2 -0
- package/src/providers/vectordb/QdrantProvider.ts +160 -48
- package/dist/chunk-3DSHW676.mjs +0 -159
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BaseVectorProvider
|
|
3
|
+
} from "./chunk-IMP6FUCY.mjs";
|
|
4
|
+
import {
|
|
5
|
+
__spreadValues
|
|
6
|
+
} from "./chunk-X4TOT24V.mjs";
|
|
7
|
+
|
|
8
|
+
// src/providers/vectordb/QdrantProvider.ts
|
|
9
|
+
import axios from "axios";
|
|
10
|
+
import crypto from "crypto";
|
|
11
|
+
var QdrantProvider = class extends BaseVectorProvider {
|
|
12
|
+
constructor(config) {
|
|
13
|
+
super(config);
|
|
14
|
+
const opts = config.options;
|
|
15
|
+
const baseUrl = opts.baseUrl;
|
|
16
|
+
if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
|
|
17
|
+
this.contentField = opts.contentField || "content";
|
|
18
|
+
this.metadataField = opts.metadataField || "metadata";
|
|
19
|
+
this.isFlatPayload = !!opts.flatPayload;
|
|
20
|
+
this.http = axios.create({
|
|
21
|
+
baseURL: baseUrl,
|
|
22
|
+
headers: __spreadValues({
|
|
23
|
+
"Content-Type": "application/json"
|
|
24
|
+
}, opts.apiKey ? { "api-key": opts.apiKey } : {})
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
async initialize() {
|
|
28
|
+
await this.ping();
|
|
29
|
+
await this.ensureCollection();
|
|
30
|
+
console.log(`[QdrantProvider] \u{1F50D} Discovering schema for collection "${this.indexName}"...`);
|
|
31
|
+
const discoveredFields = await this.discoverSchema();
|
|
32
|
+
const opts = this.config.options;
|
|
33
|
+
const configFields = opts.filterableFields || [];
|
|
34
|
+
const allFields = [.../* @__PURE__ */ new Set([...discoveredFields, ...configFields])];
|
|
35
|
+
if (allFields.length > 0) {
|
|
36
|
+
console.log(`[QdrantProvider] \u{1F6E0} Ensuring indexes for ${allFields.length} discovered fields...`);
|
|
37
|
+
for (const fieldInfo of allFields) {
|
|
38
|
+
const [fieldName, schemaType] = fieldInfo.split(":");
|
|
39
|
+
await this.ensureIndex(fieldName, schemaType || "keyword");
|
|
40
|
+
}
|
|
41
|
+
} else {
|
|
42
|
+
console.log(`[QdrantProvider] \u2139\uFE0F No fields discovered for indexing.`);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Samples points from the collection to discover available payload fields.
|
|
47
|
+
*/
|
|
48
|
+
async discoverSchema() {
|
|
49
|
+
var _a;
|
|
50
|
+
try {
|
|
51
|
+
const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
|
|
52
|
+
limit: 20,
|
|
53
|
+
with_payload: true,
|
|
54
|
+
with_vector: false
|
|
55
|
+
});
|
|
56
|
+
const points = ((_a = data.result) == null ? void 0 : _a.points) || [];
|
|
57
|
+
const keys = /* @__PURE__ */ new Set();
|
|
58
|
+
for (const point of points) {
|
|
59
|
+
const payload = point.payload || {};
|
|
60
|
+
Object.keys(payload).forEach((k) => {
|
|
61
|
+
if (k !== "namespace" && k !== this.contentField && k !== this.metadataField) {
|
|
62
|
+
keys.add(k);
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
if (!this.isFlatPayload && payload[this.metadataField]) {
|
|
66
|
+
const meta = payload[this.metadataField];
|
|
67
|
+
if (typeof meta === "object" && meta !== null) {
|
|
68
|
+
Object.keys(meta).forEach((k) => keys.add(k));
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return Array.from(keys);
|
|
73
|
+
} catch (err) {
|
|
74
|
+
console.warn(`[QdrantProvider] \u26A0\uFE0F Schema discovery failed:`, err instanceof Error ? err.message : String(err));
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Ensures the collection exists. Creates it if missing.
|
|
80
|
+
*/
|
|
81
|
+
async ensureCollection() {
|
|
82
|
+
var _a;
|
|
83
|
+
try {
|
|
84
|
+
await this.http.get(`/collections/${this.indexName}`);
|
|
85
|
+
console.log(`[QdrantProvider] \u2705 Collection "${this.indexName}" already exists.`);
|
|
86
|
+
} catch (err) {
|
|
87
|
+
if (axios.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
|
|
88
|
+
const opts = this.config.options;
|
|
89
|
+
const dimensionsForCreate = opts.dimensions || 1536;
|
|
90
|
+
console.log(`[QdrantProvider] \u23F3 Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
|
|
91
|
+
await this.http.put(`/collections/${this.indexName}`, {
|
|
92
|
+
vectors: {
|
|
93
|
+
size: dimensionsForCreate,
|
|
94
|
+
distance: "Cosine"
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
console.log(`[QdrantProvider] \u2705 Created collection "${this.indexName}"`);
|
|
98
|
+
} else {
|
|
99
|
+
throw err;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Ensures that a payload field has an index.
|
|
105
|
+
*/
|
|
106
|
+
async ensureIndex(fieldName, schema = "keyword") {
|
|
107
|
+
var _a;
|
|
108
|
+
let fullPath = fieldName;
|
|
109
|
+
if (!this.isFlatPayload && !fieldName.includes(".") && fieldName !== "namespace" && fieldName !== this.contentField) {
|
|
110
|
+
fullPath = `${this.metadataField}.${fieldName}`;
|
|
111
|
+
}
|
|
112
|
+
try {
|
|
113
|
+
await this.http.put(`/collections/${this.indexName}/index`, {
|
|
114
|
+
field_name: fullPath,
|
|
115
|
+
field_schema: schema
|
|
116
|
+
});
|
|
117
|
+
console.log(`[QdrantProvider] \u2705 Ensured ${schema} index for "${fullPath}"`);
|
|
118
|
+
} catch (err) {
|
|
119
|
+
let status;
|
|
120
|
+
if (axios.isAxiosError(err)) status = (_a = err.response) == null ? void 0 : _a.status;
|
|
121
|
+
if (status === 409) return;
|
|
122
|
+
console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
async upsert(doc, namespace) {
|
|
126
|
+
await this.batchUpsert([doc], namespace);
|
|
127
|
+
}
|
|
128
|
+
async batchUpsert(docs, namespace) {
|
|
129
|
+
const payload = {
|
|
130
|
+
points: docs.map((doc) => {
|
|
131
|
+
const pointPayload = __spreadValues({
|
|
132
|
+
[this.contentField]: doc.content
|
|
133
|
+
}, namespace ? { namespace } : {});
|
|
134
|
+
if (this.isFlatPayload) {
|
|
135
|
+
Object.assign(pointPayload, doc.metadata || {});
|
|
136
|
+
} else {
|
|
137
|
+
pointPayload[this.metadataField] = doc.metadata || {};
|
|
138
|
+
}
|
|
139
|
+
return {
|
|
140
|
+
id: this.normalizeId(doc.id),
|
|
141
|
+
vector: doc.vector,
|
|
142
|
+
payload: pointPayload
|
|
143
|
+
};
|
|
144
|
+
})
|
|
145
|
+
};
|
|
146
|
+
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
147
|
+
}
|
|
148
|
+
async query(vector, topK, namespace, _filter) {
|
|
149
|
+
const must = [];
|
|
150
|
+
if (namespace) {
|
|
151
|
+
must.push({ key: "namespace", match: { value: namespace } });
|
|
152
|
+
}
|
|
153
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
154
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
155
|
+
let filterKey = key;
|
|
156
|
+
if (!this.isFlatPayload && !key.includes(".") && key !== "namespace" && key !== this.contentField) {
|
|
157
|
+
filterKey = `${this.metadataField}.${key}`;
|
|
158
|
+
}
|
|
159
|
+
if (Array.isArray(value)) {
|
|
160
|
+
must.push({ key: filterKey, match: { any: value } });
|
|
161
|
+
} else {
|
|
162
|
+
must.push({ key: filterKey, match: { value } });
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
const payload = {
|
|
166
|
+
vector,
|
|
167
|
+
limit: topK,
|
|
168
|
+
with_payload: true,
|
|
169
|
+
params: {
|
|
170
|
+
hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
|
|
171
|
+
exact: false
|
|
172
|
+
},
|
|
173
|
+
filter: must.length > 0 ? { must } : void 0
|
|
174
|
+
};
|
|
175
|
+
const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
|
|
176
|
+
const results = (data.result || []).map((res) => {
|
|
177
|
+
const p = res.payload || {};
|
|
178
|
+
let content = p[this.contentField] || "";
|
|
179
|
+
if (!content) {
|
|
180
|
+
const stringFields = Object.entries(p).filter(([_, v]) => typeof v === "string").map(([k, v]) => ({ key: k, val: v }));
|
|
181
|
+
if (stringFields.length > 0) {
|
|
182
|
+
const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
|
|
183
|
+
content = bestMatch.val;
|
|
184
|
+
} else {
|
|
185
|
+
content = JSON.stringify(p);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
let metadata = {};
|
|
189
|
+
if (this.isFlatPayload) {
|
|
190
|
+
metadata = __spreadValues({}, p);
|
|
191
|
+
delete metadata[this.contentField];
|
|
192
|
+
delete metadata["namespace"];
|
|
193
|
+
} else {
|
|
194
|
+
metadata = p[this.metadataField] || p;
|
|
195
|
+
}
|
|
196
|
+
return {
|
|
197
|
+
id: res.id,
|
|
198
|
+
score: res.score,
|
|
199
|
+
content,
|
|
200
|
+
metadata
|
|
201
|
+
};
|
|
202
|
+
});
|
|
203
|
+
return results;
|
|
204
|
+
}
|
|
205
|
+
async delete(id, _namespace) {
|
|
206
|
+
await this.http.post(`/collections/${this.indexName}/points/delete`, {
|
|
207
|
+
points: [this.normalizeId(id)]
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
async deleteNamespace(_namespace) {
|
|
211
|
+
await this.http.post(`/collections/${this.indexName}/points/delete`, {
|
|
212
|
+
filter: {
|
|
213
|
+
must: [{ key: "namespace", match: { value: _namespace } }]
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
async ping() {
|
|
218
|
+
try {
|
|
219
|
+
await this.http.get("/healthz");
|
|
220
|
+
return true;
|
|
221
|
+
} catch (e) {
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
normalizeId(id) {
|
|
226
|
+
if (typeof id === "number") return id;
|
|
227
|
+
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
228
|
+
if (uuidRegex.test(id)) return id.toLowerCase();
|
|
229
|
+
const hash = crypto.createHash("md5").update(id).digest("hex");
|
|
230
|
+
return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
|
|
231
|
+
}
|
|
232
|
+
async disconnect() {
|
|
233
|
+
}
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
export {
|
|
237
|
+
QdrantProvider
|
|
238
|
+
};
|
|
@@ -1106,7 +1106,7 @@ var ProviderRegistry = class {
|
|
|
1106
1106
|
return MilvusProvider;
|
|
1107
1107
|
}
|
|
1108
1108
|
case "qdrant": {
|
|
1109
|
-
const { QdrantProvider } = await import("./QdrantProvider-
|
|
1109
|
+
const { QdrantProvider } = await import("./QdrantProvider-WWXFX2XF.mjs");
|
|
1110
1110
|
return QdrantProvider;
|
|
1111
1111
|
}
|
|
1112
1112
|
case "chromadb": {
|
|
@@ -2273,6 +2273,7 @@ var Pipeline = class {
|
|
|
2273
2273
|
}
|
|
2274
2274
|
const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
|
|
2275
2275
|
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
2276
|
+
console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
|
|
2276
2277
|
const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
|
|
2277
2278
|
namespace: ns,
|
|
2278
2279
|
topK: topK * 2,
|
package/dist/handlers/index.js
CHANGED
|
@@ -845,6 +845,9 @@ var init_QdrantProvider = __esm({
|
|
|
845
845
|
const opts = config.options;
|
|
846
846
|
const baseUrl = opts.baseUrl;
|
|
847
847
|
if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
|
|
848
|
+
this.contentField = opts.contentField || "content";
|
|
849
|
+
this.metadataField = opts.metadataField || "metadata";
|
|
850
|
+
this.isFlatPayload = !!opts.flatPayload;
|
|
848
851
|
this.http = import_axios4.default.create({
|
|
849
852
|
baseURL: baseUrl,
|
|
850
853
|
headers: __spreadValues({
|
|
@@ -855,7 +858,53 @@ var init_QdrantProvider = __esm({
|
|
|
855
858
|
async initialize() {
|
|
856
859
|
await this.ping();
|
|
857
860
|
await this.ensureCollection();
|
|
858
|
-
|
|
861
|
+
console.log(`[QdrantProvider] \u{1F50D} Discovering schema for collection "${this.indexName}"...`);
|
|
862
|
+
const discoveredFields = await this.discoverSchema();
|
|
863
|
+
const opts = this.config.options;
|
|
864
|
+
const configFields = opts.filterableFields || [];
|
|
865
|
+
const allFields = [.../* @__PURE__ */ new Set([...discoveredFields, ...configFields])];
|
|
866
|
+
if (allFields.length > 0) {
|
|
867
|
+
console.log(`[QdrantProvider] \u{1F6E0} Ensuring indexes for ${allFields.length} discovered fields...`);
|
|
868
|
+
for (const fieldInfo of allFields) {
|
|
869
|
+
const [fieldName, schemaType] = fieldInfo.split(":");
|
|
870
|
+
await this.ensureIndex(fieldName, schemaType || "keyword");
|
|
871
|
+
}
|
|
872
|
+
} else {
|
|
873
|
+
console.log(`[QdrantProvider] \u2139\uFE0F No fields discovered for indexing.`);
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
/**
|
|
877
|
+
* Samples points from the collection to discover available payload fields.
|
|
878
|
+
*/
|
|
879
|
+
async discoverSchema() {
|
|
880
|
+
var _a;
|
|
881
|
+
try {
|
|
882
|
+
const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
|
|
883
|
+
limit: 20,
|
|
884
|
+
with_payload: true,
|
|
885
|
+
with_vector: false
|
|
886
|
+
});
|
|
887
|
+
const points = ((_a = data.result) == null ? void 0 : _a.points) || [];
|
|
888
|
+
const keys = /* @__PURE__ */ new Set();
|
|
889
|
+
for (const point of points) {
|
|
890
|
+
const payload = point.payload || {};
|
|
891
|
+
Object.keys(payload).forEach((k) => {
|
|
892
|
+
if (k !== "namespace" && k !== this.contentField && k !== this.metadataField) {
|
|
893
|
+
keys.add(k);
|
|
894
|
+
}
|
|
895
|
+
});
|
|
896
|
+
if (!this.isFlatPayload && payload[this.metadataField]) {
|
|
897
|
+
const meta = payload[this.metadataField];
|
|
898
|
+
if (typeof meta === "object" && meta !== null) {
|
|
899
|
+
Object.keys(meta).forEach((k) => keys.add(k));
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
return Array.from(keys);
|
|
904
|
+
} catch (err) {
|
|
905
|
+
console.warn(`[QdrantProvider] \u26A0\uFE0F Schema discovery failed:`, err instanceof Error ? err.message : String(err));
|
|
906
|
+
return [];
|
|
907
|
+
}
|
|
859
908
|
}
|
|
860
909
|
/**
|
|
861
910
|
* Ensures the collection exists. Creates it if missing.
|
|
@@ -883,29 +932,25 @@ var init_QdrantProvider = __esm({
|
|
|
883
932
|
}
|
|
884
933
|
}
|
|
885
934
|
/**
|
|
886
|
-
* Ensures that
|
|
887
|
-
* Qdrant requires this for search filters to work in many configurations.
|
|
935
|
+
* Ensures that a payload field has an index.
|
|
888
936
|
*/
|
|
889
|
-
async ensureIndex() {
|
|
890
|
-
var _a
|
|
937
|
+
async ensureIndex(fieldName, schema = "keyword") {
|
|
938
|
+
var _a;
|
|
939
|
+
let fullPath = fieldName;
|
|
940
|
+
if (!this.isFlatPayload && !fieldName.includes(".") && fieldName !== "namespace" && fieldName !== this.contentField) {
|
|
941
|
+
fullPath = `${this.metadataField}.${fieldName}`;
|
|
942
|
+
}
|
|
891
943
|
try {
|
|
892
944
|
await this.http.put(`/collections/${this.indexName}/index`, {
|
|
893
|
-
field_name:
|
|
894
|
-
field_schema:
|
|
945
|
+
field_name: fullPath,
|
|
946
|
+
field_schema: schema
|
|
895
947
|
});
|
|
896
|
-
console.log(`[QdrantProvider] \u2705 Ensured
|
|
948
|
+
console.log(`[QdrantProvider] \u2705 Ensured ${schema} index for "${fullPath}"`);
|
|
897
949
|
} catch (err) {
|
|
898
950
|
let status;
|
|
899
|
-
|
|
900
|
-
if (
|
|
901
|
-
|
|
902
|
-
data = (_b = err.response) == null ? void 0 : _b.data;
|
|
903
|
-
}
|
|
904
|
-
if (status === 409) {
|
|
905
|
-
return;
|
|
906
|
-
}
|
|
907
|
-
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
908
|
-
console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
|
|
951
|
+
if (import_axios4.default.isAxiosError(err)) status = (_a = err.response) == null ? void 0 : _a.status;
|
|
952
|
+
if (status === 409) return;
|
|
953
|
+
console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
|
|
909
954
|
}
|
|
910
955
|
}
|
|
911
956
|
async upsert(doc, namespace) {
|
|
@@ -913,19 +958,41 @@ var init_QdrantProvider = __esm({
|
|
|
913
958
|
}
|
|
914
959
|
async batchUpsert(docs, namespace) {
|
|
915
960
|
const payload = {
|
|
916
|
-
points: docs.map((doc) =>
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
}
|
|
923
|
-
|
|
961
|
+
points: docs.map((doc) => {
|
|
962
|
+
const pointPayload = __spreadValues({
|
|
963
|
+
[this.contentField]: doc.content
|
|
964
|
+
}, namespace ? { namespace } : {});
|
|
965
|
+
if (this.isFlatPayload) {
|
|
966
|
+
Object.assign(pointPayload, doc.metadata || {});
|
|
967
|
+
} else {
|
|
968
|
+
pointPayload[this.metadataField] = doc.metadata || {};
|
|
969
|
+
}
|
|
970
|
+
return {
|
|
971
|
+
id: this.normalizeId(doc.id),
|
|
972
|
+
vector: doc.vector,
|
|
973
|
+
payload: pointPayload
|
|
974
|
+
};
|
|
975
|
+
})
|
|
924
976
|
};
|
|
925
977
|
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
926
978
|
}
|
|
927
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
928
979
|
async query(vector, topK, namespace, _filter) {
|
|
980
|
+
const must = [];
|
|
981
|
+
if (namespace) {
|
|
982
|
+
must.push({ key: "namespace", match: { value: namespace } });
|
|
983
|
+
}
|
|
984
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
985
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
986
|
+
let filterKey = key;
|
|
987
|
+
if (!this.isFlatPayload && !key.includes(".") && key !== "namespace" && key !== this.contentField) {
|
|
988
|
+
filterKey = `${this.metadataField}.${key}`;
|
|
989
|
+
}
|
|
990
|
+
if (Array.isArray(value)) {
|
|
991
|
+
must.push({ key: filterKey, match: { any: value } });
|
|
992
|
+
} else {
|
|
993
|
+
must.push({ key: filterKey, match: { value } });
|
|
994
|
+
}
|
|
995
|
+
}
|
|
929
996
|
const payload = {
|
|
930
997
|
vector,
|
|
931
998
|
limit: topK,
|
|
@@ -934,22 +1001,38 @@ var init_QdrantProvider = __esm({
|
|
|
934
1001
|
hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
|
|
935
1002
|
exact: false
|
|
936
1003
|
},
|
|
937
|
-
filter:
|
|
938
|
-
must: [{ key: "namespace", match: { value: namespace } }]
|
|
939
|
-
} : void 0
|
|
1004
|
+
filter: must.length > 0 ? { must } : void 0
|
|
940
1005
|
};
|
|
941
1006
|
const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
|
|
942
|
-
|
|
943
|
-
|
|
1007
|
+
const results = (data.result || []).map((res) => {
|
|
1008
|
+
const p = res.payload || {};
|
|
1009
|
+
let content = p[this.contentField] || "";
|
|
1010
|
+
if (!content) {
|
|
1011
|
+
const stringFields = Object.entries(p).filter(([_, v]) => typeof v === "string").map(([k, v]) => ({ key: k, val: v }));
|
|
1012
|
+
if (stringFields.length > 0) {
|
|
1013
|
+
const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
|
|
1014
|
+
content = bestMatch.val;
|
|
1015
|
+
} else {
|
|
1016
|
+
content = JSON.stringify(p);
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
let metadata = {};
|
|
1020
|
+
if (this.isFlatPayload) {
|
|
1021
|
+
metadata = __spreadValues({}, p);
|
|
1022
|
+
delete metadata[this.contentField];
|
|
1023
|
+
delete metadata["namespace"];
|
|
1024
|
+
} else {
|
|
1025
|
+
metadata = p[this.metadataField] || p;
|
|
1026
|
+
}
|
|
944
1027
|
return {
|
|
945
|
-
id: res
|
|
946
|
-
score: res
|
|
947
|
-
content
|
|
948
|
-
metadata
|
|
1028
|
+
id: res.id,
|
|
1029
|
+
score: res.score,
|
|
1030
|
+
content,
|
|
1031
|
+
metadata
|
|
949
1032
|
};
|
|
950
1033
|
});
|
|
1034
|
+
return results;
|
|
951
1035
|
}
|
|
952
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
953
1036
|
async delete(id, _namespace) {
|
|
954
1037
|
await this.http.post(`/collections/${this.indexName}/points/delete`, {
|
|
955
1038
|
points: [this.normalizeId(id)]
|
|
@@ -970,10 +1053,6 @@ var init_QdrantProvider = __esm({
|
|
|
970
1053
|
return false;
|
|
971
1054
|
}
|
|
972
1055
|
}
|
|
973
|
-
/**
|
|
974
|
-
* Normalizes an ID into a format Qdrant accepts (UUID or integer).
|
|
975
|
-
* For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
|
|
976
|
-
*/
|
|
977
1056
|
normalizeId(id) {
|
|
978
1057
|
if (typeof id === "number") return id;
|
|
979
1058
|
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
@@ -3749,6 +3828,7 @@ var Pipeline = class {
|
|
|
3749
3828
|
}
|
|
3750
3829
|
const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
|
|
3751
3830
|
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
3831
|
+
console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
|
|
3752
3832
|
const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
|
|
3753
3833
|
namespace: ns,
|
|
3754
3834
|
topK: topK * 2,
|
package/dist/handlers/index.mjs
CHANGED
package/dist/server.d.mts
CHANGED
|
@@ -731,18 +731,25 @@ declare class MilvusProvider extends BaseVectorProvider {
|
|
|
731
731
|
|
|
732
732
|
/**
|
|
733
733
|
* QdrantProvider — implementation for Qdrant using its REST API.
|
|
734
|
+
* Fully dynamic: automatically discovers and indexes schema from available data.
|
|
734
735
|
*/
|
|
735
736
|
declare class QdrantProvider extends BaseVectorProvider {
|
|
736
737
|
private http;
|
|
738
|
+
private contentField;
|
|
739
|
+
private metadataField;
|
|
740
|
+
private isFlatPayload;
|
|
737
741
|
constructor(config: VectorDBConfig);
|
|
738
742
|
initialize(): Promise<void>;
|
|
743
|
+
/**
|
|
744
|
+
* Samples points from the collection to discover available payload fields.
|
|
745
|
+
*/
|
|
746
|
+
private discoverSchema;
|
|
739
747
|
/**
|
|
740
748
|
* Ensures the collection exists. Creates it if missing.
|
|
741
749
|
*/
|
|
742
750
|
private ensureCollection;
|
|
743
751
|
/**
|
|
744
|
-
* Ensures that
|
|
745
|
-
* Qdrant requires this for search filters to work in many configurations.
|
|
752
|
+
* Ensures that a payload field has an index.
|
|
746
753
|
*/
|
|
747
754
|
private ensureIndex;
|
|
748
755
|
upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
|
|
@@ -751,10 +758,6 @@ declare class QdrantProvider extends BaseVectorProvider {
|
|
|
751
758
|
delete(id: string | number, _namespace?: string): Promise<void>;
|
|
752
759
|
deleteNamespace(_namespace: string): Promise<void>;
|
|
753
760
|
ping(): Promise<boolean>;
|
|
754
|
-
/**
|
|
755
|
-
* Normalizes an ID into a format Qdrant accepts (UUID or integer).
|
|
756
|
-
* For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
|
|
757
|
-
*/
|
|
758
761
|
private normalizeId;
|
|
759
762
|
disconnect(): Promise<void>;
|
|
760
763
|
}
|
package/dist/server.d.ts
CHANGED
|
@@ -731,18 +731,25 @@ declare class MilvusProvider extends BaseVectorProvider {
|
|
|
731
731
|
|
|
732
732
|
/**
|
|
733
733
|
* QdrantProvider — implementation for Qdrant using its REST API.
|
|
734
|
+
* Fully dynamic: automatically discovers and indexes schema from available data.
|
|
734
735
|
*/
|
|
735
736
|
declare class QdrantProvider extends BaseVectorProvider {
|
|
736
737
|
private http;
|
|
738
|
+
private contentField;
|
|
739
|
+
private metadataField;
|
|
740
|
+
private isFlatPayload;
|
|
737
741
|
constructor(config: VectorDBConfig);
|
|
738
742
|
initialize(): Promise<void>;
|
|
743
|
+
/**
|
|
744
|
+
* Samples points from the collection to discover available payload fields.
|
|
745
|
+
*/
|
|
746
|
+
private discoverSchema;
|
|
739
747
|
/**
|
|
740
748
|
* Ensures the collection exists. Creates it if missing.
|
|
741
749
|
*/
|
|
742
750
|
private ensureCollection;
|
|
743
751
|
/**
|
|
744
|
-
* Ensures that
|
|
745
|
-
* Qdrant requires this for search filters to work in many configurations.
|
|
752
|
+
* Ensures that a payload field has an index.
|
|
746
753
|
*/
|
|
747
754
|
private ensureIndex;
|
|
748
755
|
upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
|
|
@@ -751,10 +758,6 @@ declare class QdrantProvider extends BaseVectorProvider {
|
|
|
751
758
|
delete(id: string | number, _namespace?: string): Promise<void>;
|
|
752
759
|
deleteNamespace(_namespace: string): Promise<void>;
|
|
753
760
|
ping(): Promise<boolean>;
|
|
754
|
-
/**
|
|
755
|
-
* Normalizes an ID into a format Qdrant accepts (UUID or integer).
|
|
756
|
-
* For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
|
|
757
|
-
*/
|
|
758
761
|
private normalizeId;
|
|
759
762
|
disconnect(): Promise<void>;
|
|
760
763
|
}
|
package/dist/server.js
CHANGED
|
@@ -857,6 +857,9 @@ var init_QdrantProvider = __esm({
|
|
|
857
857
|
const opts = config.options;
|
|
858
858
|
const baseUrl = opts.baseUrl;
|
|
859
859
|
if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
|
|
860
|
+
this.contentField = opts.contentField || "content";
|
|
861
|
+
this.metadataField = opts.metadataField || "metadata";
|
|
862
|
+
this.isFlatPayload = !!opts.flatPayload;
|
|
860
863
|
this.http = import_axios4.default.create({
|
|
861
864
|
baseURL: baseUrl,
|
|
862
865
|
headers: __spreadValues({
|
|
@@ -867,7 +870,53 @@ var init_QdrantProvider = __esm({
|
|
|
867
870
|
async initialize() {
|
|
868
871
|
await this.ping();
|
|
869
872
|
await this.ensureCollection();
|
|
870
|
-
|
|
873
|
+
console.log(`[QdrantProvider] \u{1F50D} Discovering schema for collection "${this.indexName}"...`);
|
|
874
|
+
const discoveredFields = await this.discoverSchema();
|
|
875
|
+
const opts = this.config.options;
|
|
876
|
+
const configFields = opts.filterableFields || [];
|
|
877
|
+
const allFields = [.../* @__PURE__ */ new Set([...discoveredFields, ...configFields])];
|
|
878
|
+
if (allFields.length > 0) {
|
|
879
|
+
console.log(`[QdrantProvider] \u{1F6E0} Ensuring indexes for ${allFields.length} discovered fields...`);
|
|
880
|
+
for (const fieldInfo of allFields) {
|
|
881
|
+
const [fieldName, schemaType] = fieldInfo.split(":");
|
|
882
|
+
await this.ensureIndex(fieldName, schemaType || "keyword");
|
|
883
|
+
}
|
|
884
|
+
} else {
|
|
885
|
+
console.log(`[QdrantProvider] \u2139\uFE0F No fields discovered for indexing.`);
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
/**
|
|
889
|
+
* Samples points from the collection to discover available payload fields.
|
|
890
|
+
*/
|
|
891
|
+
async discoverSchema() {
|
|
892
|
+
var _a;
|
|
893
|
+
try {
|
|
894
|
+
const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
|
|
895
|
+
limit: 20,
|
|
896
|
+
with_payload: true,
|
|
897
|
+
with_vector: false
|
|
898
|
+
});
|
|
899
|
+
const points = ((_a = data.result) == null ? void 0 : _a.points) || [];
|
|
900
|
+
const keys = /* @__PURE__ */ new Set();
|
|
901
|
+
for (const point of points) {
|
|
902
|
+
const payload = point.payload || {};
|
|
903
|
+
Object.keys(payload).forEach((k) => {
|
|
904
|
+
if (k !== "namespace" && k !== this.contentField && k !== this.metadataField) {
|
|
905
|
+
keys.add(k);
|
|
906
|
+
}
|
|
907
|
+
});
|
|
908
|
+
if (!this.isFlatPayload && payload[this.metadataField]) {
|
|
909
|
+
const meta = payload[this.metadataField];
|
|
910
|
+
if (typeof meta === "object" && meta !== null) {
|
|
911
|
+
Object.keys(meta).forEach((k) => keys.add(k));
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
return Array.from(keys);
|
|
916
|
+
} catch (err) {
|
|
917
|
+
console.warn(`[QdrantProvider] \u26A0\uFE0F Schema discovery failed:`, err instanceof Error ? err.message : String(err));
|
|
918
|
+
return [];
|
|
919
|
+
}
|
|
871
920
|
}
|
|
872
921
|
/**
|
|
873
922
|
* Ensures the collection exists. Creates it if missing.
|
|
@@ -895,29 +944,25 @@ var init_QdrantProvider = __esm({
|
|
|
895
944
|
}
|
|
896
945
|
}
|
|
897
946
|
/**
|
|
898
|
-
* Ensures that
|
|
899
|
-
* Qdrant requires this for search filters to work in many configurations.
|
|
947
|
+
* Ensures that a payload field has an index.
|
|
900
948
|
*/
|
|
901
|
-
async ensureIndex() {
|
|
902
|
-
var _a
|
|
949
|
+
async ensureIndex(fieldName, schema = "keyword") {
|
|
950
|
+
var _a;
|
|
951
|
+
let fullPath = fieldName;
|
|
952
|
+
if (!this.isFlatPayload && !fieldName.includes(".") && fieldName !== "namespace" && fieldName !== this.contentField) {
|
|
953
|
+
fullPath = `${this.metadataField}.${fieldName}`;
|
|
954
|
+
}
|
|
903
955
|
try {
|
|
904
956
|
await this.http.put(`/collections/${this.indexName}/index`, {
|
|
905
|
-
field_name:
|
|
906
|
-
field_schema:
|
|
957
|
+
field_name: fullPath,
|
|
958
|
+
field_schema: schema
|
|
907
959
|
});
|
|
908
|
-
console.log(`[QdrantProvider] \u2705 Ensured
|
|
960
|
+
console.log(`[QdrantProvider] \u2705 Ensured ${schema} index for "${fullPath}"`);
|
|
909
961
|
} catch (err) {
|
|
910
962
|
let status;
|
|
911
|
-
|
|
912
|
-
if (
|
|
913
|
-
|
|
914
|
-
data = (_b = err.response) == null ? void 0 : _b.data;
|
|
915
|
-
}
|
|
916
|
-
if (status === 409) {
|
|
917
|
-
return;
|
|
918
|
-
}
|
|
919
|
-
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
920
|
-
console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
|
|
963
|
+
if (import_axios4.default.isAxiosError(err)) status = (_a = err.response) == null ? void 0 : _a.status;
|
|
964
|
+
if (status === 409) return;
|
|
965
|
+
console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
|
|
921
966
|
}
|
|
922
967
|
}
|
|
923
968
|
async upsert(doc, namespace) {
|
|
@@ -925,19 +970,41 @@ var init_QdrantProvider = __esm({
|
|
|
925
970
|
}
|
|
926
971
|
async batchUpsert(docs, namespace) {
|
|
927
972
|
const payload = {
|
|
928
|
-
points: docs.map((doc) =>
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
}
|
|
935
|
-
|
|
973
|
+
points: docs.map((doc) => {
|
|
974
|
+
const pointPayload = __spreadValues({
|
|
975
|
+
[this.contentField]: doc.content
|
|
976
|
+
}, namespace ? { namespace } : {});
|
|
977
|
+
if (this.isFlatPayload) {
|
|
978
|
+
Object.assign(pointPayload, doc.metadata || {});
|
|
979
|
+
} else {
|
|
980
|
+
pointPayload[this.metadataField] = doc.metadata || {};
|
|
981
|
+
}
|
|
982
|
+
return {
|
|
983
|
+
id: this.normalizeId(doc.id),
|
|
984
|
+
vector: doc.vector,
|
|
985
|
+
payload: pointPayload
|
|
986
|
+
};
|
|
987
|
+
})
|
|
936
988
|
};
|
|
937
989
|
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
938
990
|
}
|
|
939
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
940
991
|
async query(vector, topK, namespace, _filter) {
|
|
992
|
+
const must = [];
|
|
993
|
+
if (namespace) {
|
|
994
|
+
must.push({ key: "namespace", match: { value: namespace } });
|
|
995
|
+
}
|
|
996
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
997
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
998
|
+
let filterKey = key;
|
|
999
|
+
if (!this.isFlatPayload && !key.includes(".") && key !== "namespace" && key !== this.contentField) {
|
|
1000
|
+
filterKey = `${this.metadataField}.${key}`;
|
|
1001
|
+
}
|
|
1002
|
+
if (Array.isArray(value)) {
|
|
1003
|
+
must.push({ key: filterKey, match: { any: value } });
|
|
1004
|
+
} else {
|
|
1005
|
+
must.push({ key: filterKey, match: { value } });
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
941
1008
|
const payload = {
|
|
942
1009
|
vector,
|
|
943
1010
|
limit: topK,
|
|
@@ -946,22 +1013,38 @@ var init_QdrantProvider = __esm({
|
|
|
946
1013
|
hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
|
|
947
1014
|
exact: false
|
|
948
1015
|
},
|
|
949
|
-
filter:
|
|
950
|
-
must: [{ key: "namespace", match: { value: namespace } }]
|
|
951
|
-
} : void 0
|
|
1016
|
+
filter: must.length > 0 ? { must } : void 0
|
|
952
1017
|
};
|
|
953
1018
|
const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
|
|
954
|
-
|
|
955
|
-
|
|
1019
|
+
const results = (data.result || []).map((res) => {
|
|
1020
|
+
const p = res.payload || {};
|
|
1021
|
+
let content = p[this.contentField] || "";
|
|
1022
|
+
if (!content) {
|
|
1023
|
+
const stringFields = Object.entries(p).filter(([_, v]) => typeof v === "string").map(([k, v]) => ({ key: k, val: v }));
|
|
1024
|
+
if (stringFields.length > 0) {
|
|
1025
|
+
const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
|
|
1026
|
+
content = bestMatch.val;
|
|
1027
|
+
} else {
|
|
1028
|
+
content = JSON.stringify(p);
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
let metadata = {};
|
|
1032
|
+
if (this.isFlatPayload) {
|
|
1033
|
+
metadata = __spreadValues({}, p);
|
|
1034
|
+
delete metadata[this.contentField];
|
|
1035
|
+
delete metadata["namespace"];
|
|
1036
|
+
} else {
|
|
1037
|
+
metadata = p[this.metadataField] || p;
|
|
1038
|
+
}
|
|
956
1039
|
return {
|
|
957
|
-
id: res
|
|
958
|
-
score: res
|
|
959
|
-
content
|
|
960
|
-
metadata
|
|
1040
|
+
id: res.id,
|
|
1041
|
+
score: res.score,
|
|
1042
|
+
content,
|
|
1043
|
+
metadata
|
|
961
1044
|
};
|
|
962
1045
|
});
|
|
1046
|
+
return results;
|
|
963
1047
|
}
|
|
964
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
965
1048
|
async delete(id, _namespace) {
|
|
966
1049
|
await this.http.post(`/collections/${this.indexName}/points/delete`, {
|
|
967
1050
|
points: [this.normalizeId(id)]
|
|
@@ -982,10 +1065,6 @@ var init_QdrantProvider = __esm({
|
|
|
982
1065
|
return false;
|
|
983
1066
|
}
|
|
984
1067
|
}
|
|
985
|
-
/**
|
|
986
|
-
* Normalizes an ID into a format Qdrant accepts (UUID or integer).
|
|
987
|
-
* For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
|
|
988
|
-
*/
|
|
989
1068
|
normalizeId(id) {
|
|
990
1069
|
if (typeof id === "number") return id;
|
|
991
1070
|
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
@@ -3842,6 +3921,7 @@ var Pipeline = class {
|
|
|
3842
3921
|
}
|
|
3843
3922
|
const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
|
|
3844
3923
|
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
3924
|
+
console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
|
|
3845
3925
|
const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
|
|
3846
3926
|
namespace: ns,
|
|
3847
3927
|
topK: topK * 2,
|
package/dist/server.mjs
CHANGED
|
@@ -40,7 +40,7 @@ import {
|
|
|
40
40
|
sseFrame,
|
|
41
41
|
sseMetaFrame,
|
|
42
42
|
sseTextFrame
|
|
43
|
-
} from "./chunk-
|
|
43
|
+
} from "./chunk-G2LVK46T.mjs";
|
|
44
44
|
import "./chunk-YLTMFW4M.mjs";
|
|
45
45
|
import {
|
|
46
46
|
PineconeProvider
|
|
@@ -56,7 +56,7 @@ import {
|
|
|
56
56
|
} from "./chunk-U55XRW3U.mjs";
|
|
57
57
|
import {
|
|
58
58
|
QdrantProvider
|
|
59
|
-
} from "./chunk-
|
|
59
|
+
} from "./chunk-65S4BQL2.mjs";
|
|
60
60
|
import {
|
|
61
61
|
BaseVectorProvider
|
|
62
62
|
} from "./chunk-IMP6FUCY.mjs";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@retrivora-ai/rag-engine",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
|
|
5
5
|
"author": "Abhinav Alkuchi",
|
|
6
6
|
"license": "MIT",
|
package/src/core/Pipeline.ts
CHANGED
|
@@ -317,6 +317,8 @@ export class Pipeline {
|
|
|
317
317
|
// 2. Parallel Retrieval
|
|
318
318
|
const hints = QueryProcessor.extractQueryFieldHints(question, this.config.rag?.filterableFields);
|
|
319
319
|
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
320
|
+
|
|
321
|
+
console.log(`[Pipeline] 🧩 Extracted filters:`, JSON.stringify(filter));
|
|
320
322
|
|
|
321
323
|
const { sources: rawSources, graphData } = await this.retrieve(searchQuery, {
|
|
322
324
|
namespace: ns,
|
|
@@ -6,9 +6,13 @@ import { VectorMatch, UpsertDocument } from '../../types';
|
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* QdrantProvider — implementation for Qdrant using its REST API.
|
|
9
|
+
* Fully dynamic: automatically discovers and indexes schema from available data.
|
|
9
10
|
*/
|
|
10
11
|
export class QdrantProvider extends BaseVectorProvider {
|
|
11
12
|
private http: AxiosInstance;
|
|
13
|
+
private contentField: string;
|
|
14
|
+
private metadataField: string;
|
|
15
|
+
private isFlatPayload: boolean;
|
|
12
16
|
|
|
13
17
|
constructor(config: VectorDBConfig) {
|
|
14
18
|
super(config);
|
|
@@ -16,6 +20,10 @@ export class QdrantProvider extends BaseVectorProvider {
|
|
|
16
20
|
const baseUrl = opts.baseUrl as string;
|
|
17
21
|
if (!baseUrl) throw new Error('[QdrantProvider] baseUrl is required');
|
|
18
22
|
|
|
23
|
+
this.contentField = (opts.contentField as string) || 'content';
|
|
24
|
+
this.metadataField = (opts.metadataField as string) || 'metadata';
|
|
25
|
+
this.isFlatPayload = !!opts.flatPayload;
|
|
26
|
+
|
|
19
27
|
this.http = axios.create({
|
|
20
28
|
baseURL: baseUrl,
|
|
21
29
|
headers: {
|
|
@@ -28,7 +36,67 @@ export class QdrantProvider extends BaseVectorProvider {
|
|
|
28
36
|
async initialize(): Promise<void> {
|
|
29
37
|
await this.ping();
|
|
30
38
|
await this.ensureCollection();
|
|
31
|
-
|
|
39
|
+
|
|
40
|
+
// 1. Discover schema from existing data
|
|
41
|
+
console.log(`[QdrantProvider] 🔍 Discovering schema for collection "${this.indexName}"...`);
|
|
42
|
+
const discoveredFields = await this.discoverSchema();
|
|
43
|
+
|
|
44
|
+
// 2. Add client-provided fields (if any) as fallback/override
|
|
45
|
+
const opts = this.config.options as Record<string, unknown>;
|
|
46
|
+
const configFields = (opts.filterableFields as string[]) || [];
|
|
47
|
+
|
|
48
|
+
const allFields = [...new Set([...discoveredFields, ...configFields])];
|
|
49
|
+
|
|
50
|
+
if (allFields.length > 0) {
|
|
51
|
+
console.log(`[QdrantProvider] 🛠 Ensuring indexes for ${allFields.length} discovered fields...`);
|
|
52
|
+
for (const fieldInfo of allFields) {
|
|
53
|
+
const [fieldName, schemaType] = fieldInfo.split(':');
|
|
54
|
+
await this.ensureIndex(fieldName, (schemaType as any) || 'keyword');
|
|
55
|
+
}
|
|
56
|
+
} else {
|
|
57
|
+
console.log(`[QdrantProvider] ℹ️ No fields discovered for indexing.`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Samples points from the collection to discover available payload fields.
|
|
63
|
+
*/
|
|
64
|
+
private async discoverSchema(): Promise<string[]> {
|
|
65
|
+
try {
|
|
66
|
+
// Fetch 20 points to get a representative sample of the schema
|
|
67
|
+
const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
|
|
68
|
+
limit: 20,
|
|
69
|
+
with_payload: true,
|
|
70
|
+
with_vector: false
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
const points = data.result?.points || [];
|
|
74
|
+
const keys = new Set<string>();
|
|
75
|
+
|
|
76
|
+
for (const point of points) {
|
|
77
|
+
const payload = point.payload || {};
|
|
78
|
+
|
|
79
|
+
// Extract root keys
|
|
80
|
+
Object.keys(payload).forEach(k => {
|
|
81
|
+
if (k !== 'namespace' && k !== this.contentField && k !== this.metadataField) {
|
|
82
|
+
keys.add(k);
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
// Extract nested metadata keys if applicable
|
|
87
|
+
if (!this.isFlatPayload && payload[this.metadataField]) {
|
|
88
|
+
const meta = payload[this.metadataField];
|
|
89
|
+
if (typeof meta === 'object' && meta !== null) {
|
|
90
|
+
Object.keys(meta).forEach(k => keys.add(k));
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return Array.from(keys);
|
|
96
|
+
} catch (err) {
|
|
97
|
+
console.warn(`[QdrantProvider] ⚠️ Schema discovery failed:`, err instanceof Error ? err.message : String(err));
|
|
98
|
+
return [];
|
|
99
|
+
}
|
|
32
100
|
}
|
|
33
101
|
|
|
34
102
|
/**
|
|
@@ -58,32 +126,25 @@ export class QdrantProvider extends BaseVectorProvider {
|
|
|
58
126
|
}
|
|
59
127
|
|
|
60
128
|
/**
|
|
61
|
-
* Ensures that
|
|
62
|
-
* Qdrant requires this for search filters to work in many configurations.
|
|
129
|
+
* Ensures that a payload field has an index.
|
|
63
130
|
*/
|
|
64
|
-
private async ensureIndex(): Promise<void> {
|
|
131
|
+
private async ensureIndex(fieldName: string, schema: string = 'keyword'): Promise<void> {
|
|
132
|
+
let fullPath = fieldName;
|
|
133
|
+
if (!this.isFlatPayload && !fieldName.includes('.') && fieldName !== 'namespace' && fieldName !== this.contentField) {
|
|
134
|
+
fullPath = `${this.metadataField}.${fieldName}`;
|
|
135
|
+
}
|
|
136
|
+
|
|
65
137
|
try {
|
|
66
138
|
await this.http.put(`/collections/${this.indexName}/index`, {
|
|
67
|
-
field_name:
|
|
68
|
-
field_schema:
|
|
139
|
+
field_name: fullPath,
|
|
140
|
+
field_schema: schema,
|
|
69
141
|
});
|
|
70
|
-
console.log(`[QdrantProvider] ✅ Ensured
|
|
142
|
+
console.log(`[QdrantProvider] ✅ Ensured ${schema} index for "${fullPath}"`);
|
|
71
143
|
} catch (err) {
|
|
72
144
|
let status: number | undefined;
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
status = err.response?.status;
|
|
77
|
-
data = err.response?.data;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
// 409 means it already exists, which is fine.
|
|
81
|
-
if (status === 409) {
|
|
82
|
-
return;
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
86
|
-
console.warn(`[QdrantProvider] ⚠️ Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
|
|
145
|
+
if (axios.isAxiosError(err)) status = err.response?.status;
|
|
146
|
+
if (status === 409) return;
|
|
147
|
+
console.warn(`[QdrantProvider] ⚠️ Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
|
|
87
148
|
}
|
|
88
149
|
}
|
|
89
150
|
|
|
@@ -93,21 +154,49 @@ export class QdrantProvider extends BaseVectorProvider {
|
|
|
93
154
|
|
|
94
155
|
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
95
156
|
const payload = {
|
|
96
|
-
points: docs.map(doc =>
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
payload: {
|
|
100
|
-
content: doc.content,
|
|
101
|
-
metadata: doc.metadata || {},
|
|
157
|
+
points: docs.map(doc => {
|
|
158
|
+
const pointPayload: Record<string, any> = {
|
|
159
|
+
[this.contentField]: doc.content,
|
|
102
160
|
...(namespace ? { namespace } : {}),
|
|
103
|
-
}
|
|
104
|
-
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
if (this.isFlatPayload) {
|
|
164
|
+
Object.assign(pointPayload, doc.metadata || {});
|
|
165
|
+
} else {
|
|
166
|
+
pointPayload[this.metadataField] = doc.metadata || {};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
return {
|
|
170
|
+
id: this.normalizeId(doc.id),
|
|
171
|
+
vector: doc.vector,
|
|
172
|
+
payload: pointPayload,
|
|
173
|
+
};
|
|
174
|
+
}),
|
|
105
175
|
};
|
|
106
176
|
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
107
177
|
}
|
|
108
178
|
|
|
109
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
110
179
|
async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
|
|
180
|
+
const must: any[] = [];
|
|
181
|
+
|
|
182
|
+
if (namespace) {
|
|
183
|
+
must.push({ key: 'namespace', match: { value: namespace } });
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
187
|
+
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
188
|
+
let filterKey = key;
|
|
189
|
+
if (!this.isFlatPayload && !key.includes('.') && key !== 'namespace' && key !== this.contentField) {
|
|
190
|
+
filterKey = `${this.metadataField}.${key}`;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
if (Array.isArray(value)) {
|
|
194
|
+
must.push({ key: filterKey, match: { any: value } });
|
|
195
|
+
} else {
|
|
196
|
+
must.push({ key: filterKey, match: { value: value } });
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
111
200
|
const payload = {
|
|
112
201
|
vector: vector,
|
|
113
202
|
limit: topK,
|
|
@@ -116,20 +205,48 @@ export class QdrantProvider extends BaseVectorProvider {
|
|
|
116
205
|
hnsw_ef: (this.config.options.efSearch as number) || Math.max(topK * 20, 128),
|
|
117
206
|
exact: false
|
|
118
207
|
},
|
|
119
|
-
filter:
|
|
120
|
-
must: [{ key: 'namespace', match: { value: namespace } }]
|
|
121
|
-
} : undefined,
|
|
208
|
+
filter: must.length > 0 ? { must } : undefined,
|
|
122
209
|
};
|
|
210
|
+
|
|
123
211
|
const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
content
|
|
128
|
-
|
|
129
|
-
|
|
212
|
+
|
|
213
|
+
const results = (data.result || []).map((res: Record<string, any>) => {
|
|
214
|
+
const p = res.payload || {};
|
|
215
|
+
let content = p[this.contentField] || '';
|
|
216
|
+
|
|
217
|
+
if (!content) {
|
|
218
|
+
const stringFields = Object.entries(p)
|
|
219
|
+
.filter(([_, v]) => typeof v === 'string')
|
|
220
|
+
.map(([k, v]) => ({ key: k, val: v as string }));
|
|
221
|
+
|
|
222
|
+
if (stringFields.length > 0) {
|
|
223
|
+
const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
|
|
224
|
+
content = bestMatch.val;
|
|
225
|
+
} else {
|
|
226
|
+
content = JSON.stringify(p);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
let metadata: Record<string, any> = {};
|
|
231
|
+
if (this.isFlatPayload) {
|
|
232
|
+
metadata = { ...p };
|
|
233
|
+
delete metadata[this.contentField];
|
|
234
|
+
delete metadata['namespace'];
|
|
235
|
+
} else {
|
|
236
|
+
metadata = p[this.metadataField] || p;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
return {
|
|
240
|
+
id: res.id as string,
|
|
241
|
+
score: res.score as number,
|
|
242
|
+
content,
|
|
243
|
+
metadata,
|
|
244
|
+
};
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
return results;
|
|
130
248
|
}
|
|
131
249
|
|
|
132
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
133
250
|
async delete(id: string | number, _namespace?: string): Promise<void> {
|
|
134
251
|
await this.http.post(`/collections/${this.indexName}/points/delete`, {
|
|
135
252
|
points: [this.normalizeId(id)],
|
|
@@ -153,21 +270,16 @@ export class QdrantProvider extends BaseVectorProvider {
|
|
|
153
270
|
}
|
|
154
271
|
}
|
|
155
272
|
|
|
156
|
-
/**
|
|
157
|
-
* Normalizes an ID into a format Qdrant accepts (UUID or integer).
|
|
158
|
-
* For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
|
|
159
|
-
*/
|
|
160
273
|
private normalizeId(id: string | number): string | number {
|
|
161
274
|
if (typeof id === 'number') return id;
|
|
162
|
-
|
|
163
|
-
// Check if already a valid UUID
|
|
164
275
|
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
165
276
|
if (uuidRegex.test(id)) return id.toLowerCase();
|
|
166
|
-
|
|
167
|
-
// Hash string to deterministic UUID
|
|
168
277
|
const hash = crypto.createHash('md5').update(id).digest('hex');
|
|
169
278
|
return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
|
|
170
279
|
}
|
|
171
280
|
|
|
172
281
|
async disconnect(): Promise<void> {}
|
|
173
282
|
}
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
|
package/dist/chunk-3DSHW676.mjs
DELETED
|
@@ -1,159 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
BaseVectorProvider
|
|
3
|
-
} from "./chunk-IMP6FUCY.mjs";
|
|
4
|
-
import {
|
|
5
|
-
__spreadValues
|
|
6
|
-
} from "./chunk-X4TOT24V.mjs";
|
|
7
|
-
|
|
8
|
-
// src/providers/vectordb/QdrantProvider.ts
|
|
9
|
-
import axios from "axios";
|
|
10
|
-
import crypto from "crypto";
|
|
11
|
-
var QdrantProvider = class extends BaseVectorProvider {
|
|
12
|
-
constructor(config) {
|
|
13
|
-
super(config);
|
|
14
|
-
const opts = config.options;
|
|
15
|
-
const baseUrl = opts.baseUrl;
|
|
16
|
-
if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
|
|
17
|
-
this.http = axios.create({
|
|
18
|
-
baseURL: baseUrl,
|
|
19
|
-
headers: __spreadValues({
|
|
20
|
-
"Content-Type": "application/json"
|
|
21
|
-
}, opts.apiKey ? { "api-key": opts.apiKey } : {})
|
|
22
|
-
});
|
|
23
|
-
}
|
|
24
|
-
async initialize() {
|
|
25
|
-
await this.ping();
|
|
26
|
-
await this.ensureCollection();
|
|
27
|
-
await this.ensureIndex();
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* Ensures the collection exists. Creates it if missing.
|
|
31
|
-
*/
|
|
32
|
-
async ensureCollection() {
|
|
33
|
-
var _a;
|
|
34
|
-
try {
|
|
35
|
-
await this.http.get(`/collections/${this.indexName}`);
|
|
36
|
-
console.log(`[QdrantProvider] \u2705 Collection "${this.indexName}" already exists.`);
|
|
37
|
-
} catch (err) {
|
|
38
|
-
if (axios.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
|
|
39
|
-
const opts = this.config.options;
|
|
40
|
-
const dimensionsForCreate = opts.dimensions || 1536;
|
|
41
|
-
console.log(`[QdrantProvider] \u23F3 Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
|
|
42
|
-
await this.http.put(`/collections/${this.indexName}`, {
|
|
43
|
-
vectors: {
|
|
44
|
-
size: dimensionsForCreate,
|
|
45
|
-
distance: "Cosine"
|
|
46
|
-
}
|
|
47
|
-
});
|
|
48
|
-
console.log(`[QdrantProvider] \u2705 Created collection "${this.indexName}"`);
|
|
49
|
-
} else {
|
|
50
|
-
throw err;
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
/**
|
|
55
|
-
* Ensures that the 'namespace' field has a keyword index for efficient filtering.
|
|
56
|
-
* Qdrant requires this for search filters to work in many configurations.
|
|
57
|
-
*/
|
|
58
|
-
async ensureIndex() {
|
|
59
|
-
var _a, _b;
|
|
60
|
-
try {
|
|
61
|
-
await this.http.put(`/collections/${this.indexName}/index`, {
|
|
62
|
-
field_name: "namespace",
|
|
63
|
-
field_schema: "keyword"
|
|
64
|
-
});
|
|
65
|
-
console.log(`[QdrantProvider] \u2705 Ensured keyword index for "namespace" on collection "${this.indexName}"`);
|
|
66
|
-
} catch (err) {
|
|
67
|
-
let status;
|
|
68
|
-
let data;
|
|
69
|
-
if (axios.isAxiosError(err)) {
|
|
70
|
-
status = (_a = err.response) == null ? void 0 : _a.status;
|
|
71
|
-
data = (_b = err.response) == null ? void 0 : _b.data;
|
|
72
|
-
}
|
|
73
|
-
if (status === 409) {
|
|
74
|
-
return;
|
|
75
|
-
}
|
|
76
|
-
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
77
|
-
console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
async upsert(doc, namespace) {
|
|
81
|
-
await this.batchUpsert([doc], namespace);
|
|
82
|
-
}
|
|
83
|
-
async batchUpsert(docs, namespace) {
|
|
84
|
-
const payload = {
|
|
85
|
-
points: docs.map((doc) => ({
|
|
86
|
-
id: this.normalizeId(doc.id),
|
|
87
|
-
vector: doc.vector,
|
|
88
|
-
payload: __spreadValues({
|
|
89
|
-
content: doc.content,
|
|
90
|
-
metadata: doc.metadata || {}
|
|
91
|
-
}, namespace ? { namespace } : {})
|
|
92
|
-
}))
|
|
93
|
-
};
|
|
94
|
-
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
95
|
-
}
|
|
96
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
97
|
-
async query(vector, topK, namespace, _filter) {
|
|
98
|
-
const payload = {
|
|
99
|
-
vector,
|
|
100
|
-
limit: topK,
|
|
101
|
-
with_payload: true,
|
|
102
|
-
params: {
|
|
103
|
-
hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
|
|
104
|
-
exact: false
|
|
105
|
-
},
|
|
106
|
-
filter: namespace ? {
|
|
107
|
-
must: [{ key: "namespace", match: { value: namespace } }]
|
|
108
|
-
} : void 0
|
|
109
|
-
};
|
|
110
|
-
const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
|
|
111
|
-
return (data.result || []).map((res) => {
|
|
112
|
-
var _a, _b;
|
|
113
|
-
return {
|
|
114
|
-
id: res["id"],
|
|
115
|
-
score: res["score"],
|
|
116
|
-
content: ((_a = res["payload"]) == null ? void 0 : _a.content) || "",
|
|
117
|
-
metadata: ((_b = res["payload"]) == null ? void 0 : _b.metadata) || {}
|
|
118
|
-
};
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
122
|
-
async delete(id, _namespace) {
|
|
123
|
-
await this.http.post(`/collections/${this.indexName}/points/delete`, {
|
|
124
|
-
points: [this.normalizeId(id)]
|
|
125
|
-
});
|
|
126
|
-
}
|
|
127
|
-
async deleteNamespace(_namespace) {
|
|
128
|
-
await this.http.post(`/collections/${this.indexName}/points/delete`, {
|
|
129
|
-
filter: {
|
|
130
|
-
must: [{ key: "namespace", match: { value: _namespace } }]
|
|
131
|
-
}
|
|
132
|
-
});
|
|
133
|
-
}
|
|
134
|
-
async ping() {
|
|
135
|
-
try {
|
|
136
|
-
await this.http.get("/healthz");
|
|
137
|
-
return true;
|
|
138
|
-
} catch (e) {
|
|
139
|
-
return false;
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* Normalizes an ID into a format Qdrant accepts (UUID or integer).
|
|
144
|
-
* For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
|
|
145
|
-
*/
|
|
146
|
-
normalizeId(id) {
|
|
147
|
-
if (typeof id === "number") return id;
|
|
148
|
-
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
149
|
-
if (uuidRegex.test(id)) return id.toLowerCase();
|
|
150
|
-
const hash = crypto.createHash("md5").update(id).digest("hex");
|
|
151
|
-
return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
|
|
152
|
-
}
|
|
153
|
-
async disconnect() {
|
|
154
|
-
}
|
|
155
|
-
};
|
|
156
|
-
|
|
157
|
-
export {
|
|
158
|
-
QdrantProvider
|
|
159
|
-
};
|