@retrivora-ai/rag-engine 1.0.8 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  QdrantProvider
3
- } from "./chunk-3DSHW676.mjs";
3
+ } from "./chunk-65S4BQL2.mjs";
4
4
  import "./chunk-IMP6FUCY.mjs";
5
5
  import "./chunk-X4TOT24V.mjs";
6
6
  export {
@@ -0,0 +1,238 @@
1
+ import {
2
+ BaseVectorProvider
3
+ } from "./chunk-IMP6FUCY.mjs";
4
+ import {
5
+ __spreadValues
6
+ } from "./chunk-X4TOT24V.mjs";
7
+
8
+ // src/providers/vectordb/QdrantProvider.ts
9
+ import axios from "axios";
10
+ import crypto from "crypto";
11
+ var QdrantProvider = class extends BaseVectorProvider {
12
+ constructor(config) {
13
+ super(config);
14
+ const opts = config.options;
15
+ const baseUrl = opts.baseUrl;
16
+ if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
17
+ this.contentField = opts.contentField || "content";
18
+ this.metadataField = opts.metadataField || "metadata";
19
+ this.isFlatPayload = !!opts.flatPayload;
20
+ this.http = axios.create({
21
+ baseURL: baseUrl,
22
+ headers: __spreadValues({
23
+ "Content-Type": "application/json"
24
+ }, opts.apiKey ? { "api-key": opts.apiKey } : {})
25
+ });
26
+ }
27
+ async initialize() {
28
+ await this.ping();
29
+ await this.ensureCollection();
30
+ console.log(`[QdrantProvider] \u{1F50D} Discovering schema for collection "${this.indexName}"...`);
31
+ const discoveredFields = await this.discoverSchema();
32
+ const opts = this.config.options;
33
+ const configFields = opts.filterableFields || [];
34
+ const allFields = [.../* @__PURE__ */ new Set([...discoveredFields, ...configFields])];
35
+ if (allFields.length > 0) {
36
+ console.log(`[QdrantProvider] \u{1F6E0} Ensuring indexes for ${allFields.length} discovered fields...`);
37
+ for (const fieldInfo of allFields) {
38
+ const [fieldName, schemaType] = fieldInfo.split(":");
39
+ await this.ensureIndex(fieldName, schemaType || "keyword");
40
+ }
41
+ } else {
42
+ console.log(`[QdrantProvider] \u2139\uFE0F No fields discovered for indexing.`);
43
+ }
44
+ }
45
+ /**
46
+ * Samples points from the collection to discover available payload fields.
47
+ */
48
+ async discoverSchema() {
49
+ var _a;
50
+ try {
51
+ const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
52
+ limit: 20,
53
+ with_payload: true,
54
+ with_vector: false
55
+ });
56
+ const points = ((_a = data.result) == null ? void 0 : _a.points) || [];
57
+ const keys = /* @__PURE__ */ new Set();
58
+ for (const point of points) {
59
+ const payload = point.payload || {};
60
+ Object.keys(payload).forEach((k) => {
61
+ if (k !== "namespace" && k !== this.contentField && k !== this.metadataField) {
62
+ keys.add(k);
63
+ }
64
+ });
65
+ if (!this.isFlatPayload && payload[this.metadataField]) {
66
+ const meta = payload[this.metadataField];
67
+ if (typeof meta === "object" && meta !== null) {
68
+ Object.keys(meta).forEach((k) => keys.add(k));
69
+ }
70
+ }
71
+ }
72
+ return Array.from(keys);
73
+ } catch (err) {
74
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Schema discovery failed:`, err instanceof Error ? err.message : String(err));
75
+ return [];
76
+ }
77
+ }
78
+ /**
79
+ * Ensures the collection exists. Creates it if missing.
80
+ */
81
+ async ensureCollection() {
82
+ var _a;
83
+ try {
84
+ await this.http.get(`/collections/${this.indexName}`);
85
+ console.log(`[QdrantProvider] \u2705 Collection "${this.indexName}" already exists.`);
86
+ } catch (err) {
87
+ if (axios.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
88
+ const opts = this.config.options;
89
+ const dimensionsForCreate = opts.dimensions || 1536;
90
+ console.log(`[QdrantProvider] \u23F3 Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
91
+ await this.http.put(`/collections/${this.indexName}`, {
92
+ vectors: {
93
+ size: dimensionsForCreate,
94
+ distance: "Cosine"
95
+ }
96
+ });
97
+ console.log(`[QdrantProvider] \u2705 Created collection "${this.indexName}"`);
98
+ } else {
99
+ throw err;
100
+ }
101
+ }
102
+ }
103
+ /**
104
+ * Ensures that a payload field has an index.
105
+ */
106
+ async ensureIndex(fieldName, schema = "keyword") {
107
+ var _a;
108
+ let fullPath = fieldName;
109
+ if (!this.isFlatPayload && !fieldName.includes(".") && fieldName !== "namespace" && fieldName !== this.contentField) {
110
+ fullPath = `${this.metadataField}.${fieldName}`;
111
+ }
112
+ try {
113
+ await this.http.put(`/collections/${this.indexName}/index`, {
114
+ field_name: fullPath,
115
+ field_schema: schema
116
+ });
117
+ console.log(`[QdrantProvider] \u2705 Ensured ${schema} index for "${fullPath}"`);
118
+ } catch (err) {
119
+ let status;
120
+ if (axios.isAxiosError(err)) status = (_a = err.response) == null ? void 0 : _a.status;
121
+ if (status === 409) return;
122
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
123
+ }
124
+ }
125
+ async upsert(doc, namespace) {
126
+ await this.batchUpsert([doc], namespace);
127
+ }
128
+ async batchUpsert(docs, namespace) {
129
+ const payload = {
130
+ points: docs.map((doc) => {
131
+ const pointPayload = __spreadValues({
132
+ [this.contentField]: doc.content
133
+ }, namespace ? { namespace } : {});
134
+ if (this.isFlatPayload) {
135
+ Object.assign(pointPayload, doc.metadata || {});
136
+ } else {
137
+ pointPayload[this.metadataField] = doc.metadata || {};
138
+ }
139
+ return {
140
+ id: this.normalizeId(doc.id),
141
+ vector: doc.vector,
142
+ payload: pointPayload
143
+ };
144
+ })
145
+ };
146
+ await this.http.put(`/collections/${this.indexName}/points`, payload);
147
+ }
148
+ async query(vector, topK, namespace, _filter) {
149
+ const must = [];
150
+ if (namespace) {
151
+ must.push({ key: "namespace", match: { value: namespace } });
152
+ }
153
+ const sanitizedFilter = this.sanitizeFilter(_filter);
154
+ for (const [key, value] of Object.entries(sanitizedFilter)) {
155
+ let filterKey = key;
156
+ if (!this.isFlatPayload && !key.includes(".") && key !== "namespace" && key !== this.contentField) {
157
+ filterKey = `${this.metadataField}.${key}`;
158
+ }
159
+ if (Array.isArray(value)) {
160
+ must.push({ key: filterKey, match: { any: value } });
161
+ } else {
162
+ must.push({ key: filterKey, match: { value } });
163
+ }
164
+ }
165
+ const payload = {
166
+ vector,
167
+ limit: topK,
168
+ with_payload: true,
169
+ params: {
170
+ hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
171
+ exact: false
172
+ },
173
+ filter: must.length > 0 ? { must } : void 0
174
+ };
175
+ const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
176
+ const results = (data.result || []).map((res) => {
177
+ const p = res.payload || {};
178
+ let content = p[this.contentField] || "";
179
+ if (!content) {
180
+ const stringFields = Object.entries(p).filter(([_, v]) => typeof v === "string").map(([k, v]) => ({ key: k, val: v }));
181
+ if (stringFields.length > 0) {
182
+ const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
183
+ content = bestMatch.val;
184
+ } else {
185
+ content = JSON.stringify(p);
186
+ }
187
+ }
188
+ let metadata = {};
189
+ if (this.isFlatPayload) {
190
+ metadata = __spreadValues({}, p);
191
+ delete metadata[this.contentField];
192
+ delete metadata["namespace"];
193
+ } else {
194
+ metadata = p[this.metadataField] || p;
195
+ }
196
+ return {
197
+ id: res.id,
198
+ score: res.score,
199
+ content,
200
+ metadata
201
+ };
202
+ });
203
+ return results;
204
+ }
205
+ async delete(id, _namespace) {
206
+ await this.http.post(`/collections/${this.indexName}/points/delete`, {
207
+ points: [this.normalizeId(id)]
208
+ });
209
+ }
210
+ async deleteNamespace(_namespace) {
211
+ await this.http.post(`/collections/${this.indexName}/points/delete`, {
212
+ filter: {
213
+ must: [{ key: "namespace", match: { value: _namespace } }]
214
+ }
215
+ });
216
+ }
217
+ async ping() {
218
+ try {
219
+ await this.http.get("/healthz");
220
+ return true;
221
+ } catch (e) {
222
+ return false;
223
+ }
224
+ }
225
+ normalizeId(id) {
226
+ if (typeof id === "number") return id;
227
+ const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
228
+ if (uuidRegex.test(id)) return id.toLowerCase();
229
+ const hash = crypto.createHash("md5").update(id).digest("hex");
230
+ return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
231
+ }
232
+ async disconnect() {
233
+ }
234
+ };
235
+
236
+ export {
237
+ QdrantProvider
238
+ };
@@ -1106,7 +1106,7 @@ var ProviderRegistry = class {
1106
1106
  return MilvusProvider;
1107
1107
  }
1108
1108
  case "qdrant": {
1109
- const { QdrantProvider } = await import("./QdrantProvider-YNUNEOZH.mjs");
1109
+ const { QdrantProvider } = await import("./QdrantProvider-WWXFX2XF.mjs");
1110
1110
  return QdrantProvider;
1111
1111
  }
1112
1112
  case "chromadb": {
@@ -2273,6 +2273,7 @@ var Pipeline = class {
2273
2273
  }
2274
2274
  const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
2275
2275
  const filter = QueryProcessor.buildQueryFilter(question, hints);
2276
+ console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
2276
2277
  const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
2277
2278
  namespace: ns,
2278
2279
  topK: topK * 2,
@@ -845,6 +845,9 @@ var init_QdrantProvider = __esm({
845
845
  const opts = config.options;
846
846
  const baseUrl = opts.baseUrl;
847
847
  if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
848
+ this.contentField = opts.contentField || "content";
849
+ this.metadataField = opts.metadataField || "metadata";
850
+ this.isFlatPayload = !!opts.flatPayload;
848
851
  this.http = import_axios4.default.create({
849
852
  baseURL: baseUrl,
850
853
  headers: __spreadValues({
@@ -855,7 +858,53 @@ var init_QdrantProvider = __esm({
855
858
  async initialize() {
856
859
  await this.ping();
857
860
  await this.ensureCollection();
858
- await this.ensureIndex();
861
+ console.log(`[QdrantProvider] \u{1F50D} Discovering schema for collection "${this.indexName}"...`);
862
+ const discoveredFields = await this.discoverSchema();
863
+ const opts = this.config.options;
864
+ const configFields = opts.filterableFields || [];
865
+ const allFields = [.../* @__PURE__ */ new Set([...discoveredFields, ...configFields])];
866
+ if (allFields.length > 0) {
867
+ console.log(`[QdrantProvider] \u{1F6E0} Ensuring indexes for ${allFields.length} discovered fields...`);
868
+ for (const fieldInfo of allFields) {
869
+ const [fieldName, schemaType] = fieldInfo.split(":");
870
+ await this.ensureIndex(fieldName, schemaType || "keyword");
871
+ }
872
+ } else {
873
+ console.log(`[QdrantProvider] \u2139\uFE0F No fields discovered for indexing.`);
874
+ }
875
+ }
876
+ /**
877
+ * Samples points from the collection to discover available payload fields.
878
+ */
879
+ async discoverSchema() {
880
+ var _a;
881
+ try {
882
+ const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
883
+ limit: 20,
884
+ with_payload: true,
885
+ with_vector: false
886
+ });
887
+ const points = ((_a = data.result) == null ? void 0 : _a.points) || [];
888
+ const keys = /* @__PURE__ */ new Set();
889
+ for (const point of points) {
890
+ const payload = point.payload || {};
891
+ Object.keys(payload).forEach((k) => {
892
+ if (k !== "namespace" && k !== this.contentField && k !== this.metadataField) {
893
+ keys.add(k);
894
+ }
895
+ });
896
+ if (!this.isFlatPayload && payload[this.metadataField]) {
897
+ const meta = payload[this.metadataField];
898
+ if (typeof meta === "object" && meta !== null) {
899
+ Object.keys(meta).forEach((k) => keys.add(k));
900
+ }
901
+ }
902
+ }
903
+ return Array.from(keys);
904
+ } catch (err) {
905
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Schema discovery failed:`, err instanceof Error ? err.message : String(err));
906
+ return [];
907
+ }
859
908
  }
860
909
  /**
861
910
  * Ensures the collection exists. Creates it if missing.
@@ -883,29 +932,25 @@ var init_QdrantProvider = __esm({
883
932
  }
884
933
  }
885
934
  /**
886
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
887
- * Qdrant requires this for search filters to work in many configurations.
935
+ * Ensures that a payload field has an index.
888
936
  */
889
- async ensureIndex() {
890
- var _a, _b;
937
+ async ensureIndex(fieldName, schema = "keyword") {
938
+ var _a;
939
+ let fullPath = fieldName;
940
+ if (!this.isFlatPayload && !fieldName.includes(".") && fieldName !== "namespace" && fieldName !== this.contentField) {
941
+ fullPath = `${this.metadataField}.${fieldName}`;
942
+ }
891
943
  try {
892
944
  await this.http.put(`/collections/${this.indexName}/index`, {
893
- field_name: "namespace",
894
- field_schema: "keyword"
945
+ field_name: fullPath,
946
+ field_schema: schema
895
947
  });
896
- console.log(`[QdrantProvider] \u2705 Ensured keyword index for "namespace" on collection "${this.indexName}"`);
948
+ console.log(`[QdrantProvider] \u2705 Ensured ${schema} index for "${fullPath}"`);
897
949
  } catch (err) {
898
950
  let status;
899
- let data;
900
- if (import_axios4.default.isAxiosError(err)) {
901
- status = (_a = err.response) == null ? void 0 : _a.status;
902
- data = (_b = err.response) == null ? void 0 : _b.data;
903
- }
904
- if (status === 409) {
905
- return;
906
- }
907
- const errorMessage = err instanceof Error ? err.message : String(err);
908
- console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
951
+ if (import_axios4.default.isAxiosError(err)) status = (_a = err.response) == null ? void 0 : _a.status;
952
+ if (status === 409) return;
953
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
909
954
  }
910
955
  }
911
956
  async upsert(doc, namespace) {
@@ -913,19 +958,41 @@ var init_QdrantProvider = __esm({
913
958
  }
914
959
  async batchUpsert(docs, namespace) {
915
960
  const payload = {
916
- points: docs.map((doc) => ({
917
- id: this.normalizeId(doc.id),
918
- vector: doc.vector,
919
- payload: __spreadValues({
920
- content: doc.content,
921
- metadata: doc.metadata || {}
922
- }, namespace ? { namespace } : {})
923
- }))
961
+ points: docs.map((doc) => {
962
+ const pointPayload = __spreadValues({
963
+ [this.contentField]: doc.content
964
+ }, namespace ? { namespace } : {});
965
+ if (this.isFlatPayload) {
966
+ Object.assign(pointPayload, doc.metadata || {});
967
+ } else {
968
+ pointPayload[this.metadataField] = doc.metadata || {};
969
+ }
970
+ return {
971
+ id: this.normalizeId(doc.id),
972
+ vector: doc.vector,
973
+ payload: pointPayload
974
+ };
975
+ })
924
976
  };
925
977
  await this.http.put(`/collections/${this.indexName}/points`, payload);
926
978
  }
927
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
928
979
  async query(vector, topK, namespace, _filter) {
980
+ const must = [];
981
+ if (namespace) {
982
+ must.push({ key: "namespace", match: { value: namespace } });
983
+ }
984
+ const sanitizedFilter = this.sanitizeFilter(_filter);
985
+ for (const [key, value] of Object.entries(sanitizedFilter)) {
986
+ let filterKey = key;
987
+ if (!this.isFlatPayload && !key.includes(".") && key !== "namespace" && key !== this.contentField) {
988
+ filterKey = `${this.metadataField}.${key}`;
989
+ }
990
+ if (Array.isArray(value)) {
991
+ must.push({ key: filterKey, match: { any: value } });
992
+ } else {
993
+ must.push({ key: filterKey, match: { value } });
994
+ }
995
+ }
929
996
  const payload = {
930
997
  vector,
931
998
  limit: topK,
@@ -934,22 +1001,38 @@ var init_QdrantProvider = __esm({
934
1001
  hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
935
1002
  exact: false
936
1003
  },
937
- filter: namespace ? {
938
- must: [{ key: "namespace", match: { value: namespace } }]
939
- } : void 0
1004
+ filter: must.length > 0 ? { must } : void 0
940
1005
  };
941
1006
  const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
942
- return (data.result || []).map((res) => {
943
- var _a, _b;
1007
+ const results = (data.result || []).map((res) => {
1008
+ const p = res.payload || {};
1009
+ let content = p[this.contentField] || "";
1010
+ if (!content) {
1011
+ const stringFields = Object.entries(p).filter(([_, v]) => typeof v === "string").map(([k, v]) => ({ key: k, val: v }));
1012
+ if (stringFields.length > 0) {
1013
+ const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
1014
+ content = bestMatch.val;
1015
+ } else {
1016
+ content = JSON.stringify(p);
1017
+ }
1018
+ }
1019
+ let metadata = {};
1020
+ if (this.isFlatPayload) {
1021
+ metadata = __spreadValues({}, p);
1022
+ delete metadata[this.contentField];
1023
+ delete metadata["namespace"];
1024
+ } else {
1025
+ metadata = p[this.metadataField] || p;
1026
+ }
944
1027
  return {
945
- id: res["id"],
946
- score: res["score"],
947
- content: ((_a = res["payload"]) == null ? void 0 : _a.content) || "",
948
- metadata: ((_b = res["payload"]) == null ? void 0 : _b.metadata) || {}
1028
+ id: res.id,
1029
+ score: res.score,
1030
+ content,
1031
+ metadata
949
1032
  };
950
1033
  });
1034
+ return results;
951
1035
  }
952
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
953
1036
  async delete(id, _namespace) {
954
1037
  await this.http.post(`/collections/${this.indexName}/points/delete`, {
955
1038
  points: [this.normalizeId(id)]
@@ -970,10 +1053,6 @@ var init_QdrantProvider = __esm({
970
1053
  return false;
971
1054
  }
972
1055
  }
973
- /**
974
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
975
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
976
- */
977
1056
  normalizeId(id) {
978
1057
  if (typeof id === "number") return id;
979
1058
  const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
@@ -3749,6 +3828,7 @@ var Pipeline = class {
3749
3828
  }
3750
3829
  const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
3751
3830
  const filter = QueryProcessor.buildQueryFilter(question, hints);
3831
+ console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
3752
3832
  const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
3753
3833
  namespace: ns,
3754
3834
  topK: topK * 2,
@@ -8,7 +8,7 @@ import {
8
8
  sseFrame,
9
9
  sseMetaFrame,
10
10
  sseTextFrame
11
- } from "../chunk-3GKA3PJF.mjs";
11
+ } from "../chunk-G2LVK46T.mjs";
12
12
  import "../chunk-YLTMFW4M.mjs";
13
13
  import "../chunk-X4TOT24V.mjs";
14
14
  export {
package/dist/server.d.mts CHANGED
@@ -731,18 +731,25 @@ declare class MilvusProvider extends BaseVectorProvider {
731
731
 
732
732
  /**
733
733
  * QdrantProvider — implementation for Qdrant using its REST API.
734
+ * Fully dynamic: automatically discovers and indexes schema from available data.
734
735
  */
735
736
  declare class QdrantProvider extends BaseVectorProvider {
736
737
  private http;
738
+ private contentField;
739
+ private metadataField;
740
+ private isFlatPayload;
737
741
  constructor(config: VectorDBConfig);
738
742
  initialize(): Promise<void>;
743
+ /**
744
+ * Samples points from the collection to discover available payload fields.
745
+ */
746
+ private discoverSchema;
739
747
  /**
740
748
  * Ensures the collection exists. Creates it if missing.
741
749
  */
742
750
  private ensureCollection;
743
751
  /**
744
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
745
- * Qdrant requires this for search filters to work in many configurations.
752
+ * Ensures that a payload field has an index.
746
753
  */
747
754
  private ensureIndex;
748
755
  upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
@@ -751,10 +758,6 @@ declare class QdrantProvider extends BaseVectorProvider {
751
758
  delete(id: string | number, _namespace?: string): Promise<void>;
752
759
  deleteNamespace(_namespace: string): Promise<void>;
753
760
  ping(): Promise<boolean>;
754
- /**
755
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
756
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
757
- */
758
761
  private normalizeId;
759
762
  disconnect(): Promise<void>;
760
763
  }
package/dist/server.d.ts CHANGED
@@ -731,18 +731,25 @@ declare class MilvusProvider extends BaseVectorProvider {
731
731
 
732
732
  /**
733
733
  * QdrantProvider — implementation for Qdrant using its REST API.
734
+ * Fully dynamic: automatically discovers and indexes schema from available data.
734
735
  */
735
736
  declare class QdrantProvider extends BaseVectorProvider {
736
737
  private http;
738
+ private contentField;
739
+ private metadataField;
740
+ private isFlatPayload;
737
741
  constructor(config: VectorDBConfig);
738
742
  initialize(): Promise<void>;
743
+ /**
744
+ * Samples points from the collection to discover available payload fields.
745
+ */
746
+ private discoverSchema;
739
747
  /**
740
748
  * Ensures the collection exists. Creates it if missing.
741
749
  */
742
750
  private ensureCollection;
743
751
  /**
744
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
745
- * Qdrant requires this for search filters to work in many configurations.
752
+ * Ensures that a payload field has an index.
746
753
  */
747
754
  private ensureIndex;
748
755
  upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
@@ -751,10 +758,6 @@ declare class QdrantProvider extends BaseVectorProvider {
751
758
  delete(id: string | number, _namespace?: string): Promise<void>;
752
759
  deleteNamespace(_namespace: string): Promise<void>;
753
760
  ping(): Promise<boolean>;
754
- /**
755
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
756
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
757
- */
758
761
  private normalizeId;
759
762
  disconnect(): Promise<void>;
760
763
  }
package/dist/server.js CHANGED
@@ -857,6 +857,9 @@ var init_QdrantProvider = __esm({
857
857
  const opts = config.options;
858
858
  const baseUrl = opts.baseUrl;
859
859
  if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
860
+ this.contentField = opts.contentField || "content";
861
+ this.metadataField = opts.metadataField || "metadata";
862
+ this.isFlatPayload = !!opts.flatPayload;
860
863
  this.http = import_axios4.default.create({
861
864
  baseURL: baseUrl,
862
865
  headers: __spreadValues({
@@ -867,7 +870,53 @@ var init_QdrantProvider = __esm({
867
870
  async initialize() {
868
871
  await this.ping();
869
872
  await this.ensureCollection();
870
- await this.ensureIndex();
873
+ console.log(`[QdrantProvider] \u{1F50D} Discovering schema for collection "${this.indexName}"...`);
874
+ const discoveredFields = await this.discoverSchema();
875
+ const opts = this.config.options;
876
+ const configFields = opts.filterableFields || [];
877
+ const allFields = [.../* @__PURE__ */ new Set([...discoveredFields, ...configFields])];
878
+ if (allFields.length > 0) {
879
+ console.log(`[QdrantProvider] \u{1F6E0} Ensuring indexes for ${allFields.length} discovered fields...`);
880
+ for (const fieldInfo of allFields) {
881
+ const [fieldName, schemaType] = fieldInfo.split(":");
882
+ await this.ensureIndex(fieldName, schemaType || "keyword");
883
+ }
884
+ } else {
885
+ console.log(`[QdrantProvider] \u2139\uFE0F No fields discovered for indexing.`);
886
+ }
887
+ }
888
+ /**
889
+ * Samples points from the collection to discover available payload fields.
890
+ */
891
+ async discoverSchema() {
892
+ var _a;
893
+ try {
894
+ const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
895
+ limit: 20,
896
+ with_payload: true,
897
+ with_vector: false
898
+ });
899
+ const points = ((_a = data.result) == null ? void 0 : _a.points) || [];
900
+ const keys = /* @__PURE__ */ new Set();
901
+ for (const point of points) {
902
+ const payload = point.payload || {};
903
+ Object.keys(payload).forEach((k) => {
904
+ if (k !== "namespace" && k !== this.contentField && k !== this.metadataField) {
905
+ keys.add(k);
906
+ }
907
+ });
908
+ if (!this.isFlatPayload && payload[this.metadataField]) {
909
+ const meta = payload[this.metadataField];
910
+ if (typeof meta === "object" && meta !== null) {
911
+ Object.keys(meta).forEach((k) => keys.add(k));
912
+ }
913
+ }
914
+ }
915
+ return Array.from(keys);
916
+ } catch (err) {
917
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Schema discovery failed:`, err instanceof Error ? err.message : String(err));
918
+ return [];
919
+ }
871
920
  }
872
921
  /**
873
922
  * Ensures the collection exists. Creates it if missing.
@@ -895,29 +944,25 @@ var init_QdrantProvider = __esm({
895
944
  }
896
945
  }
897
946
  /**
898
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
899
- * Qdrant requires this for search filters to work in many configurations.
947
+ * Ensures that a payload field has an index.
900
948
  */
901
- async ensureIndex() {
902
- var _a, _b;
949
+ async ensureIndex(fieldName, schema = "keyword") {
950
+ var _a;
951
+ let fullPath = fieldName;
952
+ if (!this.isFlatPayload && !fieldName.includes(".") && fieldName !== "namespace" && fieldName !== this.contentField) {
953
+ fullPath = `${this.metadataField}.${fieldName}`;
954
+ }
903
955
  try {
904
956
  await this.http.put(`/collections/${this.indexName}/index`, {
905
- field_name: "namespace",
906
- field_schema: "keyword"
957
+ field_name: fullPath,
958
+ field_schema: schema
907
959
  });
908
- console.log(`[QdrantProvider] \u2705 Ensured keyword index for "namespace" on collection "${this.indexName}"`);
960
+ console.log(`[QdrantProvider] \u2705 Ensured ${schema} index for "${fullPath}"`);
909
961
  } catch (err) {
910
962
  let status;
911
- let data;
912
- if (import_axios4.default.isAxiosError(err)) {
913
- status = (_a = err.response) == null ? void 0 : _a.status;
914
- data = (_b = err.response) == null ? void 0 : _b.data;
915
- }
916
- if (status === 409) {
917
- return;
918
- }
919
- const errorMessage = err instanceof Error ? err.message : String(err);
920
- console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
963
+ if (import_axios4.default.isAxiosError(err)) status = (_a = err.response) == null ? void 0 : _a.status;
964
+ if (status === 409) return;
965
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
921
966
  }
922
967
  }
923
968
  async upsert(doc, namespace) {
@@ -925,19 +970,41 @@ var init_QdrantProvider = __esm({
925
970
  }
926
971
  async batchUpsert(docs, namespace) {
927
972
  const payload = {
928
- points: docs.map((doc) => ({
929
- id: this.normalizeId(doc.id),
930
- vector: doc.vector,
931
- payload: __spreadValues({
932
- content: doc.content,
933
- metadata: doc.metadata || {}
934
- }, namespace ? { namespace } : {})
935
- }))
973
+ points: docs.map((doc) => {
974
+ const pointPayload = __spreadValues({
975
+ [this.contentField]: doc.content
976
+ }, namespace ? { namespace } : {});
977
+ if (this.isFlatPayload) {
978
+ Object.assign(pointPayload, doc.metadata || {});
979
+ } else {
980
+ pointPayload[this.metadataField] = doc.metadata || {};
981
+ }
982
+ return {
983
+ id: this.normalizeId(doc.id),
984
+ vector: doc.vector,
985
+ payload: pointPayload
986
+ };
987
+ })
936
988
  };
937
989
  await this.http.put(`/collections/${this.indexName}/points`, payload);
938
990
  }
939
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
940
991
  async query(vector, topK, namespace, _filter) {
992
+ const must = [];
993
+ if (namespace) {
994
+ must.push({ key: "namespace", match: { value: namespace } });
995
+ }
996
+ const sanitizedFilter = this.sanitizeFilter(_filter);
997
+ for (const [key, value] of Object.entries(sanitizedFilter)) {
998
+ let filterKey = key;
999
+ if (!this.isFlatPayload && !key.includes(".") && key !== "namespace" && key !== this.contentField) {
1000
+ filterKey = `${this.metadataField}.${key}`;
1001
+ }
1002
+ if (Array.isArray(value)) {
1003
+ must.push({ key: filterKey, match: { any: value } });
1004
+ } else {
1005
+ must.push({ key: filterKey, match: { value } });
1006
+ }
1007
+ }
941
1008
  const payload = {
942
1009
  vector,
943
1010
  limit: topK,
@@ -946,22 +1013,38 @@ var init_QdrantProvider = __esm({
946
1013
  hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
947
1014
  exact: false
948
1015
  },
949
- filter: namespace ? {
950
- must: [{ key: "namespace", match: { value: namespace } }]
951
- } : void 0
1016
+ filter: must.length > 0 ? { must } : void 0
952
1017
  };
953
1018
  const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
954
- return (data.result || []).map((res) => {
955
- var _a, _b;
1019
+ const results = (data.result || []).map((res) => {
1020
+ const p = res.payload || {};
1021
+ let content = p[this.contentField] || "";
1022
+ if (!content) {
1023
+ const stringFields = Object.entries(p).filter(([_, v]) => typeof v === "string").map(([k, v]) => ({ key: k, val: v }));
1024
+ if (stringFields.length > 0) {
1025
+ const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
1026
+ content = bestMatch.val;
1027
+ } else {
1028
+ content = JSON.stringify(p);
1029
+ }
1030
+ }
1031
+ let metadata = {};
1032
+ if (this.isFlatPayload) {
1033
+ metadata = __spreadValues({}, p);
1034
+ delete metadata[this.contentField];
1035
+ delete metadata["namespace"];
1036
+ } else {
1037
+ metadata = p[this.metadataField] || p;
1038
+ }
956
1039
  return {
957
- id: res["id"],
958
- score: res["score"],
959
- content: ((_a = res["payload"]) == null ? void 0 : _a.content) || "",
960
- metadata: ((_b = res["payload"]) == null ? void 0 : _b.metadata) || {}
1040
+ id: res.id,
1041
+ score: res.score,
1042
+ content,
1043
+ metadata
961
1044
  };
962
1045
  });
1046
+ return results;
963
1047
  }
964
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
965
1048
  async delete(id, _namespace) {
966
1049
  await this.http.post(`/collections/${this.indexName}/points/delete`, {
967
1050
  points: [this.normalizeId(id)]
@@ -982,10 +1065,6 @@ var init_QdrantProvider = __esm({
982
1065
  return false;
983
1066
  }
984
1067
  }
985
- /**
986
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
987
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
988
- */
989
1068
  normalizeId(id) {
990
1069
  if (typeof id === "number") return id;
991
1070
  const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
@@ -3842,6 +3921,7 @@ var Pipeline = class {
3842
3921
  }
3843
3922
  const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
3844
3923
  const filter = QueryProcessor.buildQueryFilter(question, hints);
3924
+ console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
3845
3925
  const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
3846
3926
  namespace: ns,
3847
3927
  topK: topK * 2,
package/dist/server.mjs CHANGED
@@ -40,7 +40,7 @@ import {
40
40
  sseFrame,
41
41
  sseMetaFrame,
42
42
  sseTextFrame
43
- } from "./chunk-3GKA3PJF.mjs";
43
+ } from "./chunk-G2LVK46T.mjs";
44
44
  import "./chunk-YLTMFW4M.mjs";
45
45
  import {
46
46
  PineconeProvider
@@ -56,7 +56,7 @@ import {
56
56
  } from "./chunk-U55XRW3U.mjs";
57
57
  import {
58
58
  QdrantProvider
59
- } from "./chunk-3DSHW676.mjs";
59
+ } from "./chunk-65S4BQL2.mjs";
60
60
  import {
61
61
  BaseVectorProvider
62
62
  } from "./chunk-IMP6FUCY.mjs";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@retrivora-ai/rag-engine",
3
- "version": "1.0.8",
3
+ "version": "1.1.0",
4
4
  "description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
5
5
  "author": "Abhinav Alkuchi",
6
6
  "license": "MIT",
@@ -317,6 +317,8 @@ export class Pipeline {
317
317
  // 2. Parallel Retrieval
318
318
  const hints = QueryProcessor.extractQueryFieldHints(question, this.config.rag?.filterableFields);
319
319
  const filter = QueryProcessor.buildQueryFilter(question, hints);
320
+
321
+ console.log(`[Pipeline] 🧩 Extracted filters:`, JSON.stringify(filter));
320
322
 
321
323
  const { sources: rawSources, graphData } = await this.retrieve(searchQuery, {
322
324
  namespace: ns,
@@ -6,9 +6,13 @@ import { VectorMatch, UpsertDocument } from '../../types';
6
6
 
7
7
  /**
8
8
  * QdrantProvider — implementation for Qdrant using its REST API.
9
+ * Fully dynamic: automatically discovers and indexes schema from available data.
9
10
  */
10
11
  export class QdrantProvider extends BaseVectorProvider {
11
12
  private http: AxiosInstance;
13
+ private contentField: string;
14
+ private metadataField: string;
15
+ private isFlatPayload: boolean;
12
16
 
13
17
  constructor(config: VectorDBConfig) {
14
18
  super(config);
@@ -16,6 +20,10 @@ export class QdrantProvider extends BaseVectorProvider {
16
20
  const baseUrl = opts.baseUrl as string;
17
21
  if (!baseUrl) throw new Error('[QdrantProvider] baseUrl is required');
18
22
 
23
+ this.contentField = (opts.contentField as string) || 'content';
24
+ this.metadataField = (opts.metadataField as string) || 'metadata';
25
+ this.isFlatPayload = !!opts.flatPayload;
26
+
19
27
  this.http = axios.create({
20
28
  baseURL: baseUrl,
21
29
  headers: {
@@ -28,7 +36,67 @@ export class QdrantProvider extends BaseVectorProvider {
28
36
  async initialize(): Promise<void> {
29
37
  await this.ping();
30
38
  await this.ensureCollection();
31
- await this.ensureIndex();
39
+
40
+ // 1. Discover schema from existing data
41
+ console.log(`[QdrantProvider] 🔍 Discovering schema for collection "${this.indexName}"...`);
42
+ const discoveredFields = await this.discoverSchema();
43
+
44
+ // 2. Add client-provided fields (if any) as fallback/override
45
+ const opts = this.config.options as Record<string, unknown>;
46
+ const configFields = (opts.filterableFields as string[]) || [];
47
+
48
+ const allFields = [...new Set([...discoveredFields, ...configFields])];
49
+
50
+ if (allFields.length > 0) {
51
+ console.log(`[QdrantProvider] 🛠 Ensuring indexes for ${allFields.length} discovered fields...`);
52
+ for (const fieldInfo of allFields) {
53
+ const [fieldName, schemaType] = fieldInfo.split(':');
54
+ await this.ensureIndex(fieldName, (schemaType as any) || 'keyword');
55
+ }
56
+ } else {
57
+ console.log(`[QdrantProvider] ℹ️ No fields discovered for indexing.`);
58
+ }
59
+ }
60
+
61
+ /**
62
+ * Samples points from the collection to discover available payload fields.
63
+ */
64
+ private async discoverSchema(): Promise<string[]> {
65
+ try {
66
+ // Fetch 20 points to get a representative sample of the schema
67
+ const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
68
+ limit: 20,
69
+ with_payload: true,
70
+ with_vector: false
71
+ });
72
+
73
+ const points = data.result?.points || [];
74
+ const keys = new Set<string>();
75
+
76
+ for (const point of points) {
77
+ const payload = point.payload || {};
78
+
79
+ // Extract root keys
80
+ Object.keys(payload).forEach(k => {
81
+ if (k !== 'namespace' && k !== this.contentField && k !== this.metadataField) {
82
+ keys.add(k);
83
+ }
84
+ });
85
+
86
+ // Extract nested metadata keys if applicable
87
+ if (!this.isFlatPayload && payload[this.metadataField]) {
88
+ const meta = payload[this.metadataField];
89
+ if (typeof meta === 'object' && meta !== null) {
90
+ Object.keys(meta).forEach(k => keys.add(k));
91
+ }
92
+ }
93
+ }
94
+
95
+ return Array.from(keys);
96
+ } catch (err) {
97
+ console.warn(`[QdrantProvider] ⚠️ Schema discovery failed:`, err instanceof Error ? err.message : String(err));
98
+ return [];
99
+ }
32
100
  }
33
101
 
34
102
  /**
@@ -58,32 +126,25 @@ export class QdrantProvider extends BaseVectorProvider {
58
126
  }
59
127
 
60
128
  /**
61
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
62
- * Qdrant requires this for search filters to work in many configurations.
129
+ * Ensures that a payload field has an index.
63
130
  */
64
- private async ensureIndex(): Promise<void> {
131
+ private async ensureIndex(fieldName: string, schema: string = 'keyword'): Promise<void> {
132
+ let fullPath = fieldName;
133
+ if (!this.isFlatPayload && !fieldName.includes('.') && fieldName !== 'namespace' && fieldName !== this.contentField) {
134
+ fullPath = `${this.metadataField}.${fieldName}`;
135
+ }
136
+
65
137
  try {
66
138
  await this.http.put(`/collections/${this.indexName}/index`, {
67
- field_name: 'namespace',
68
- field_schema: 'keyword',
139
+ field_name: fullPath,
140
+ field_schema: schema,
69
141
  });
70
- console.log(`[QdrantProvider] ✅ Ensured keyword index for "namespace" on collection "${this.indexName}"`);
142
+ console.log(`[QdrantProvider] ✅ Ensured ${schema} index for "${fullPath}"`);
71
143
  } catch (err) {
72
144
  let status: number | undefined;
73
- let data: unknown;
74
-
75
- if (axios.isAxiosError(err)) {
76
- status = err.response?.status;
77
- data = err.response?.data;
78
- }
79
-
80
- // 409 means it already exists, which is fine.
81
- if (status === 409) {
82
- return;
83
- }
84
-
85
- const errorMessage = err instanceof Error ? err.message : String(err);
86
- console.warn(`[QdrantProvider] ⚠️ Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
145
+ if (axios.isAxiosError(err)) status = err.response?.status;
146
+ if (status === 409) return;
147
+ console.warn(`[QdrantProvider] ⚠️ Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
87
148
  }
88
149
  }
89
150
 
@@ -93,21 +154,49 @@ export class QdrantProvider extends BaseVectorProvider {
93
154
 
94
155
  async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
95
156
  const payload = {
96
- points: docs.map(doc => ({
97
- id: this.normalizeId(doc.id),
98
- vector: doc.vector,
99
- payload: {
100
- content: doc.content,
101
- metadata: doc.metadata || {},
157
+ points: docs.map(doc => {
158
+ const pointPayload: Record<string, any> = {
159
+ [this.contentField]: doc.content,
102
160
  ...(namespace ? { namespace } : {}),
103
- },
104
- })),
161
+ };
162
+
163
+ if (this.isFlatPayload) {
164
+ Object.assign(pointPayload, doc.metadata || {});
165
+ } else {
166
+ pointPayload[this.metadataField] = doc.metadata || {};
167
+ }
168
+
169
+ return {
170
+ id: this.normalizeId(doc.id),
171
+ vector: doc.vector,
172
+ payload: pointPayload,
173
+ };
174
+ }),
105
175
  };
106
176
  await this.http.put(`/collections/${this.indexName}/points`, payload);
107
177
  }
108
178
 
109
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
110
179
  async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
180
+ const must: any[] = [];
181
+
182
+ if (namespace) {
183
+ must.push({ key: 'namespace', match: { value: namespace } });
184
+ }
185
+
186
+ const sanitizedFilter = this.sanitizeFilter(_filter);
187
+ for (const [key, value] of Object.entries(sanitizedFilter)) {
188
+ let filterKey = key;
189
+ if (!this.isFlatPayload && !key.includes('.') && key !== 'namespace' && key !== this.contentField) {
190
+ filterKey = `${this.metadataField}.${key}`;
191
+ }
192
+
193
+ if (Array.isArray(value)) {
194
+ must.push({ key: filterKey, match: { any: value } });
195
+ } else {
196
+ must.push({ key: filterKey, match: { value: value } });
197
+ }
198
+ }
199
+
111
200
  const payload = {
112
201
  vector: vector,
113
202
  limit: topK,
@@ -116,20 +205,48 @@ export class QdrantProvider extends BaseVectorProvider {
116
205
  hnsw_ef: (this.config.options.efSearch as number) || Math.max(topK * 20, 128),
117
206
  exact: false
118
207
  },
119
- filter: namespace ? {
120
- must: [{ key: 'namespace', match: { value: namespace } }]
121
- } : undefined,
208
+ filter: must.length > 0 ? { must } : undefined,
122
209
  };
210
+
123
211
  const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
124
- return (data.result || []).map((res: Record<string, unknown>) => ({
125
- id: res['id'] as string,
126
- score: res['score'] as number,
127
- content: (res['payload'] as Record<string, unknown>)?.content as string || '',
128
- metadata: (res['payload'] as Record<string, unknown>)?.metadata as Record<string, unknown> || {},
129
- }));
212
+
213
+ const results = (data.result || []).map((res: Record<string, any>) => {
214
+ const p = res.payload || {};
215
+ let content = p[this.contentField] || '';
216
+
217
+ if (!content) {
218
+ const stringFields = Object.entries(p)
219
+ .filter(([_, v]) => typeof v === 'string')
220
+ .map(([k, v]) => ({ key: k, val: v as string }));
221
+
222
+ if (stringFields.length > 0) {
223
+ const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
224
+ content = bestMatch.val;
225
+ } else {
226
+ content = JSON.stringify(p);
227
+ }
228
+ }
229
+
230
+ let metadata: Record<string, any> = {};
231
+ if (this.isFlatPayload) {
232
+ metadata = { ...p };
233
+ delete metadata[this.contentField];
234
+ delete metadata['namespace'];
235
+ } else {
236
+ metadata = p[this.metadataField] || p;
237
+ }
238
+
239
+ return {
240
+ id: res.id as string,
241
+ score: res.score as number,
242
+ content,
243
+ metadata,
244
+ };
245
+ });
246
+
247
+ return results;
130
248
  }
131
249
 
132
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
133
250
  async delete(id: string | number, _namespace?: string): Promise<void> {
134
251
  await this.http.post(`/collections/${this.indexName}/points/delete`, {
135
252
  points: [this.normalizeId(id)],
@@ -153,21 +270,16 @@ export class QdrantProvider extends BaseVectorProvider {
153
270
  }
154
271
  }
155
272
 
156
- /**
157
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
158
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
159
- */
160
273
  private normalizeId(id: string | number): string | number {
161
274
  if (typeof id === 'number') return id;
162
-
163
- // Check if already a valid UUID
164
275
  const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
165
276
  if (uuidRegex.test(id)) return id.toLowerCase();
166
-
167
- // Hash string to deterministic UUID
168
277
  const hash = crypto.createHash('md5').update(id).digest('hex');
169
278
  return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
170
279
  }
171
280
 
172
281
  async disconnect(): Promise<void> {}
173
282
  }
283
+
284
+
285
+
@@ -1,159 +0,0 @@
1
- import {
2
- BaseVectorProvider
3
- } from "./chunk-IMP6FUCY.mjs";
4
- import {
5
- __spreadValues
6
- } from "./chunk-X4TOT24V.mjs";
7
-
8
- // src/providers/vectordb/QdrantProvider.ts
9
- import axios from "axios";
10
- import crypto from "crypto";
11
- var QdrantProvider = class extends BaseVectorProvider {
12
- constructor(config) {
13
- super(config);
14
- const opts = config.options;
15
- const baseUrl = opts.baseUrl;
16
- if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
17
- this.http = axios.create({
18
- baseURL: baseUrl,
19
- headers: __spreadValues({
20
- "Content-Type": "application/json"
21
- }, opts.apiKey ? { "api-key": opts.apiKey } : {})
22
- });
23
- }
24
- async initialize() {
25
- await this.ping();
26
- await this.ensureCollection();
27
- await this.ensureIndex();
28
- }
29
- /**
30
- * Ensures the collection exists. Creates it if missing.
31
- */
32
- async ensureCollection() {
33
- var _a;
34
- try {
35
- await this.http.get(`/collections/${this.indexName}`);
36
- console.log(`[QdrantProvider] \u2705 Collection "${this.indexName}" already exists.`);
37
- } catch (err) {
38
- if (axios.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
39
- const opts = this.config.options;
40
- const dimensionsForCreate = opts.dimensions || 1536;
41
- console.log(`[QdrantProvider] \u23F3 Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
42
- await this.http.put(`/collections/${this.indexName}`, {
43
- vectors: {
44
- size: dimensionsForCreate,
45
- distance: "Cosine"
46
- }
47
- });
48
- console.log(`[QdrantProvider] \u2705 Created collection "${this.indexName}"`);
49
- } else {
50
- throw err;
51
- }
52
- }
53
- }
54
- /**
55
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
56
- * Qdrant requires this for search filters to work in many configurations.
57
- */
58
- async ensureIndex() {
59
- var _a, _b;
60
- try {
61
- await this.http.put(`/collections/${this.indexName}/index`, {
62
- field_name: "namespace",
63
- field_schema: "keyword"
64
- });
65
- console.log(`[QdrantProvider] \u2705 Ensured keyword index for "namespace" on collection "${this.indexName}"`);
66
- } catch (err) {
67
- let status;
68
- let data;
69
- if (axios.isAxiosError(err)) {
70
- status = (_a = err.response) == null ? void 0 : _a.status;
71
- data = (_b = err.response) == null ? void 0 : _b.data;
72
- }
73
- if (status === 409) {
74
- return;
75
- }
76
- const errorMessage = err instanceof Error ? err.message : String(err);
77
- console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
78
- }
79
- }
80
- async upsert(doc, namespace) {
81
- await this.batchUpsert([doc], namespace);
82
- }
83
- async batchUpsert(docs, namespace) {
84
- const payload = {
85
- points: docs.map((doc) => ({
86
- id: this.normalizeId(doc.id),
87
- vector: doc.vector,
88
- payload: __spreadValues({
89
- content: doc.content,
90
- metadata: doc.metadata || {}
91
- }, namespace ? { namespace } : {})
92
- }))
93
- };
94
- await this.http.put(`/collections/${this.indexName}/points`, payload);
95
- }
96
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
97
- async query(vector, topK, namespace, _filter) {
98
- const payload = {
99
- vector,
100
- limit: topK,
101
- with_payload: true,
102
- params: {
103
- hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
104
- exact: false
105
- },
106
- filter: namespace ? {
107
- must: [{ key: "namespace", match: { value: namespace } }]
108
- } : void 0
109
- };
110
- const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
111
- return (data.result || []).map((res) => {
112
- var _a, _b;
113
- return {
114
- id: res["id"],
115
- score: res["score"],
116
- content: ((_a = res["payload"]) == null ? void 0 : _a.content) || "",
117
- metadata: ((_b = res["payload"]) == null ? void 0 : _b.metadata) || {}
118
- };
119
- });
120
- }
121
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
122
- async delete(id, _namespace) {
123
- await this.http.post(`/collections/${this.indexName}/points/delete`, {
124
- points: [this.normalizeId(id)]
125
- });
126
- }
127
- async deleteNamespace(_namespace) {
128
- await this.http.post(`/collections/${this.indexName}/points/delete`, {
129
- filter: {
130
- must: [{ key: "namespace", match: { value: _namespace } }]
131
- }
132
- });
133
- }
134
- async ping() {
135
- try {
136
- await this.http.get("/healthz");
137
- return true;
138
- } catch (e) {
139
- return false;
140
- }
141
- }
142
- /**
143
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
144
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
145
- */
146
- normalizeId(id) {
147
- if (typeof id === "number") return id;
148
- const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
149
- if (uuidRegex.test(id)) return id.toLowerCase();
150
- const hash = crypto.createHash("md5").update(id).digest("hex");
151
- return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
152
- }
153
- async disconnect() {
154
- }
155
- };
156
-
157
- export {
158
- QdrantProvider
159
- };