@retrivora-ai/rag-engine 1.0.9 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  QdrantProvider
3
- } from "./chunk-NT5VP7MT.mjs";
3
+ } from "./chunk-65S4BQL2.mjs";
4
4
  import "./chunk-IMP6FUCY.mjs";
5
5
  import "./chunk-X4TOT24V.mjs";
6
6
  export {
@@ -0,0 +1,238 @@
1
+ import {
2
+ BaseVectorProvider
3
+ } from "./chunk-IMP6FUCY.mjs";
4
+ import {
5
+ __spreadValues
6
+ } from "./chunk-X4TOT24V.mjs";
7
+
8
+ // src/providers/vectordb/QdrantProvider.ts
9
+ import axios from "axios";
10
+ import crypto from "crypto";
11
+ var QdrantProvider = class extends BaseVectorProvider {
12
+ constructor(config) {
13
+ super(config);
14
+ const opts = config.options;
15
+ const baseUrl = opts.baseUrl;
16
+ if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
17
+ this.contentField = opts.contentField || "content";
18
+ this.metadataField = opts.metadataField || "metadata";
19
+ this.isFlatPayload = !!opts.flatPayload;
20
+ this.http = axios.create({
21
+ baseURL: baseUrl,
22
+ headers: __spreadValues({
23
+ "Content-Type": "application/json"
24
+ }, opts.apiKey ? { "api-key": opts.apiKey } : {})
25
+ });
26
+ }
27
+ async initialize() {
28
+ await this.ping();
29
+ await this.ensureCollection();
30
+ console.log(`[QdrantProvider] \u{1F50D} Discovering schema for collection "${this.indexName}"...`);
31
+ const discoveredFields = await this.discoverSchema();
32
+ const opts = this.config.options;
33
+ const configFields = opts.filterableFields || [];
34
+ const allFields = [.../* @__PURE__ */ new Set([...discoveredFields, ...configFields])];
35
+ if (allFields.length > 0) {
36
+ console.log(`[QdrantProvider] \u{1F6E0} Ensuring indexes for ${allFields.length} discovered fields...`);
37
+ for (const fieldInfo of allFields) {
38
+ const [fieldName, schemaType] = fieldInfo.split(":");
39
+ await this.ensureIndex(fieldName, schemaType || "keyword");
40
+ }
41
+ } else {
42
+ console.log(`[QdrantProvider] \u2139\uFE0F No fields discovered for indexing.`);
43
+ }
44
+ }
45
+ /**
46
+ * Samples points from the collection to discover available payload fields.
47
+ */
48
+ async discoverSchema() {
49
+ var _a;
50
+ try {
51
+ const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
52
+ limit: 20,
53
+ with_payload: true,
54
+ with_vector: false
55
+ });
56
+ const points = ((_a = data.result) == null ? void 0 : _a.points) || [];
57
+ const keys = /* @__PURE__ */ new Set();
58
+ for (const point of points) {
59
+ const payload = point.payload || {};
60
+ Object.keys(payload).forEach((k) => {
61
+ if (k !== "namespace" && k !== this.contentField && k !== this.metadataField) {
62
+ keys.add(k);
63
+ }
64
+ });
65
+ if (!this.isFlatPayload && payload[this.metadataField]) {
66
+ const meta = payload[this.metadataField];
67
+ if (typeof meta === "object" && meta !== null) {
68
+ Object.keys(meta).forEach((k) => keys.add(k));
69
+ }
70
+ }
71
+ }
72
+ return Array.from(keys);
73
+ } catch (err) {
74
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Schema discovery failed:`, err instanceof Error ? err.message : String(err));
75
+ return [];
76
+ }
77
+ }
78
+ /**
79
+ * Ensures the collection exists. Creates it if missing.
80
+ */
81
+ async ensureCollection() {
82
+ var _a;
83
+ try {
84
+ await this.http.get(`/collections/${this.indexName}`);
85
+ console.log(`[QdrantProvider] \u2705 Collection "${this.indexName}" already exists.`);
86
+ } catch (err) {
87
+ if (axios.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
88
+ const opts = this.config.options;
89
+ const dimensionsForCreate = opts.dimensions || 1536;
90
+ console.log(`[QdrantProvider] \u23F3 Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
91
+ await this.http.put(`/collections/${this.indexName}`, {
92
+ vectors: {
93
+ size: dimensionsForCreate,
94
+ distance: "Cosine"
95
+ }
96
+ });
97
+ console.log(`[QdrantProvider] \u2705 Created collection "${this.indexName}"`);
98
+ } else {
99
+ throw err;
100
+ }
101
+ }
102
+ }
103
+ /**
104
+ * Ensures that a payload field has an index.
105
+ */
106
+ async ensureIndex(fieldName, schema = "keyword") {
107
+ var _a;
108
+ let fullPath = fieldName;
109
+ if (!this.isFlatPayload && !fieldName.includes(".") && fieldName !== "namespace" && fieldName !== this.contentField) {
110
+ fullPath = `${this.metadataField}.${fieldName}`;
111
+ }
112
+ try {
113
+ await this.http.put(`/collections/${this.indexName}/index`, {
114
+ field_name: fullPath,
115
+ field_schema: schema
116
+ });
117
+ console.log(`[QdrantProvider] \u2705 Ensured ${schema} index for "${fullPath}"`);
118
+ } catch (err) {
119
+ let status;
120
+ if (axios.isAxiosError(err)) status = (_a = err.response) == null ? void 0 : _a.status;
121
+ if (status === 409) return;
122
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
123
+ }
124
+ }
125
+ async upsert(doc, namespace) {
126
+ await this.batchUpsert([doc], namespace);
127
+ }
128
+ async batchUpsert(docs, namespace) {
129
+ const payload = {
130
+ points: docs.map((doc) => {
131
+ const pointPayload = __spreadValues({
132
+ [this.contentField]: doc.content
133
+ }, namespace ? { namespace } : {});
134
+ if (this.isFlatPayload) {
135
+ Object.assign(pointPayload, doc.metadata || {});
136
+ } else {
137
+ pointPayload[this.metadataField] = doc.metadata || {};
138
+ }
139
+ return {
140
+ id: this.normalizeId(doc.id),
141
+ vector: doc.vector,
142
+ payload: pointPayload
143
+ };
144
+ })
145
+ };
146
+ await this.http.put(`/collections/${this.indexName}/points`, payload);
147
+ }
148
+ async query(vector, topK, namespace, _filter) {
149
+ const must = [];
150
+ if (namespace) {
151
+ must.push({ key: "namespace", match: { value: namespace } });
152
+ }
153
+ const sanitizedFilter = this.sanitizeFilter(_filter);
154
+ for (const [key, value] of Object.entries(sanitizedFilter)) {
155
+ let filterKey = key;
156
+ if (!this.isFlatPayload && !key.includes(".") && key !== "namespace" && key !== this.contentField) {
157
+ filterKey = `${this.metadataField}.${key}`;
158
+ }
159
+ if (Array.isArray(value)) {
160
+ must.push({ key: filterKey, match: { any: value } });
161
+ } else {
162
+ must.push({ key: filterKey, match: { value } });
163
+ }
164
+ }
165
+ const payload = {
166
+ vector,
167
+ limit: topK,
168
+ with_payload: true,
169
+ params: {
170
+ hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
171
+ exact: false
172
+ },
173
+ filter: must.length > 0 ? { must } : void 0
174
+ };
175
+ const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
176
+ const results = (data.result || []).map((res) => {
177
+ const p = res.payload || {};
178
+ let content = p[this.contentField] || "";
179
+ if (!content) {
180
+ const stringFields = Object.entries(p).filter(([_, v]) => typeof v === "string").map(([k, v]) => ({ key: k, val: v }));
181
+ if (stringFields.length > 0) {
182
+ const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
183
+ content = bestMatch.val;
184
+ } else {
185
+ content = JSON.stringify(p);
186
+ }
187
+ }
188
+ let metadata = {};
189
+ if (this.isFlatPayload) {
190
+ metadata = __spreadValues({}, p);
191
+ delete metadata[this.contentField];
192
+ delete metadata["namespace"];
193
+ } else {
194
+ metadata = p[this.metadataField] || p;
195
+ }
196
+ return {
197
+ id: res.id,
198
+ score: res.score,
199
+ content,
200
+ metadata
201
+ };
202
+ });
203
+ return results;
204
+ }
205
+ async delete(id, _namespace) {
206
+ await this.http.post(`/collections/${this.indexName}/points/delete`, {
207
+ points: [this.normalizeId(id)]
208
+ });
209
+ }
210
+ async deleteNamespace(_namespace) {
211
+ await this.http.post(`/collections/${this.indexName}/points/delete`, {
212
+ filter: {
213
+ must: [{ key: "namespace", match: { value: _namespace } }]
214
+ }
215
+ });
216
+ }
217
+ async ping() {
218
+ try {
219
+ await this.http.get("/healthz");
220
+ return true;
221
+ } catch (e) {
222
+ return false;
223
+ }
224
+ }
225
+ normalizeId(id) {
226
+ if (typeof id === "number") return id;
227
+ const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
228
+ if (uuidRegex.test(id)) return id.toLowerCase();
229
+ const hash = crypto.createHash("md5").update(id).digest("hex");
230
+ return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
231
+ }
232
+ async disconnect() {
233
+ }
234
+ };
235
+
236
+ export {
237
+ QdrantProvider
238
+ };
@@ -1106,7 +1106,7 @@ var ProviderRegistry = class {
1106
1106
  return MilvusProvider;
1107
1107
  }
1108
1108
  case "qdrant": {
1109
- const { QdrantProvider } = await import("./QdrantProvider-NGFXVDCF.mjs");
1109
+ const { QdrantProvider } = await import("./QdrantProvider-WWXFX2XF.mjs");
1110
1110
  return QdrantProvider;
1111
1111
  }
1112
1112
  case "chromadb": {
@@ -2273,6 +2273,7 @@ var Pipeline = class {
2273
2273
  }
2274
2274
  const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
2275
2275
  const filter = QueryProcessor.buildQueryFilter(question, hints);
2276
+ console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
2276
2277
  const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
2277
2278
  namespace: ns,
2278
2279
  topK: topK * 2,
@@ -845,6 +845,9 @@ var init_QdrantProvider = __esm({
845
845
  const opts = config.options;
846
846
  const baseUrl = opts.baseUrl;
847
847
  if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
848
+ this.contentField = opts.contentField || "content";
849
+ this.metadataField = opts.metadataField || "metadata";
850
+ this.isFlatPayload = !!opts.flatPayload;
848
851
  this.http = import_axios4.default.create({
849
852
  baseURL: baseUrl,
850
853
  headers: __spreadValues({
@@ -855,7 +858,53 @@ var init_QdrantProvider = __esm({
855
858
  async initialize() {
856
859
  await this.ping();
857
860
  await this.ensureCollection();
858
- await this.ensureIndex();
861
+ console.log(`[QdrantProvider] \u{1F50D} Discovering schema for collection "${this.indexName}"...`);
862
+ const discoveredFields = await this.discoverSchema();
863
+ const opts = this.config.options;
864
+ const configFields = opts.filterableFields || [];
865
+ const allFields = [.../* @__PURE__ */ new Set([...discoveredFields, ...configFields])];
866
+ if (allFields.length > 0) {
867
+ console.log(`[QdrantProvider] \u{1F6E0} Ensuring indexes for ${allFields.length} discovered fields...`);
868
+ for (const fieldInfo of allFields) {
869
+ const [fieldName, schemaType] = fieldInfo.split(":");
870
+ await this.ensureIndex(fieldName, schemaType || "keyword");
871
+ }
872
+ } else {
873
+ console.log(`[QdrantProvider] \u2139\uFE0F No fields discovered for indexing.`);
874
+ }
875
+ }
876
+ /**
877
+ * Samples points from the collection to discover available payload fields.
878
+ */
879
+ async discoverSchema() {
880
+ var _a;
881
+ try {
882
+ const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
883
+ limit: 20,
884
+ with_payload: true,
885
+ with_vector: false
886
+ });
887
+ const points = ((_a = data.result) == null ? void 0 : _a.points) || [];
888
+ const keys = /* @__PURE__ */ new Set();
889
+ for (const point of points) {
890
+ const payload = point.payload || {};
891
+ Object.keys(payload).forEach((k) => {
892
+ if (k !== "namespace" && k !== this.contentField && k !== this.metadataField) {
893
+ keys.add(k);
894
+ }
895
+ });
896
+ if (!this.isFlatPayload && payload[this.metadataField]) {
897
+ const meta = payload[this.metadataField];
898
+ if (typeof meta === "object" && meta !== null) {
899
+ Object.keys(meta).forEach((k) => keys.add(k));
900
+ }
901
+ }
902
+ }
903
+ return Array.from(keys);
904
+ } catch (err) {
905
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Schema discovery failed:`, err instanceof Error ? err.message : String(err));
906
+ return [];
907
+ }
859
908
  }
860
909
  /**
861
910
  * Ensures the collection exists. Creates it if missing.
@@ -883,29 +932,25 @@ var init_QdrantProvider = __esm({
883
932
  }
884
933
  }
885
934
  /**
886
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
887
- * Qdrant requires this for search filters to work in many configurations.
935
+ * Ensures that a payload field has an index.
888
936
  */
889
- async ensureIndex() {
890
- var _a, _b;
937
+ async ensureIndex(fieldName, schema = "keyword") {
938
+ var _a;
939
+ let fullPath = fieldName;
940
+ if (!this.isFlatPayload && !fieldName.includes(".") && fieldName !== "namespace" && fieldName !== this.contentField) {
941
+ fullPath = `${this.metadataField}.${fieldName}`;
942
+ }
891
943
  try {
892
944
  await this.http.put(`/collections/${this.indexName}/index`, {
893
- field_name: "namespace",
894
- field_schema: "keyword"
945
+ field_name: fullPath,
946
+ field_schema: schema
895
947
  });
896
- console.log(`[QdrantProvider] \u2705 Ensured keyword index for "namespace" on collection "${this.indexName}"`);
948
+ console.log(`[QdrantProvider] \u2705 Ensured ${schema} index for "${fullPath}"`);
897
949
  } catch (err) {
898
950
  let status;
899
- let data;
900
- if (import_axios4.default.isAxiosError(err)) {
901
- status = (_a = err.response) == null ? void 0 : _a.status;
902
- data = (_b = err.response) == null ? void 0 : _b.data;
903
- }
904
- if (status === 409) {
905
- return;
906
- }
907
- const errorMessage = err instanceof Error ? err.message : String(err);
908
- console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
951
+ if (import_axios4.default.isAxiosError(err)) status = (_a = err.response) == null ? void 0 : _a.status;
952
+ if (status === 409) return;
953
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
909
954
  }
910
955
  }
911
956
  async upsert(doc, namespace) {
@@ -913,14 +958,21 @@ var init_QdrantProvider = __esm({
913
958
  }
914
959
  async batchUpsert(docs, namespace) {
915
960
  const payload = {
916
- points: docs.map((doc) => ({
917
- id: this.normalizeId(doc.id),
918
- vector: doc.vector,
919
- payload: __spreadValues({
920
- content: doc.content,
921
- metadata: doc.metadata || {}
922
- }, namespace ? { namespace } : {})
923
- }))
961
+ points: docs.map((doc) => {
962
+ const pointPayload = __spreadValues({
963
+ [this.contentField]: doc.content
964
+ }, namespace ? { namespace } : {});
965
+ if (this.isFlatPayload) {
966
+ Object.assign(pointPayload, doc.metadata || {});
967
+ } else {
968
+ pointPayload[this.metadataField] = doc.metadata || {};
969
+ }
970
+ return {
971
+ id: this.normalizeId(doc.id),
972
+ vector: doc.vector,
973
+ payload: pointPayload
974
+ };
975
+ })
924
976
  };
925
977
  await this.http.put(`/collections/${this.indexName}/points`, payload);
926
978
  }
@@ -931,7 +983,15 @@ var init_QdrantProvider = __esm({
931
983
  }
932
984
  const sanitizedFilter = this.sanitizeFilter(_filter);
933
985
  for (const [key, value] of Object.entries(sanitizedFilter)) {
934
- must.push({ key: `metadata.${key}`, match: { value } });
986
+ let filterKey = key;
987
+ if (!this.isFlatPayload && !key.includes(".") && key !== "namespace" && key !== this.contentField) {
988
+ filterKey = `${this.metadataField}.${key}`;
989
+ }
990
+ if (Array.isArray(value)) {
991
+ must.push({ key: filterKey, match: { any: value } });
992
+ } else {
993
+ must.push({ key: filterKey, match: { value } });
994
+ }
935
995
  }
936
996
  const payload = {
937
997
  vector,
@@ -943,28 +1003,36 @@ var init_QdrantProvider = __esm({
943
1003
  },
944
1004
  filter: must.length > 0 ? { must } : void 0
945
1005
  };
946
- console.log(`[QdrantProvider] \u{1F50D} Searching "${this.indexName}" | Namespace: ${namespace || "default"} | TopK: ${topK}`);
947
- if (must.length > (namespace ? 1 : 0)) {
948
- console.log(`[QdrantProvider] \u{1F6E0} Filters:`, JSON.stringify(must.filter((m) => m.key !== "namespace")));
949
- }
950
1006
  const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
951
1007
  const results = (data.result || []).map((res) => {
952
- var _a, _b;
1008
+ const p = res.payload || {};
1009
+ let content = p[this.contentField] || "";
1010
+ if (!content) {
1011
+ const stringFields = Object.entries(p).filter(([_, v]) => typeof v === "string").map(([k, v]) => ({ key: k, val: v }));
1012
+ if (stringFields.length > 0) {
1013
+ const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
1014
+ content = bestMatch.val;
1015
+ } else {
1016
+ content = JSON.stringify(p);
1017
+ }
1018
+ }
1019
+ let metadata = {};
1020
+ if (this.isFlatPayload) {
1021
+ metadata = __spreadValues({}, p);
1022
+ delete metadata[this.contentField];
1023
+ delete metadata["namespace"];
1024
+ } else {
1025
+ metadata = p[this.metadataField] || p;
1026
+ }
953
1027
  return {
954
- id: res["id"],
955
- score: res["score"],
956
- content: ((_a = res["payload"]) == null ? void 0 : _a.content) || "",
957
- metadata: ((_b = res["payload"]) == null ? void 0 : _b.metadata) || {}
1028
+ id: res.id,
1029
+ score: res.score,
1030
+ content,
1031
+ metadata
958
1032
  };
959
1033
  });
960
- if (results.length === 0) {
961
- console.warn(`[QdrantProvider] \u26A0\uFE0F Retrieval returned 0 results. Check if data exists in namespace "${namespace}"`);
962
- } else {
963
- console.log(`[QdrantProvider] \u2705 Found ${results.length} results (best score: ${results[0].score.toFixed(4)})`);
964
- }
965
1034
  return results;
966
1035
  }
967
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
968
1036
  async delete(id, _namespace) {
969
1037
  await this.http.post(`/collections/${this.indexName}/points/delete`, {
970
1038
  points: [this.normalizeId(id)]
@@ -985,10 +1053,6 @@ var init_QdrantProvider = __esm({
985
1053
  return false;
986
1054
  }
987
1055
  }
988
- /**
989
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
990
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
991
- */
992
1056
  normalizeId(id) {
993
1057
  if (typeof id === "number") return id;
994
1058
  const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
@@ -3764,6 +3828,7 @@ var Pipeline = class {
3764
3828
  }
3765
3829
  const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
3766
3830
  const filter = QueryProcessor.buildQueryFilter(question, hints);
3831
+ console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
3767
3832
  const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
3768
3833
  namespace: ns,
3769
3834
  topK: topK * 2,
@@ -8,7 +8,7 @@ import {
8
8
  sseFrame,
9
9
  sseMetaFrame,
10
10
  sseTextFrame
11
- } from "../chunk-7S2SRQGL.mjs";
11
+ } from "../chunk-G2LVK46T.mjs";
12
12
  import "../chunk-YLTMFW4M.mjs";
13
13
  import "../chunk-X4TOT24V.mjs";
14
14
  export {
package/dist/server.d.mts CHANGED
@@ -731,18 +731,25 @@ declare class MilvusProvider extends BaseVectorProvider {
731
731
 
732
732
  /**
733
733
  * QdrantProvider — implementation for Qdrant using its REST API.
734
+ * Fully dynamic: automatically discovers and indexes schema from available data.
734
735
  */
735
736
  declare class QdrantProvider extends BaseVectorProvider {
736
737
  private http;
738
+ private contentField;
739
+ private metadataField;
740
+ private isFlatPayload;
737
741
  constructor(config: VectorDBConfig);
738
742
  initialize(): Promise<void>;
743
+ /**
744
+ * Samples points from the collection to discover available payload fields.
745
+ */
746
+ private discoverSchema;
739
747
  /**
740
748
  * Ensures the collection exists. Creates it if missing.
741
749
  */
742
750
  private ensureCollection;
743
751
  /**
744
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
745
- * Qdrant requires this for search filters to work in many configurations.
752
+ * Ensures that a payload field has an index.
746
753
  */
747
754
  private ensureIndex;
748
755
  upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
@@ -751,10 +758,6 @@ declare class QdrantProvider extends BaseVectorProvider {
751
758
  delete(id: string | number, _namespace?: string): Promise<void>;
752
759
  deleteNamespace(_namespace: string): Promise<void>;
753
760
  ping(): Promise<boolean>;
754
- /**
755
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
756
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
757
- */
758
761
  private normalizeId;
759
762
  disconnect(): Promise<void>;
760
763
  }
package/dist/server.d.ts CHANGED
@@ -731,18 +731,25 @@ declare class MilvusProvider extends BaseVectorProvider {
731
731
 
732
732
  /**
733
733
  * QdrantProvider — implementation for Qdrant using its REST API.
734
+ * Fully dynamic: automatically discovers and indexes schema from available data.
734
735
  */
735
736
  declare class QdrantProvider extends BaseVectorProvider {
736
737
  private http;
738
+ private contentField;
739
+ private metadataField;
740
+ private isFlatPayload;
737
741
  constructor(config: VectorDBConfig);
738
742
  initialize(): Promise<void>;
743
+ /**
744
+ * Samples points from the collection to discover available payload fields.
745
+ */
746
+ private discoverSchema;
739
747
  /**
740
748
  * Ensures the collection exists. Creates it if missing.
741
749
  */
742
750
  private ensureCollection;
743
751
  /**
744
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
745
- * Qdrant requires this for search filters to work in many configurations.
752
+ * Ensures that a payload field has an index.
746
753
  */
747
754
  private ensureIndex;
748
755
  upsert(doc: UpsertDocument, namespace?: string): Promise<void>;
@@ -751,10 +758,6 @@ declare class QdrantProvider extends BaseVectorProvider {
751
758
  delete(id: string | number, _namespace?: string): Promise<void>;
752
759
  deleteNamespace(_namespace: string): Promise<void>;
753
760
  ping(): Promise<boolean>;
754
- /**
755
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
756
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
757
- */
758
761
  private normalizeId;
759
762
  disconnect(): Promise<void>;
760
763
  }
package/dist/server.js CHANGED
@@ -857,6 +857,9 @@ var init_QdrantProvider = __esm({
857
857
  const opts = config.options;
858
858
  const baseUrl = opts.baseUrl;
859
859
  if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
860
+ this.contentField = opts.contentField || "content";
861
+ this.metadataField = opts.metadataField || "metadata";
862
+ this.isFlatPayload = !!opts.flatPayload;
860
863
  this.http = import_axios4.default.create({
861
864
  baseURL: baseUrl,
862
865
  headers: __spreadValues({
@@ -867,7 +870,53 @@ var init_QdrantProvider = __esm({
867
870
  async initialize() {
868
871
  await this.ping();
869
872
  await this.ensureCollection();
870
- await this.ensureIndex();
873
+ console.log(`[QdrantProvider] \u{1F50D} Discovering schema for collection "${this.indexName}"...`);
874
+ const discoveredFields = await this.discoverSchema();
875
+ const opts = this.config.options;
876
+ const configFields = opts.filterableFields || [];
877
+ const allFields = [.../* @__PURE__ */ new Set([...discoveredFields, ...configFields])];
878
+ if (allFields.length > 0) {
879
+ console.log(`[QdrantProvider] \u{1F6E0} Ensuring indexes for ${allFields.length} discovered fields...`);
880
+ for (const fieldInfo of allFields) {
881
+ const [fieldName, schemaType] = fieldInfo.split(":");
882
+ await this.ensureIndex(fieldName, schemaType || "keyword");
883
+ }
884
+ } else {
885
+ console.log(`[QdrantProvider] \u2139\uFE0F No fields discovered for indexing.`);
886
+ }
887
+ }
888
+ /**
889
+ * Samples points from the collection to discover available payload fields.
890
+ */
891
+ async discoverSchema() {
892
+ var _a;
893
+ try {
894
+ const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
895
+ limit: 20,
896
+ with_payload: true,
897
+ with_vector: false
898
+ });
899
+ const points = ((_a = data.result) == null ? void 0 : _a.points) || [];
900
+ const keys = /* @__PURE__ */ new Set();
901
+ for (const point of points) {
902
+ const payload = point.payload || {};
903
+ Object.keys(payload).forEach((k) => {
904
+ if (k !== "namespace" && k !== this.contentField && k !== this.metadataField) {
905
+ keys.add(k);
906
+ }
907
+ });
908
+ if (!this.isFlatPayload && payload[this.metadataField]) {
909
+ const meta = payload[this.metadataField];
910
+ if (typeof meta === "object" && meta !== null) {
911
+ Object.keys(meta).forEach((k) => keys.add(k));
912
+ }
913
+ }
914
+ }
915
+ return Array.from(keys);
916
+ } catch (err) {
917
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Schema discovery failed:`, err instanceof Error ? err.message : String(err));
918
+ return [];
919
+ }
871
920
  }
872
921
  /**
873
922
  * Ensures the collection exists. Creates it if missing.
@@ -895,29 +944,25 @@ var init_QdrantProvider = __esm({
895
944
  }
896
945
  }
897
946
  /**
898
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
899
- * Qdrant requires this for search filters to work in many configurations.
947
+ * Ensures that a payload field has an index.
900
948
  */
901
- async ensureIndex() {
902
- var _a, _b;
949
+ async ensureIndex(fieldName, schema = "keyword") {
950
+ var _a;
951
+ let fullPath = fieldName;
952
+ if (!this.isFlatPayload && !fieldName.includes(".") && fieldName !== "namespace" && fieldName !== this.contentField) {
953
+ fullPath = `${this.metadataField}.${fieldName}`;
954
+ }
903
955
  try {
904
956
  await this.http.put(`/collections/${this.indexName}/index`, {
905
- field_name: "namespace",
906
- field_schema: "keyword"
957
+ field_name: fullPath,
958
+ field_schema: schema
907
959
  });
908
- console.log(`[QdrantProvider] \u2705 Ensured keyword index for "namespace" on collection "${this.indexName}"`);
960
+ console.log(`[QdrantProvider] \u2705 Ensured ${schema} index for "${fullPath}"`);
909
961
  } catch (err) {
910
962
  let status;
911
- let data;
912
- if (import_axios4.default.isAxiosError(err)) {
913
- status = (_a = err.response) == null ? void 0 : _a.status;
914
- data = (_b = err.response) == null ? void 0 : _b.data;
915
- }
916
- if (status === 409) {
917
- return;
918
- }
919
- const errorMessage = err instanceof Error ? err.message : String(err);
920
- console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
963
+ if (import_axios4.default.isAxiosError(err)) status = (_a = err.response) == null ? void 0 : _a.status;
964
+ if (status === 409) return;
965
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
921
966
  }
922
967
  }
923
968
  async upsert(doc, namespace) {
@@ -925,14 +970,21 @@ var init_QdrantProvider = __esm({
925
970
  }
926
971
  async batchUpsert(docs, namespace) {
927
972
  const payload = {
928
- points: docs.map((doc) => ({
929
- id: this.normalizeId(doc.id),
930
- vector: doc.vector,
931
- payload: __spreadValues({
932
- content: doc.content,
933
- metadata: doc.metadata || {}
934
- }, namespace ? { namespace } : {})
935
- }))
973
+ points: docs.map((doc) => {
974
+ const pointPayload = __spreadValues({
975
+ [this.contentField]: doc.content
976
+ }, namespace ? { namespace } : {});
977
+ if (this.isFlatPayload) {
978
+ Object.assign(pointPayload, doc.metadata || {});
979
+ } else {
980
+ pointPayload[this.metadataField] = doc.metadata || {};
981
+ }
982
+ return {
983
+ id: this.normalizeId(doc.id),
984
+ vector: doc.vector,
985
+ payload: pointPayload
986
+ };
987
+ })
936
988
  };
937
989
  await this.http.put(`/collections/${this.indexName}/points`, payload);
938
990
  }
@@ -943,7 +995,15 @@ var init_QdrantProvider = __esm({
943
995
  }
944
996
  const sanitizedFilter = this.sanitizeFilter(_filter);
945
997
  for (const [key, value] of Object.entries(sanitizedFilter)) {
946
- must.push({ key: `metadata.${key}`, match: { value } });
998
+ let filterKey = key;
999
+ if (!this.isFlatPayload && !key.includes(".") && key !== "namespace" && key !== this.contentField) {
1000
+ filterKey = `${this.metadataField}.${key}`;
1001
+ }
1002
+ if (Array.isArray(value)) {
1003
+ must.push({ key: filterKey, match: { any: value } });
1004
+ } else {
1005
+ must.push({ key: filterKey, match: { value } });
1006
+ }
947
1007
  }
948
1008
  const payload = {
949
1009
  vector,
@@ -955,28 +1015,36 @@ var init_QdrantProvider = __esm({
955
1015
  },
956
1016
  filter: must.length > 0 ? { must } : void 0
957
1017
  };
958
- console.log(`[QdrantProvider] \u{1F50D} Searching "${this.indexName}" | Namespace: ${namespace || "default"} | TopK: ${topK}`);
959
- if (must.length > (namespace ? 1 : 0)) {
960
- console.log(`[QdrantProvider] \u{1F6E0} Filters:`, JSON.stringify(must.filter((m) => m.key !== "namespace")));
961
- }
962
1018
  const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
963
1019
  const results = (data.result || []).map((res) => {
964
- var _a, _b;
1020
+ const p = res.payload || {};
1021
+ let content = p[this.contentField] || "";
1022
+ if (!content) {
1023
+ const stringFields = Object.entries(p).filter(([_, v]) => typeof v === "string").map(([k, v]) => ({ key: k, val: v }));
1024
+ if (stringFields.length > 0) {
1025
+ const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
1026
+ content = bestMatch.val;
1027
+ } else {
1028
+ content = JSON.stringify(p);
1029
+ }
1030
+ }
1031
+ let metadata = {};
1032
+ if (this.isFlatPayload) {
1033
+ metadata = __spreadValues({}, p);
1034
+ delete metadata[this.contentField];
1035
+ delete metadata["namespace"];
1036
+ } else {
1037
+ metadata = p[this.metadataField] || p;
1038
+ }
965
1039
  return {
966
- id: res["id"],
967
- score: res["score"],
968
- content: ((_a = res["payload"]) == null ? void 0 : _a.content) || "",
969
- metadata: ((_b = res["payload"]) == null ? void 0 : _b.metadata) || {}
1040
+ id: res.id,
1041
+ score: res.score,
1042
+ content,
1043
+ metadata
970
1044
  };
971
1045
  });
972
- if (results.length === 0) {
973
- console.warn(`[QdrantProvider] \u26A0\uFE0F Retrieval returned 0 results. Check if data exists in namespace "${namespace}"`);
974
- } else {
975
- console.log(`[QdrantProvider] \u2705 Found ${results.length} results (best score: ${results[0].score.toFixed(4)})`);
976
- }
977
1046
  return results;
978
1047
  }
979
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
980
1048
  async delete(id, _namespace) {
981
1049
  await this.http.post(`/collections/${this.indexName}/points/delete`, {
982
1050
  points: [this.normalizeId(id)]
@@ -997,10 +1065,6 @@ var init_QdrantProvider = __esm({
997
1065
  return false;
998
1066
  }
999
1067
  }
1000
- /**
1001
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
1002
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
1003
- */
1004
1068
  normalizeId(id) {
1005
1069
  if (typeof id === "number") return id;
1006
1070
  const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
@@ -3857,6 +3921,7 @@ var Pipeline = class {
3857
3921
  }
3858
3922
  const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
3859
3923
  const filter = QueryProcessor.buildQueryFilter(question, hints);
3924
+ console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
3860
3925
  const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
3861
3926
  namespace: ns,
3862
3927
  topK: topK * 2,
package/dist/server.mjs CHANGED
@@ -40,7 +40,7 @@ import {
40
40
  sseFrame,
41
41
  sseMetaFrame,
42
42
  sseTextFrame
43
- } from "./chunk-7S2SRQGL.mjs";
43
+ } from "./chunk-G2LVK46T.mjs";
44
44
  import "./chunk-YLTMFW4M.mjs";
45
45
  import {
46
46
  PineconeProvider
@@ -56,7 +56,7 @@ import {
56
56
  } from "./chunk-U55XRW3U.mjs";
57
57
  import {
58
58
  QdrantProvider
59
- } from "./chunk-NT5VP7MT.mjs";
59
+ } from "./chunk-65S4BQL2.mjs";
60
60
  import {
61
61
  BaseVectorProvider
62
62
  } from "./chunk-IMP6FUCY.mjs";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@retrivora-ai/rag-engine",
3
- "version": "1.0.9",
3
+ "version": "1.1.0",
4
4
  "description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
5
5
  "author": "Abhinav Alkuchi",
6
6
  "license": "MIT",
@@ -317,6 +317,8 @@ export class Pipeline {
317
317
  // 2. Parallel Retrieval
318
318
  const hints = QueryProcessor.extractQueryFieldHints(question, this.config.rag?.filterableFields);
319
319
  const filter = QueryProcessor.buildQueryFilter(question, hints);
320
+
321
+ console.log(`[Pipeline] 🧩 Extracted filters:`, JSON.stringify(filter));
320
322
 
321
323
  const { sources: rawSources, graphData } = await this.retrieve(searchQuery, {
322
324
  namespace: ns,
@@ -6,9 +6,13 @@ import { VectorMatch, UpsertDocument } from '../../types';
6
6
 
7
7
  /**
8
8
  * QdrantProvider — implementation for Qdrant using its REST API.
9
+ * Fully dynamic: automatically discovers and indexes schema from available data.
9
10
  */
10
11
  export class QdrantProvider extends BaseVectorProvider {
11
12
  private http: AxiosInstance;
13
+ private contentField: string;
14
+ private metadataField: string;
15
+ private isFlatPayload: boolean;
12
16
 
13
17
  constructor(config: VectorDBConfig) {
14
18
  super(config);
@@ -16,6 +20,10 @@ export class QdrantProvider extends BaseVectorProvider {
16
20
  const baseUrl = opts.baseUrl as string;
17
21
  if (!baseUrl) throw new Error('[QdrantProvider] baseUrl is required');
18
22
 
23
+ this.contentField = (opts.contentField as string) || 'content';
24
+ this.metadataField = (opts.metadataField as string) || 'metadata';
25
+ this.isFlatPayload = !!opts.flatPayload;
26
+
19
27
  this.http = axios.create({
20
28
  baseURL: baseUrl,
21
29
  headers: {
@@ -28,7 +36,67 @@ export class QdrantProvider extends BaseVectorProvider {
28
36
  async initialize(): Promise<void> {
29
37
  await this.ping();
30
38
  await this.ensureCollection();
31
- await this.ensureIndex();
39
+
40
+ // 1. Discover schema from existing data
41
+ console.log(`[QdrantProvider] 🔍 Discovering schema for collection "${this.indexName}"...`);
42
+ const discoveredFields = await this.discoverSchema();
43
+
44
+ // 2. Add client-provided fields (if any) as fallback/override
45
+ const opts = this.config.options as Record<string, unknown>;
46
+ const configFields = (opts.filterableFields as string[]) || [];
47
+
48
+ const allFields = [...new Set([...discoveredFields, ...configFields])];
49
+
50
+ if (allFields.length > 0) {
51
+ console.log(`[QdrantProvider] 🛠 Ensuring indexes for ${allFields.length} discovered fields...`);
52
+ for (const fieldInfo of allFields) {
53
+ const [fieldName, schemaType] = fieldInfo.split(':');
54
+ await this.ensureIndex(fieldName, (schemaType as any) || 'keyword');
55
+ }
56
+ } else {
57
+ console.log(`[QdrantProvider] ℹ️ No fields discovered for indexing.`);
58
+ }
59
+ }
60
+
61
+ /**
62
+ * Samples points from the collection to discover available payload fields.
63
+ */
64
+ private async discoverSchema(): Promise<string[]> {
65
+ try {
66
+ // Fetch 20 points to get a representative sample of the schema
67
+ const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
68
+ limit: 20,
69
+ with_payload: true,
70
+ with_vector: false
71
+ });
72
+
73
+ const points = data.result?.points || [];
74
+ const keys = new Set<string>();
75
+
76
+ for (const point of points) {
77
+ const payload = point.payload || {};
78
+
79
+ // Extract root keys
80
+ Object.keys(payload).forEach(k => {
81
+ if (k !== 'namespace' && k !== this.contentField && k !== this.metadataField) {
82
+ keys.add(k);
83
+ }
84
+ });
85
+
86
+ // Extract nested metadata keys if applicable
87
+ if (!this.isFlatPayload && payload[this.metadataField]) {
88
+ const meta = payload[this.metadataField];
89
+ if (typeof meta === 'object' && meta !== null) {
90
+ Object.keys(meta).forEach(k => keys.add(k));
91
+ }
92
+ }
93
+ }
94
+
95
+ return Array.from(keys);
96
+ } catch (err) {
97
+ console.warn(`[QdrantProvider] ⚠️ Schema discovery failed:`, err instanceof Error ? err.message : String(err));
98
+ return [];
99
+ }
32
100
  }
33
101
 
34
102
  /**
@@ -58,32 +126,25 @@ export class QdrantProvider extends BaseVectorProvider {
58
126
  }
59
127
 
60
128
  /**
61
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
62
- * Qdrant requires this for search filters to work in many configurations.
129
+ * Ensures that a payload field has an index.
63
130
  */
64
- private async ensureIndex(): Promise<void> {
131
+ private async ensureIndex(fieldName: string, schema: string = 'keyword'): Promise<void> {
132
+ let fullPath = fieldName;
133
+ if (!this.isFlatPayload && !fieldName.includes('.') && fieldName !== 'namespace' && fieldName !== this.contentField) {
134
+ fullPath = `${this.metadataField}.${fieldName}`;
135
+ }
136
+
65
137
  try {
66
138
  await this.http.put(`/collections/${this.indexName}/index`, {
67
- field_name: 'namespace',
68
- field_schema: 'keyword',
139
+ field_name: fullPath,
140
+ field_schema: schema,
69
141
  });
70
- console.log(`[QdrantProvider] ✅ Ensured keyword index for "namespace" on collection "${this.indexName}"`);
142
+ console.log(`[QdrantProvider] ✅ Ensured ${schema} index for "${fullPath}"`);
71
143
  } catch (err) {
72
144
  let status: number | undefined;
73
- let data: unknown;
74
-
75
- if (axios.isAxiosError(err)) {
76
- status = err.response?.status;
77
- data = err.response?.data;
78
- }
79
-
80
- // 409 means it already exists, which is fine.
81
- if (status === 409) {
82
- return;
83
- }
84
-
85
- const errorMessage = err instanceof Error ? err.message : String(err);
86
- console.warn(`[QdrantProvider] ⚠️ Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
145
+ if (axios.isAxiosError(err)) status = err.response?.status;
146
+ if (status === 409) return;
147
+ console.warn(`[QdrantProvider] ⚠️ Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
87
148
  }
88
149
  }
89
150
 
@@ -93,15 +154,24 @@ export class QdrantProvider extends BaseVectorProvider {
93
154
 
94
155
  async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
95
156
  const payload = {
96
- points: docs.map(doc => ({
97
- id: this.normalizeId(doc.id),
98
- vector: doc.vector,
99
- payload: {
100
- content: doc.content,
101
- metadata: doc.metadata || {},
157
+ points: docs.map(doc => {
158
+ const pointPayload: Record<string, any> = {
159
+ [this.contentField]: doc.content,
102
160
  ...(namespace ? { namespace } : {}),
103
- },
104
- })),
161
+ };
162
+
163
+ if (this.isFlatPayload) {
164
+ Object.assign(pointPayload, doc.metadata || {});
165
+ } else {
166
+ pointPayload[this.metadataField] = doc.metadata || {};
167
+ }
168
+
169
+ return {
170
+ id: this.normalizeId(doc.id),
171
+ vector: doc.vector,
172
+ payload: pointPayload,
173
+ };
174
+ }),
105
175
  };
106
176
  await this.http.put(`/collections/${this.indexName}/points`, payload);
107
177
  }
@@ -113,12 +183,18 @@ export class QdrantProvider extends BaseVectorProvider {
113
183
  must.push({ key: 'namespace', match: { value: namespace } });
114
184
  }
115
185
 
116
- // 1. Apply metadata filters
117
186
  const sanitizedFilter = this.sanitizeFilter(_filter);
118
187
  for (const [key, value] of Object.entries(sanitizedFilter)) {
119
- // Qdrant uses dot notation for nested payload fields.
120
- // Our payload structure is { content, metadata: { ... }, namespace }
121
- must.push({ key: `metadata.${key}`, match: { value: value } });
188
+ let filterKey = key;
189
+ if (!this.isFlatPayload && !key.includes('.') && key !== 'namespace' && key !== this.contentField) {
190
+ filterKey = `${this.metadataField}.${key}`;
191
+ }
192
+
193
+ if (Array.isArray(value)) {
194
+ must.push({ key: filterKey, match: { any: value } });
195
+ } else {
196
+ must.push({ key: filterKey, match: { value: value } });
197
+ }
122
198
  }
123
199
 
124
200
  const payload = {
@@ -132,31 +208,45 @@ export class QdrantProvider extends BaseVectorProvider {
132
208
  filter: must.length > 0 ? { must } : undefined,
133
209
  };
134
210
 
135
- console.log(`[QdrantProvider] 🔍 Searching "${this.indexName}" | Namespace: ${namespace || 'default'} | TopK: ${topK}`);
136
- if (must.length > (namespace ? 1 : 0)) {
137
- console.log(`[QdrantProvider] 🛠 Filters:`, JSON.stringify(must.filter(m => m.key !== 'namespace')));
138
- }
139
-
140
211
  const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
141
212
 
142
- const results = (data.result || []).map((res: Record<string, unknown>) => ({
143
- id: res['id'] as string,
144
- score: res['score'] as number,
145
- content: (res['payload'] as Record<string, unknown>)?.content as string || '',
146
- metadata: (res['payload'] as Record<string, unknown>)?.metadata as Record<string, unknown> || {},
147
- }));
148
-
149
- if (results.length === 0) {
150
- console.warn(`[QdrantProvider] ⚠️ Retrieval returned 0 results. Check if data exists in namespace "${namespace}"`);
151
- } else {
152
- console.log(`[QdrantProvider] Found ${results.length} results (best score: ${results[0].score.toFixed(4)})`);
153
- }
213
+ const results = (data.result || []).map((res: Record<string, any>) => {
214
+ const p = res.payload || {};
215
+ let content = p[this.contentField] || '';
216
+
217
+ if (!content) {
218
+ const stringFields = Object.entries(p)
219
+ .filter(([_, v]) => typeof v === 'string')
220
+ .map(([k, v]) => ({ key: k, val: v as string }));
221
+
222
+ if (stringFields.length > 0) {
223
+ const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
224
+ content = bestMatch.val;
225
+ } else {
226
+ content = JSON.stringify(p);
227
+ }
228
+ }
229
+
230
+ let metadata: Record<string, any> = {};
231
+ if (this.isFlatPayload) {
232
+ metadata = { ...p };
233
+ delete metadata[this.contentField];
234
+ delete metadata['namespace'];
235
+ } else {
236
+ metadata = p[this.metadataField] || p;
237
+ }
238
+
239
+ return {
240
+ id: res.id as string,
241
+ score: res.score as number,
242
+ content,
243
+ metadata,
244
+ };
245
+ });
154
246
 
155
247
  return results;
156
248
  }
157
249
 
158
-
159
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
160
250
  async delete(id: string | number, _namespace?: string): Promise<void> {
161
251
  await this.http.post(`/collections/${this.indexName}/points/delete`, {
162
252
  points: [this.normalizeId(id)],
@@ -180,21 +270,16 @@ export class QdrantProvider extends BaseVectorProvider {
180
270
  }
181
271
  }
182
272
 
183
- /**
184
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
185
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
186
- */
187
273
  private normalizeId(id: string | number): string | number {
188
274
  if (typeof id === 'number') return id;
189
-
190
- // Check if already a valid UUID
191
275
  const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
192
276
  if (uuidRegex.test(id)) return id.toLowerCase();
193
-
194
- // Hash string to deterministic UUID
195
277
  const hash = crypto.createHash('md5').update(id).digest('hex');
196
278
  return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
197
279
  }
198
280
 
199
281
  async disconnect(): Promise<void> {}
200
282
  }
283
+
284
+
285
+
@@ -1,174 +0,0 @@
1
- import {
2
- BaseVectorProvider
3
- } from "./chunk-IMP6FUCY.mjs";
4
- import {
5
- __spreadValues
6
- } from "./chunk-X4TOT24V.mjs";
7
-
8
- // src/providers/vectordb/QdrantProvider.ts
9
- import axios from "axios";
10
- import crypto from "crypto";
11
- var QdrantProvider = class extends BaseVectorProvider {
12
- constructor(config) {
13
- super(config);
14
- const opts = config.options;
15
- const baseUrl = opts.baseUrl;
16
- if (!baseUrl) throw new Error("[QdrantProvider] baseUrl is required");
17
- this.http = axios.create({
18
- baseURL: baseUrl,
19
- headers: __spreadValues({
20
- "Content-Type": "application/json"
21
- }, opts.apiKey ? { "api-key": opts.apiKey } : {})
22
- });
23
- }
24
- async initialize() {
25
- await this.ping();
26
- await this.ensureCollection();
27
- await this.ensureIndex();
28
- }
29
- /**
30
- * Ensures the collection exists. Creates it if missing.
31
- */
32
- async ensureCollection() {
33
- var _a;
34
- try {
35
- await this.http.get(`/collections/${this.indexName}`);
36
- console.log(`[QdrantProvider] \u2705 Collection "${this.indexName}" already exists.`);
37
- } catch (err) {
38
- if (axios.isAxiosError(err) && ((_a = err.response) == null ? void 0 : _a.status) === 404) {
39
- const opts = this.config.options;
40
- const dimensionsForCreate = opts.dimensions || 1536;
41
- console.log(`[QdrantProvider] \u23F3 Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
42
- await this.http.put(`/collections/${this.indexName}`, {
43
- vectors: {
44
- size: dimensionsForCreate,
45
- distance: "Cosine"
46
- }
47
- });
48
- console.log(`[QdrantProvider] \u2705 Created collection "${this.indexName}"`);
49
- } else {
50
- throw err;
51
- }
52
- }
53
- }
54
- /**
55
- * Ensures that the 'namespace' field has a keyword index for efficient filtering.
56
- * Qdrant requires this for search filters to work in many configurations.
57
- */
58
- async ensureIndex() {
59
- var _a, _b;
60
- try {
61
- await this.http.put(`/collections/${this.indexName}/index`, {
62
- field_name: "namespace",
63
- field_schema: "keyword"
64
- });
65
- console.log(`[QdrantProvider] \u2705 Ensured keyword index for "namespace" on collection "${this.indexName}"`);
66
- } catch (err) {
67
- let status;
68
- let data;
69
- if (axios.isAxiosError(err)) {
70
- status = (_a = err.response) == null ? void 0 : _a.status;
71
- data = (_b = err.response) == null ? void 0 : _b.data;
72
- }
73
- if (status === 409) {
74
- return;
75
- }
76
- const errorMessage = err instanceof Error ? err.message : String(err);
77
- console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
78
- }
79
- }
80
- async upsert(doc, namespace) {
81
- await this.batchUpsert([doc], namespace);
82
- }
83
- async batchUpsert(docs, namespace) {
84
- const payload = {
85
- points: docs.map((doc) => ({
86
- id: this.normalizeId(doc.id),
87
- vector: doc.vector,
88
- payload: __spreadValues({
89
- content: doc.content,
90
- metadata: doc.metadata || {}
91
- }, namespace ? { namespace } : {})
92
- }))
93
- };
94
- await this.http.put(`/collections/${this.indexName}/points`, payload);
95
- }
96
- async query(vector, topK, namespace, _filter) {
97
- const must = [];
98
- if (namespace) {
99
- must.push({ key: "namespace", match: { value: namespace } });
100
- }
101
- const sanitizedFilter = this.sanitizeFilter(_filter);
102
- for (const [key, value] of Object.entries(sanitizedFilter)) {
103
- must.push({ key: `metadata.${key}`, match: { value } });
104
- }
105
- const payload = {
106
- vector,
107
- limit: topK,
108
- with_payload: true,
109
- params: {
110
- hnsw_ef: this.config.options.efSearch || Math.max(topK * 20, 128),
111
- exact: false
112
- },
113
- filter: must.length > 0 ? { must } : void 0
114
- };
115
- console.log(`[QdrantProvider] \u{1F50D} Searching "${this.indexName}" | Namespace: ${namespace || "default"} | TopK: ${topK}`);
116
- if (must.length > (namespace ? 1 : 0)) {
117
- console.log(`[QdrantProvider] \u{1F6E0} Filters:`, JSON.stringify(must.filter((m) => m.key !== "namespace")));
118
- }
119
- const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
120
- const results = (data.result || []).map((res) => {
121
- var _a, _b;
122
- return {
123
- id: res["id"],
124
- score: res["score"],
125
- content: ((_a = res["payload"]) == null ? void 0 : _a.content) || "",
126
- metadata: ((_b = res["payload"]) == null ? void 0 : _b.metadata) || {}
127
- };
128
- });
129
- if (results.length === 0) {
130
- console.warn(`[QdrantProvider] \u26A0\uFE0F Retrieval returned 0 results. Check if data exists in namespace "${namespace}"`);
131
- } else {
132
- console.log(`[QdrantProvider] \u2705 Found ${results.length} results (best score: ${results[0].score.toFixed(4)})`);
133
- }
134
- return results;
135
- }
136
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
137
- async delete(id, _namespace) {
138
- await this.http.post(`/collections/${this.indexName}/points/delete`, {
139
- points: [this.normalizeId(id)]
140
- });
141
- }
142
- async deleteNamespace(_namespace) {
143
- await this.http.post(`/collections/${this.indexName}/points/delete`, {
144
- filter: {
145
- must: [{ key: "namespace", match: { value: _namespace } }]
146
- }
147
- });
148
- }
149
- async ping() {
150
- try {
151
- await this.http.get("/healthz");
152
- return true;
153
- } catch (e) {
154
- return false;
155
- }
156
- }
157
- /**
158
- * Normalizes an ID into a format Qdrant accepts (UUID or integer).
159
- * For string IDs that aren't UUIDs, it generates a deterministic UUID v5-like hash.
160
- */
161
- normalizeId(id) {
162
- if (typeof id === "number") return id;
163
- const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
164
- if (uuidRegex.test(id)) return id.toLowerCase();
165
- const hash = crypto.createHash("md5").update(id).digest("hex");
166
- return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
167
- }
168
- async disconnect() {
169
- }
170
- };
171
-
172
- export {
173
- QdrantProvider
174
- };