@mastra/opensearch 1.1.0 → 1.1.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +6 -4
- package/dist/docs/SKILL.md +4 -7
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/{docs-rag-retrieval.md → reference-rag-retrieval.md} +168 -31
- package/dist/docs/references/{docs-rag-vector-databases.md → reference-rag-vector-databases.md} +104 -37
- package/dist/docs/references/reference-vectors-opensearch.md +4 -0
- package/dist/index.cjs +704 -907
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +701 -903
- package/dist/index.js.map +1 -1
- package/dist/vector/filter.d.ts.map +1 -1
- package/dist/vector/index.d.ts.map +1 -1
- package/package.json +18 -18
- package/CHANGELOG.md +0 -992
package/dist/index.js
CHANGED
|
@@ -1,911 +1,709 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { createVectorErrorId } from
|
|
3
|
-
import { MastraVector,
|
|
4
|
-
import { Client } from
|
|
5
|
-
import { BaseFilterTranslator } from
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
import { ErrorCategory, ErrorDomain, MastraError } from "@mastra/core/error";
|
|
2
|
+
import { createVectorErrorId } from "@mastra/core/storage";
|
|
3
|
+
import { MastraVector, validateTopK, validateUpsert } from "@mastra/core/vector";
|
|
4
|
+
import { Client } from "@opensearch-project/opensearch";
|
|
5
|
+
import { BaseFilterTranslator } from "@mastra/core/vector/filter";
|
|
6
|
+
//#region src/vector/filter.ts
|
|
7
|
+
/**
|
|
8
|
+
* Translator for OpenSearch filter queries.
|
|
9
|
+
* Maintains OpenSearch-compatible syntax while ensuring proper validation
|
|
10
|
+
* and normalization of values.
|
|
11
|
+
*/
|
|
8
12
|
var OpenSearchFilterTranslator = class extends BaseFilterTranslator {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
return this.translateRegexOperator(field, value);
|
|
216
|
-
}
|
|
217
|
-
const fieldWithKeyword = this.addKeywordIfNeeded(field, value);
|
|
218
|
-
return { term: { [fieldWithKeyword]: value } };
|
|
219
|
-
}
|
|
220
|
-
/**
|
|
221
|
-
* Translates regex patterns to OpenSearch query syntax
|
|
222
|
-
*/
|
|
223
|
-
translateRegexOperator(field, value) {
|
|
224
|
-
const regexValue = typeof value === "string" ? value : value.toString();
|
|
225
|
-
if (regexValue.includes("\n") || regexValue.includes("\r")) {
|
|
226
|
-
return { match: { [field]: value } };
|
|
227
|
-
}
|
|
228
|
-
let processedRegex = regexValue;
|
|
229
|
-
const hasStartAnchor = regexValue.startsWith("^");
|
|
230
|
-
const hasEndAnchor = regexValue.endsWith("$");
|
|
231
|
-
if (hasStartAnchor || hasEndAnchor) {
|
|
232
|
-
if (hasStartAnchor) {
|
|
233
|
-
processedRegex = processedRegex.substring(1);
|
|
234
|
-
}
|
|
235
|
-
if (hasEndAnchor) {
|
|
236
|
-
processedRegex = processedRegex.substring(0, processedRegex.length - 1);
|
|
237
|
-
}
|
|
238
|
-
let wildcardPattern = processedRegex;
|
|
239
|
-
if (!hasStartAnchor) {
|
|
240
|
-
wildcardPattern = "*" + wildcardPattern;
|
|
241
|
-
}
|
|
242
|
-
if (!hasEndAnchor) {
|
|
243
|
-
wildcardPattern = wildcardPattern + "*";
|
|
244
|
-
}
|
|
245
|
-
return { wildcard: { [field]: wildcardPattern } };
|
|
246
|
-
}
|
|
247
|
-
const escapedRegex = regexValue.replace(/\\/g, "\\\\");
|
|
248
|
-
return { regexp: { [field]: escapedRegex } };
|
|
249
|
-
}
|
|
250
|
-
addKeywordIfNeeded(field, value) {
|
|
251
|
-
if (typeof value === "string") {
|
|
252
|
-
return `${field}.keyword`;
|
|
253
|
-
}
|
|
254
|
-
if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
|
|
255
|
-
return `${field}.keyword`;
|
|
256
|
-
}
|
|
257
|
-
return field;
|
|
258
|
-
}
|
|
259
|
-
/**
|
|
260
|
-
* Helper method to handle special cases for the $not operator
|
|
261
|
-
*/
|
|
262
|
-
handleNotOperatorSpecialCases(value, field) {
|
|
263
|
-
if (value === null) {
|
|
264
|
-
return { exists: { field } };
|
|
265
|
-
}
|
|
266
|
-
if (typeof value === "object" && value !== null) {
|
|
267
|
-
if ("$eq" in value && value.$eq === null) {
|
|
268
|
-
return { exists: { field } };
|
|
269
|
-
}
|
|
270
|
-
if ("$ne" in value && value.$ne === null) {
|
|
271
|
-
return {
|
|
272
|
-
bool: {
|
|
273
|
-
must_not: [{ exists: { field } }]
|
|
274
|
-
}
|
|
275
|
-
};
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
return null;
|
|
279
|
-
}
|
|
280
|
-
translateOperator(operator, value, field) {
|
|
281
|
-
if (!this.isOperator(operator)) {
|
|
282
|
-
throw new Error(`Unsupported operator: ${operator}`);
|
|
283
|
-
}
|
|
284
|
-
if (operator === "$not" && field) {
|
|
285
|
-
const specialCaseResult = this.handleNotOperatorSpecialCases(value, field);
|
|
286
|
-
if (specialCaseResult) {
|
|
287
|
-
return specialCaseResult;
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
if (this.isLogicalOperator(operator)) {
|
|
291
|
-
if (operator === "$not" && field && typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
292
|
-
const entries = Object.entries(value);
|
|
293
|
-
if (entries.length > 0) {
|
|
294
|
-
if (entries.every(([op]) => this.isOperator(op))) {
|
|
295
|
-
const translatedCondition = this.translateFieldConditions(field, value);
|
|
296
|
-
return {
|
|
297
|
-
bool: {
|
|
298
|
-
must_not: [translatedCondition]
|
|
299
|
-
}
|
|
300
|
-
};
|
|
301
|
-
}
|
|
302
|
-
if (entries.length === 1 && entries[0] && this.isOperator(entries[0][0])) {
|
|
303
|
-
const [nestedOp, nestedVal] = entries[0];
|
|
304
|
-
const translatedNested = this.translateFieldOperator(field, nestedOp, nestedVal);
|
|
305
|
-
return {
|
|
306
|
-
bool: {
|
|
307
|
-
must_not: [translatedNested]
|
|
308
|
-
}
|
|
309
|
-
};
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
}
|
|
313
|
-
return this.translateLogicalOperator(operator, value);
|
|
314
|
-
}
|
|
315
|
-
if (field) {
|
|
316
|
-
return this.translateFieldOperator(field, operator, value);
|
|
317
|
-
}
|
|
318
|
-
return value;
|
|
319
|
-
}
|
|
320
|
-
/**
|
|
321
|
-
* Translates field conditions to OpenSearch query syntax
|
|
322
|
-
* Handles special cases like range queries and multiple operators
|
|
323
|
-
*/
|
|
324
|
-
translateFieldConditions(field, conditions) {
|
|
325
|
-
if (this.canOptimizeToRangeQuery(conditions)) {
|
|
326
|
-
return this.createRangeQuery(field, conditions);
|
|
327
|
-
}
|
|
328
|
-
const queryConditions = [];
|
|
329
|
-
Object.entries(conditions).forEach(([operator, value]) => {
|
|
330
|
-
if (this.isOperator(operator)) {
|
|
331
|
-
queryConditions.push(this.translateOperator(operator, value, field));
|
|
332
|
-
} else {
|
|
333
|
-
const fieldWithKeyword = this.addKeywordIfNeeded(`${field}.${operator}`, value);
|
|
334
|
-
queryConditions.push({ term: { [fieldWithKeyword]: value } });
|
|
335
|
-
}
|
|
336
|
-
});
|
|
337
|
-
if (queryConditions.length === 1) {
|
|
338
|
-
return queryConditions[0];
|
|
339
|
-
}
|
|
340
|
-
return {
|
|
341
|
-
bool: {
|
|
342
|
-
must: queryConditions
|
|
343
|
-
}
|
|
344
|
-
};
|
|
345
|
-
}
|
|
346
|
-
/**
|
|
347
|
-
* Checks if conditions can be optimized to a range query
|
|
348
|
-
*/
|
|
349
|
-
canOptimizeToRangeQuery(conditions) {
|
|
350
|
-
return Object.keys(conditions).every((op) => this.isNumericOperator(op)) && Object.keys(conditions).length > 0;
|
|
351
|
-
}
|
|
352
|
-
/**
|
|
353
|
-
* Creates a range query from numeric operators
|
|
354
|
-
*/
|
|
355
|
-
createRangeQuery(field, conditions) {
|
|
356
|
-
const rangeParams = Object.fromEntries(
|
|
357
|
-
Object.entries(conditions).map(([op, val]) => [op.replace("$", ""), this.normalizeComparisonValue(val)])
|
|
358
|
-
);
|
|
359
|
-
return { range: { [field]: rangeParams } };
|
|
360
|
-
}
|
|
13
|
+
getSupportedOperators() {
|
|
14
|
+
return {
|
|
15
|
+
...BaseFilterTranslator.DEFAULT_OPERATORS,
|
|
16
|
+
logical: [
|
|
17
|
+
"$and",
|
|
18
|
+
"$or",
|
|
19
|
+
"$not"
|
|
20
|
+
],
|
|
21
|
+
array: [
|
|
22
|
+
"$in",
|
|
23
|
+
"$nin",
|
|
24
|
+
"$all"
|
|
25
|
+
],
|
|
26
|
+
regex: ["$regex"],
|
|
27
|
+
custom: []
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
translate(filter) {
|
|
31
|
+
if (this.isEmpty(filter)) return void 0;
|
|
32
|
+
this.validateFilter(filter);
|
|
33
|
+
return this.translateNode(filter);
|
|
34
|
+
}
|
|
35
|
+
translateNode(node) {
|
|
36
|
+
if (this.isPrimitive(node) || Array.isArray(node)) return node;
|
|
37
|
+
const entries = Object.entries(node);
|
|
38
|
+
const logicalOperators = [];
|
|
39
|
+
const fieldConditions = [];
|
|
40
|
+
entries.forEach(([key, value]) => {
|
|
41
|
+
if (this.isLogicalOperator(key)) logicalOperators.push([key, value]);
|
|
42
|
+
else fieldConditions.push([key, value]);
|
|
43
|
+
});
|
|
44
|
+
if (logicalOperators.length === 1 && fieldConditions.length === 0) {
|
|
45
|
+
const [operator, value] = logicalOperators[0];
|
|
46
|
+
if (!Array.isArray(value) && typeof value !== "object") throw new Error(`Invalid logical operator structure: ${operator} must have an array or object value`);
|
|
47
|
+
return this.translateLogicalOperator(operator, value);
|
|
48
|
+
}
|
|
49
|
+
const fieldConditionQueries = fieldConditions.map(([key, value]) => {
|
|
50
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
51
|
+
const hasOperators = Object.keys(value).some((k) => this.isOperator(k));
|
|
52
|
+
const nestedField = `metadata.${key}`;
|
|
53
|
+
return hasOperators ? this.translateFieldConditions(nestedField, value) : this.translateNestedObject(nestedField, value);
|
|
54
|
+
}
|
|
55
|
+
if (Array.isArray(value)) return { terms: { [this.addKeywordIfNeeded(`metadata.${key}`, value)]: value } };
|
|
56
|
+
return { term: { [this.addKeywordIfNeeded(`metadata.${key}`, value)]: value } };
|
|
57
|
+
});
|
|
58
|
+
if (logicalOperators.length > 0) return { bool: { must: [...logicalOperators.map(([operator, value]) => this.translateOperator(operator, value)), ...fieldConditionQueries] } };
|
|
59
|
+
if (fieldConditionQueries.length > 1) return { bool: { must: fieldConditionQueries } };
|
|
60
|
+
if (fieldConditionQueries.length === 1) return fieldConditionQueries[0];
|
|
61
|
+
return { match_all: {} };
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Handles translation of nested objects with dot notation fields
|
|
65
|
+
*/
|
|
66
|
+
translateNestedObject(field, value) {
|
|
67
|
+
return { bool: { must: Object.entries(value).map(([subField, subValue]) => {
|
|
68
|
+
const fullField = `${field}.${subField}`;
|
|
69
|
+
if (this.isOperator(subField)) return this.translateOperator(subField, subValue, field);
|
|
70
|
+
if (typeof subValue === "object" && subValue !== null && !Array.isArray(subValue)) {
|
|
71
|
+
if (Object.keys(subValue).some((k) => this.isOperator(k))) return this.translateFieldConditions(fullField, subValue);
|
|
72
|
+
return this.translateNestedObject(fullField, subValue);
|
|
73
|
+
}
|
|
74
|
+
return { term: { [this.addKeywordIfNeeded(fullField, subValue)]: subValue } };
|
|
75
|
+
}) } };
|
|
76
|
+
}
|
|
77
|
+
translateLogicalOperator(operator, value) {
|
|
78
|
+
const conditions = Array.isArray(value) ? value.map((item) => this.translateNode(item)) : [this.translateNode(value)];
|
|
79
|
+
switch (operator) {
|
|
80
|
+
case "$and":
|
|
81
|
+
if (Array.isArray(value) && value.length === 0) return { match_all: {} };
|
|
82
|
+
return { bool: { must: conditions } };
|
|
83
|
+
case "$or":
|
|
84
|
+
if (Array.isArray(value) && value.length === 0) return { bool: { must_not: [{ match_all: {} }] } };
|
|
85
|
+
return { bool: { should: conditions } };
|
|
86
|
+
case "$not": return { bool: { must_not: conditions } };
|
|
87
|
+
default: return value;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
translateFieldOperator(field, operator, value) {
|
|
91
|
+
if (this.isBasicOperator(operator)) {
|
|
92
|
+
const normalizedValue = this.normalizeComparisonValue(value);
|
|
93
|
+
const fieldWithKeyword = this.addKeywordIfNeeded(field, value);
|
|
94
|
+
switch (operator) {
|
|
95
|
+
case "$eq":
|
|
96
|
+
if (value === null) return { bool: { must_not: [{ exists: { field } }] } };
|
|
97
|
+
return { term: { [fieldWithKeyword]: normalizedValue } };
|
|
98
|
+
case "$ne":
|
|
99
|
+
if (value === null) return { exists: { field } };
|
|
100
|
+
return { bool: { must_not: [{ term: { [fieldWithKeyword]: normalizedValue } }] } };
|
|
101
|
+
default: return { term: { [fieldWithKeyword]: normalizedValue } };
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (this.isNumericOperator(operator)) {
|
|
105
|
+
const normalizedValue = this.normalizeComparisonValue(value);
|
|
106
|
+
const rangeOp = operator.replace("$", "");
|
|
107
|
+
return { range: { [field]: { [rangeOp]: normalizedValue } } };
|
|
108
|
+
}
|
|
109
|
+
if (this.isArrayOperator(operator)) {
|
|
110
|
+
if (!Array.isArray(value)) throw new Error(`Invalid array operator value: ${operator} requires an array value`);
|
|
111
|
+
const normalizedValues = this.normalizeArrayValues(value);
|
|
112
|
+
const fieldWithKeyword = this.addKeywordIfNeeded(field, value);
|
|
113
|
+
switch (operator) {
|
|
114
|
+
case "$in": return { terms: { [fieldWithKeyword]: normalizedValues } };
|
|
115
|
+
case "$nin":
|
|
116
|
+
if (normalizedValues.length === 0) return { match_all: {} };
|
|
117
|
+
return { bool: { must_not: [{ terms: { [fieldWithKeyword]: normalizedValues } }] } };
|
|
118
|
+
case "$all":
|
|
119
|
+
if (normalizedValues.length === 0) return { bool: { must_not: [{ match_all: {} }] } };
|
|
120
|
+
return { bool: { must: normalizedValues.map((v) => ({ term: { [fieldWithKeyword]: v } })) } };
|
|
121
|
+
default: return { terms: { [fieldWithKeyword]: normalizedValues } };
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (this.isElementOperator(operator)) switch (operator) {
|
|
125
|
+
case "$exists": return value ? { exists: { field } } : { bool: { must_not: [{ exists: { field } }] } };
|
|
126
|
+
default: return { exists: { field } };
|
|
127
|
+
}
|
|
128
|
+
if (this.isRegexOperator(operator)) return this.translateRegexOperator(field, value);
|
|
129
|
+
return { term: { [this.addKeywordIfNeeded(field, value)]: value } };
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Translates regex patterns to OpenSearch query syntax
|
|
133
|
+
*/
|
|
134
|
+
translateRegexOperator(field, value) {
|
|
135
|
+
const regexValue = typeof value === "string" ? value : value.toString();
|
|
136
|
+
if (regexValue.includes("\n") || regexValue.includes("\r")) return { match: { [field]: value } };
|
|
137
|
+
let processedRegex = regexValue;
|
|
138
|
+
const hasStartAnchor = regexValue.startsWith("^");
|
|
139
|
+
const hasEndAnchor = regexValue.endsWith("$");
|
|
140
|
+
if (hasStartAnchor || hasEndAnchor) {
|
|
141
|
+
if (hasStartAnchor) processedRegex = processedRegex.substring(1);
|
|
142
|
+
if (hasEndAnchor) processedRegex = processedRegex.substring(0, processedRegex.length - 1);
|
|
143
|
+
let wildcardPattern = processedRegex;
|
|
144
|
+
if (!hasStartAnchor) wildcardPattern = "*" + wildcardPattern;
|
|
145
|
+
if (!hasEndAnchor) wildcardPattern = wildcardPattern + "*";
|
|
146
|
+
return { wildcard: { [field]: wildcardPattern } };
|
|
147
|
+
}
|
|
148
|
+
const escapedRegex = regexValue.replace(/\\/g, "\\\\");
|
|
149
|
+
return { regexp: { [field]: escapedRegex } };
|
|
150
|
+
}
|
|
151
|
+
addKeywordIfNeeded(field, value) {
|
|
152
|
+
if (typeof value === "string") return `${field}.keyword`;
|
|
153
|
+
if (Array.isArray(value) && value.every((item) => typeof item === "string")) return `${field}.keyword`;
|
|
154
|
+
return field;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Helper method to handle special cases for the $not operator
|
|
158
|
+
*/
|
|
159
|
+
handleNotOperatorSpecialCases(value, field) {
|
|
160
|
+
if (value === null) return { exists: { field } };
|
|
161
|
+
if (typeof value === "object" && value !== null) {
|
|
162
|
+
if ("$eq" in value && value.$eq === null) return { exists: { field } };
|
|
163
|
+
if ("$ne" in value && value.$ne === null) return { bool: { must_not: [{ exists: { field } }] } };
|
|
164
|
+
}
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
translateOperator(operator, value, field) {
|
|
168
|
+
if (!this.isOperator(operator)) throw new Error(`Unsupported operator: ${operator}`);
|
|
169
|
+
if (operator === "$not" && field) {
|
|
170
|
+
const specialCaseResult = this.handleNotOperatorSpecialCases(value, field);
|
|
171
|
+
if (specialCaseResult) return specialCaseResult;
|
|
172
|
+
}
|
|
173
|
+
if (this.isLogicalOperator(operator)) {
|
|
174
|
+
if (operator === "$not" && field && typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
175
|
+
const entries = Object.entries(value);
|
|
176
|
+
if (entries.length > 0) {
|
|
177
|
+
if (entries.every(([op]) => this.isOperator(op))) return { bool: { must_not: [this.translateFieldConditions(field, value)] } };
|
|
178
|
+
if (entries.length === 1 && entries[0] && this.isOperator(entries[0][0])) {
|
|
179
|
+
const [nestedOp, nestedVal] = entries[0];
|
|
180
|
+
return { bool: { must_not: [this.translateFieldOperator(field, nestedOp, nestedVal)] } };
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return this.translateLogicalOperator(operator, value);
|
|
185
|
+
}
|
|
186
|
+
if (field) return this.translateFieldOperator(field, operator, value);
|
|
187
|
+
return value;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Translates field conditions to OpenSearch query syntax
|
|
191
|
+
* Handles special cases like range queries and multiple operators
|
|
192
|
+
*/
|
|
193
|
+
translateFieldConditions(field, conditions) {
|
|
194
|
+
if (this.canOptimizeToRangeQuery(conditions)) return this.createRangeQuery(field, conditions);
|
|
195
|
+
const queryConditions = [];
|
|
196
|
+
Object.entries(conditions).forEach(([operator, value]) => {
|
|
197
|
+
if (this.isOperator(operator)) queryConditions.push(this.translateOperator(operator, value, field));
|
|
198
|
+
else {
|
|
199
|
+
const fieldWithKeyword = this.addKeywordIfNeeded(`${field}.${operator}`, value);
|
|
200
|
+
queryConditions.push({ term: { [fieldWithKeyword]: value } });
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
if (queryConditions.length === 1) return queryConditions[0];
|
|
204
|
+
return { bool: { must: queryConditions } };
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Checks if conditions can be optimized to a range query
|
|
208
|
+
*/
|
|
209
|
+
canOptimizeToRangeQuery(conditions) {
|
|
210
|
+
return Object.keys(conditions).every((op) => this.isNumericOperator(op)) && Object.keys(conditions).length > 0;
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Creates a range query from numeric operators
|
|
214
|
+
*/
|
|
215
|
+
createRangeQuery(field, conditions) {
|
|
216
|
+
const rangeParams = Object.fromEntries(Object.entries(conditions).map(([op, val]) => [op.replace("$", ""), this.normalizeComparisonValue(val)]));
|
|
217
|
+
return { range: { [field]: rangeParams } };
|
|
218
|
+
}
|
|
361
219
|
};
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
220
|
+
//#endregion
|
|
221
|
+
//#region src/vector/index.ts
|
|
222
|
+
const METRIC_MAPPING = {
|
|
223
|
+
cosine: "cosinesimil",
|
|
224
|
+
euclidean: "l2",
|
|
225
|
+
dotproduct: "innerproduct"
|
|
368
226
|
};
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
227
|
+
const REVERSE_METRIC_MAPPING = {
|
|
228
|
+
cosinesimil: "cosine",
|
|
229
|
+
l2: "euclidean",
|
|
230
|
+
innerproduct: "dotproduct"
|
|
373
231
|
};
|
|
374
232
|
var OpenSearchVector = class extends MastraVector {
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
id: createVectorErrorId("OPENSEARCH", "DELETE_VECTORS", "NO_TARGET"),
|
|
848
|
-
domain: ErrorDomain.STORAGE,
|
|
849
|
-
category: ErrorCategory.USER,
|
|
850
|
-
text: "Either filter or ids must be provided",
|
|
851
|
-
details: { indexName }
|
|
852
|
-
});
|
|
853
|
-
}
|
|
854
|
-
if (ids && ids.length === 0) {
|
|
855
|
-
throw new MastraError({
|
|
856
|
-
id: createVectorErrorId("OPENSEARCH", "DELETE_VECTORS", "EMPTY_IDS"),
|
|
857
|
-
domain: ErrorDomain.STORAGE,
|
|
858
|
-
category: ErrorCategory.USER,
|
|
859
|
-
text: "Cannot delete with empty ids array",
|
|
860
|
-
details: { indexName }
|
|
861
|
-
});
|
|
862
|
-
}
|
|
863
|
-
if (filter && Object.keys(filter).length === 0) {
|
|
864
|
-
throw new MastraError({
|
|
865
|
-
id: createVectorErrorId("OPENSEARCH", "DELETE_VECTORS", "EMPTY_FILTER"),
|
|
866
|
-
domain: ErrorDomain.STORAGE,
|
|
867
|
-
category: ErrorCategory.USER,
|
|
868
|
-
text: "Cannot delete with empty filter",
|
|
869
|
-
details: { indexName }
|
|
870
|
-
});
|
|
871
|
-
}
|
|
872
|
-
try {
|
|
873
|
-
if (ids) {
|
|
874
|
-
const bulkBody = ids.flatMap((id) => [{ delete: { _index: indexName, _id: id } }]);
|
|
875
|
-
await this.client.bulk({
|
|
876
|
-
body: bulkBody,
|
|
877
|
-
refresh: true
|
|
878
|
-
});
|
|
879
|
-
} else if (filter) {
|
|
880
|
-
const translator = new OpenSearchFilterTranslator();
|
|
881
|
-
const translatedFilter = translator.translate(filter);
|
|
882
|
-
await this.client.deleteByQuery({
|
|
883
|
-
index: indexName,
|
|
884
|
-
body: {
|
|
885
|
-
query: translatedFilter || { match_all: {} }
|
|
886
|
-
},
|
|
887
|
-
refresh: true
|
|
888
|
-
});
|
|
889
|
-
}
|
|
890
|
-
} catch (error) {
|
|
891
|
-
if (error instanceof MastraError) throw error;
|
|
892
|
-
throw new MastraError(
|
|
893
|
-
{
|
|
894
|
-
id: createVectorErrorId("OPENSEARCH", "DELETE_VECTORS", "FAILED"),
|
|
895
|
-
domain: ErrorDomain.STORAGE,
|
|
896
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
897
|
-
details: {
|
|
898
|
-
indexName,
|
|
899
|
-
...filter && { filter: JSON.stringify(filter) },
|
|
900
|
-
...ids && { idsCount: ids.length }
|
|
901
|
-
}
|
|
902
|
-
},
|
|
903
|
-
error
|
|
904
|
-
);
|
|
905
|
-
}
|
|
906
|
-
}
|
|
233
|
+
client;
|
|
234
|
+
/**
|
|
235
|
+
* Creates a new OpenSearchVector client.
|
|
236
|
+
*
|
|
237
|
+
* @param config - OpenSearch client configuration options plus a required id.
|
|
238
|
+
* @see OpenSearchVectorConfig for all available options.
|
|
239
|
+
*/
|
|
240
|
+
constructor({ id, ...clientOptions }) {
|
|
241
|
+
super({ id });
|
|
242
|
+
this.client = new Client(clientOptions);
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Creates a new collection with the specified configuration.
|
|
246
|
+
*
|
|
247
|
+
* @param {string} indexName - The name of the collection to create.
|
|
248
|
+
* @param {number} dimension - The dimension of the vectors to be stored in the collection.
|
|
249
|
+
* @param {'cosine' | 'euclidean' | 'dotproduct'} [metric=cosine] - The metric to use to sort vectors in the collection.
|
|
250
|
+
* @returns {Promise<void>} A promise that resolves when the collection is created.
|
|
251
|
+
*/
|
|
252
|
+
async createIndex({ indexName, dimension, metric = "cosine" }) {
|
|
253
|
+
if (!Number.isInteger(dimension) || dimension <= 0) throw new MastraError({
|
|
254
|
+
id: createVectorErrorId("OPENSEARCH", "CREATE_INDEX", "INVALID_ARGS"),
|
|
255
|
+
domain: ErrorDomain.STORAGE,
|
|
256
|
+
category: ErrorCategory.USER,
|
|
257
|
+
text: "Dimension must be a positive integer",
|
|
258
|
+
details: {
|
|
259
|
+
indexName,
|
|
260
|
+
dimension
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
try {
|
|
264
|
+
await this.client.indices.create({
|
|
265
|
+
index: indexName,
|
|
266
|
+
body: {
|
|
267
|
+
settings: { index: { knn: true } },
|
|
268
|
+
mappings: { properties: {
|
|
269
|
+
metadata: { type: "object" },
|
|
270
|
+
id: { type: "keyword" },
|
|
271
|
+
embedding: {
|
|
272
|
+
type: "knn_vector",
|
|
273
|
+
dimension,
|
|
274
|
+
method: {
|
|
275
|
+
name: "hnsw",
|
|
276
|
+
space_type: METRIC_MAPPING[metric],
|
|
277
|
+
engine: "faiss",
|
|
278
|
+
parameters: {
|
|
279
|
+
ef_construction: 128,
|
|
280
|
+
m: 16
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
} }
|
|
285
|
+
}
|
|
286
|
+
});
|
|
287
|
+
} catch (error) {
|
|
288
|
+
const message = error?.message || error?.toString();
|
|
289
|
+
if (message && message.toLowerCase().includes("already exists")) {
|
|
290
|
+
await this.validateExistingIndex(indexName, dimension, metric);
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
throw new MastraError({
|
|
294
|
+
id: createVectorErrorId("OPENSEARCH", "CREATE_INDEX", "FAILED"),
|
|
295
|
+
domain: ErrorDomain.STORAGE,
|
|
296
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
297
|
+
details: {
|
|
298
|
+
indexName,
|
|
299
|
+
dimension,
|
|
300
|
+
metric
|
|
301
|
+
}
|
|
302
|
+
}, error);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Lists all indexes.
|
|
307
|
+
*
|
|
308
|
+
* @returns {Promise<string[]>} A promise that resolves to an array of indexes.
|
|
309
|
+
*/
|
|
310
|
+
async listIndexes() {
|
|
311
|
+
try {
|
|
312
|
+
return (await this.client.cat.indices({ format: "json" })).body.map((record) => record.index).filter((index) => index !== void 0);
|
|
313
|
+
} catch (error) {
|
|
314
|
+
throw new MastraError({
|
|
315
|
+
id: createVectorErrorId("OPENSEARCH", "LIST_INDEXES", "FAILED"),
|
|
316
|
+
domain: ErrorDomain.STORAGE,
|
|
317
|
+
category: ErrorCategory.THIRD_PARTY
|
|
318
|
+
}, error);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Retrieves statistics about a vector index.
|
|
323
|
+
*
|
|
324
|
+
* @param {string} indexName - The name of the index to describe
|
|
325
|
+
* @returns A promise that resolves to the index statistics including dimension, count and metric
|
|
326
|
+
*/
|
|
327
|
+
async describeIndex({ indexName }) {
|
|
328
|
+
const { body: indexInfo } = await this.client.indices.get({ index: indexName });
|
|
329
|
+
const embedding = (indexInfo[indexName]?.mappings)?.properties?.embedding;
|
|
330
|
+
const spaceType = embedding.method.space_type;
|
|
331
|
+
const { body: countInfo } = await this.client.count({ index: indexName });
|
|
332
|
+
return {
|
|
333
|
+
dimension: Number(embedding.dimension),
|
|
334
|
+
count: Number(countInfo.count),
|
|
335
|
+
metric: REVERSE_METRIC_MAPPING[spaceType]
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Deletes the specified index.
|
|
340
|
+
*
|
|
341
|
+
* @param {string} indexName - The name of the index to delete.
|
|
342
|
+
* @returns {Promise<void>} A promise that resolves when the index is deleted.
|
|
343
|
+
*/
|
|
344
|
+
async deleteIndex({ indexName }) {
|
|
345
|
+
try {
|
|
346
|
+
await this.client.indices.delete({ index: indexName });
|
|
347
|
+
} catch (error) {
|
|
348
|
+
const mastraError = new MastraError({
|
|
349
|
+
id: createVectorErrorId("OPENSEARCH", "DELETE_INDEX", "FAILED"),
|
|
350
|
+
domain: ErrorDomain.STORAGE,
|
|
351
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
352
|
+
details: { indexName }
|
|
353
|
+
}, error);
|
|
354
|
+
this.logger?.error(mastraError.toString());
|
|
355
|
+
this.logger?.trackException(mastraError);
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* Inserts or updates vectors in the specified collection.
|
|
360
|
+
*
|
|
361
|
+
* @param {string} indexName - The name of the collection to upsert into.
|
|
362
|
+
* @param {number[][]} vectors - An array of vectors to upsert.
|
|
363
|
+
* @param {Record<string, any>[]} [metadata] - An optional array of metadata objects corresponding to each vector.
|
|
364
|
+
* @param {string[]} [ids] - An optional array of IDs corresponding to each vector. If not provided, new IDs will be generated.
|
|
365
|
+
* @returns {Promise<string[]>} A promise that resolves to an array of IDs of the upserted vectors.
|
|
366
|
+
*/
|
|
367
|
+
async upsert({ indexName, vectors, metadata = [], ids }) {
|
|
368
|
+
validateUpsert("OPENSEARCH", vectors, metadata, ids, true);
|
|
369
|
+
const vectorIds = ids || vectors.map(() => crypto.randomUUID());
|
|
370
|
+
const operations = [];
|
|
371
|
+
try {
|
|
372
|
+
const indexInfo = await this.describeIndex({ indexName });
|
|
373
|
+
this.validateVectorDimensions(vectors, indexInfo.dimension);
|
|
374
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
375
|
+
const operation = { index: {
|
|
376
|
+
_index: indexName,
|
|
377
|
+
_id: vectorIds[i]
|
|
378
|
+
} };
|
|
379
|
+
const document = {
|
|
380
|
+
id: vectorIds[i],
|
|
381
|
+
embedding: vectors[i],
|
|
382
|
+
metadata: metadata[i] || {}
|
|
383
|
+
};
|
|
384
|
+
operations.push(operation);
|
|
385
|
+
operations.push(document);
|
|
386
|
+
}
|
|
387
|
+
if (operations.length > 0) await this.client.bulk({
|
|
388
|
+
body: operations,
|
|
389
|
+
refresh: true
|
|
390
|
+
});
|
|
391
|
+
return vectorIds;
|
|
392
|
+
} catch (error) {
|
|
393
|
+
throw new MastraError({
|
|
394
|
+
id: createVectorErrorId("OPENSEARCH", "UPSERT", "FAILED"),
|
|
395
|
+
domain: ErrorDomain.STORAGE,
|
|
396
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
397
|
+
details: {
|
|
398
|
+
indexName,
|
|
399
|
+
vectorCount: vectors?.length || 0
|
|
400
|
+
}
|
|
401
|
+
}, error);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
/**
|
|
405
|
+
* Queries the specified collection using a vector and optional filter.
|
|
406
|
+
*
|
|
407
|
+
* @param {string} indexName - The name of the collection to query.
|
|
408
|
+
* @param {number[]} queryVector - The vector to query with.
|
|
409
|
+
* @param {number} [topK] - The maximum number of results to return.
|
|
410
|
+
* @param {Record<string, any>} [filter] - An optional filter to apply to the query. For more on filters in OpenSearch, see the filtering reference: https://opensearch.org/docs/latest/query-dsl/
|
|
411
|
+
* @param {boolean} [includeVectors=false] - Whether to include the vectors in the response.
|
|
412
|
+
* @returns {Promise<QueryResult[]>} A promise that resolves to an array of query results.
|
|
413
|
+
*/
|
|
414
|
+
async query({ indexName, queryVector, filter, topK = 10, includeVector = false }) {
|
|
415
|
+
if (!queryVector) throw new MastraError({
|
|
416
|
+
id: createVectorErrorId("OPENSEARCH", "QUERY", "MISSING_VECTOR"),
|
|
417
|
+
text: "queryVector is required for OpenSearch queries. Metadata-only queries are not supported by this vector store.",
|
|
418
|
+
domain: ErrorDomain.STORAGE,
|
|
419
|
+
category: ErrorCategory.USER,
|
|
420
|
+
details: { indexName }
|
|
421
|
+
});
|
|
422
|
+
validateTopK("OPENSEARCH", topK);
|
|
423
|
+
try {
|
|
424
|
+
const translatedFilter = this.transformFilter(filter);
|
|
425
|
+
return (await this.client.search({
|
|
426
|
+
index: indexName,
|
|
427
|
+
body: {
|
|
428
|
+
query: { bool: {
|
|
429
|
+
must: { knn: { embedding: {
|
|
430
|
+
vector: queryVector,
|
|
431
|
+
k: topK
|
|
432
|
+
} } },
|
|
433
|
+
filter: translatedFilter ? [translatedFilter] : []
|
|
434
|
+
} },
|
|
435
|
+
_source: [
|
|
436
|
+
"id",
|
|
437
|
+
"metadata",
|
|
438
|
+
"embedding"
|
|
439
|
+
]
|
|
440
|
+
}
|
|
441
|
+
})).body.hits.hits.map((hit) => {
|
|
442
|
+
const source = hit._source || {};
|
|
443
|
+
return {
|
|
444
|
+
id: String(source.id || ""),
|
|
445
|
+
score: typeof hit._score === "number" ? hit._score : 0,
|
|
446
|
+
metadata: source.metadata || {},
|
|
447
|
+
...includeVector && { vector: source.embedding }
|
|
448
|
+
};
|
|
449
|
+
});
|
|
450
|
+
} catch (error) {
|
|
451
|
+
throw new MastraError({
|
|
452
|
+
id: createVectorErrorId("OPENSEARCH", "QUERY", "FAILED"),
|
|
453
|
+
domain: ErrorDomain.STORAGE,
|
|
454
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
455
|
+
details: {
|
|
456
|
+
indexName,
|
|
457
|
+
topK
|
|
458
|
+
}
|
|
459
|
+
}, error);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* Validates the dimensions of the vectors.
|
|
464
|
+
*
|
|
465
|
+
* @param {number[][]} vectors - The vectors to validate.
|
|
466
|
+
* @param {number} dimension - The dimension of the vectors.
|
|
467
|
+
* @returns {void}
|
|
468
|
+
*/
|
|
469
|
+
validateVectorDimensions(vectors, dimension) {
|
|
470
|
+
if (vectors.some((vector) => vector.length !== dimension)) throw new Error("Vector dimension does not match index dimension");
|
|
471
|
+
}
|
|
472
|
+
/**
|
|
473
|
+
* Transforms the filter to the OpenSearch DSL.
|
|
474
|
+
*
|
|
475
|
+
* @param {OpenSearchVectorFilter} filter - The filter to transform.
|
|
476
|
+
* @returns {Record<string, any>} The transformed filter.
|
|
477
|
+
*/
|
|
478
|
+
transformFilter(filter) {
|
|
479
|
+
return new OpenSearchFilterTranslator().translate(filter);
|
|
480
|
+
}
|
|
481
|
+
/**
|
|
482
|
+
* Updates vectors by ID or filter with the provided vector and/or metadata.
|
|
483
|
+
* @param params - Parameters containing either id or filter for targeting vectors to update
|
|
484
|
+
* @param params.indexName - The name of the index containing the vector(s).
|
|
485
|
+
* @param params.id - The ID of a single vector to update (mutually exclusive with filter).
|
|
486
|
+
* @param params.filter - A filter to match multiple vectors to update (mutually exclusive with id).
|
|
487
|
+
* @param params.update - An object containing the vector and/or metadata to update.
|
|
488
|
+
* @returns A promise that resolves when the update is complete.
|
|
489
|
+
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
490
|
+
*/
|
|
491
|
+
async updateVector(params) {
|
|
492
|
+
const { indexName, update } = params;
|
|
493
|
+
if ("id" in params && "filter" in params && params.id && params.filter) throw new MastraError({
|
|
494
|
+
id: createVectorErrorId("OPENSEARCH", "UPDATE_VECTOR", "MUTUALLY_EXCLUSIVE"),
|
|
495
|
+
domain: ErrorDomain.STORAGE,
|
|
496
|
+
category: ErrorCategory.USER,
|
|
497
|
+
text: "id and filter are mutually exclusive",
|
|
498
|
+
details: { indexName }
|
|
499
|
+
});
|
|
500
|
+
if (!update.vector && !update.metadata) throw new MastraError({
|
|
501
|
+
id: createVectorErrorId("OPENSEARCH", "UPDATE_VECTOR", "NO_UPDATES"),
|
|
502
|
+
domain: ErrorDomain.STORAGE,
|
|
503
|
+
category: ErrorCategory.USER,
|
|
504
|
+
text: "No updates provided",
|
|
505
|
+
details: { indexName }
|
|
506
|
+
});
|
|
507
|
+
if ("filter" in params && params.filter && Object.keys(params.filter).length === 0) throw new MastraError({
|
|
508
|
+
id: createVectorErrorId("OPENSEARCH", "UPDATE_VECTOR", "EMPTY_FILTER"),
|
|
509
|
+
domain: ErrorDomain.STORAGE,
|
|
510
|
+
category: ErrorCategory.USER,
|
|
511
|
+
text: "Cannot update with empty filter",
|
|
512
|
+
details: { indexName }
|
|
513
|
+
});
|
|
514
|
+
if ("id" in params && params.id) await this.updateVectorById(indexName, params.id, update);
|
|
515
|
+
else if ("filter" in params && params.filter) await this.updateVectorsByFilter(indexName, params.filter, update);
|
|
516
|
+
else throw new MastraError({
|
|
517
|
+
id: createVectorErrorId("OPENSEARCH", "UPDATE_VECTOR", "NO_TARGET"),
|
|
518
|
+
domain: ErrorDomain.STORAGE,
|
|
519
|
+
category: ErrorCategory.USER,
|
|
520
|
+
text: "Either id or filter must be provided",
|
|
521
|
+
details: { indexName }
|
|
522
|
+
});
|
|
523
|
+
}
|
|
524
|
+
/**
|
|
525
|
+
* Updates a single vector by its ID.
|
|
526
|
+
*/
|
|
527
|
+
async updateVectorById(indexName, id, update) {
|
|
528
|
+
let existingDoc;
|
|
529
|
+
try {
|
|
530
|
+
const { body } = await this.client.get({
|
|
531
|
+
index: indexName,
|
|
532
|
+
id
|
|
533
|
+
}).catch(() => {
|
|
534
|
+
throw new Error(`Document with ID ${id} not found in index ${indexName}`);
|
|
535
|
+
});
|
|
536
|
+
if (!body || !body._source) throw new Error(`Document with ID ${id} has no source data in index ${indexName}`);
|
|
537
|
+
existingDoc = body;
|
|
538
|
+
} catch (error) {
|
|
539
|
+
throw new MastraError({
|
|
540
|
+
id: createVectorErrorId("OPENSEARCH", "UPDATE_VECTOR", "FAILED"),
|
|
541
|
+
domain: ErrorDomain.STORAGE,
|
|
542
|
+
category: ErrorCategory.USER,
|
|
543
|
+
details: {
|
|
544
|
+
indexName,
|
|
545
|
+
id
|
|
546
|
+
}
|
|
547
|
+
}, error);
|
|
548
|
+
}
|
|
549
|
+
const source = existingDoc._source;
|
|
550
|
+
const updatedDoc = { id: source?.id || id };
|
|
551
|
+
try {
|
|
552
|
+
if (update.vector) {
|
|
553
|
+
const indexInfo = await this.describeIndex({ indexName });
|
|
554
|
+
this.validateVectorDimensions([update.vector], indexInfo.dimension);
|
|
555
|
+
updatedDoc.embedding = update.vector;
|
|
556
|
+
} else if (source?.embedding) updatedDoc.embedding = source.embedding;
|
|
557
|
+
if (update.metadata) updatedDoc.metadata = update.metadata;
|
|
558
|
+
else updatedDoc.metadata = source?.metadata || {};
|
|
559
|
+
await this.client.index({
|
|
560
|
+
index: indexName,
|
|
561
|
+
id,
|
|
562
|
+
body: updatedDoc,
|
|
563
|
+
refresh: true
|
|
564
|
+
});
|
|
565
|
+
} catch (error) {
|
|
566
|
+
throw new MastraError({
|
|
567
|
+
id: createVectorErrorId("OPENSEARCH", "UPDATE_VECTOR", "FAILED"),
|
|
568
|
+
domain: ErrorDomain.STORAGE,
|
|
569
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
570
|
+
details: {
|
|
571
|
+
indexName,
|
|
572
|
+
id
|
|
573
|
+
}
|
|
574
|
+
}, error);
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
/**
|
|
578
|
+
* Updates multiple vectors matching a filter.
|
|
579
|
+
*/
|
|
580
|
+
async updateVectorsByFilter(indexName, filter, update) {
|
|
581
|
+
try {
|
|
582
|
+
const translatedFilter = new OpenSearchFilterTranslator().translate(filter);
|
|
583
|
+
const scriptSource = [];
|
|
584
|
+
const scriptParams = {};
|
|
585
|
+
if (update.vector) {
|
|
586
|
+
scriptSource.push("ctx._source.embedding = params.embedding");
|
|
587
|
+
scriptParams.embedding = update.vector;
|
|
588
|
+
}
|
|
589
|
+
if (update.metadata) {
|
|
590
|
+
scriptSource.push("ctx._source.metadata = params.metadata");
|
|
591
|
+
scriptParams.metadata = update.metadata;
|
|
592
|
+
}
|
|
593
|
+
await this.client.updateByQuery({
|
|
594
|
+
index: indexName,
|
|
595
|
+
body: {
|
|
596
|
+
query: translatedFilter || { match_all: {} },
|
|
597
|
+
script: {
|
|
598
|
+
source: scriptSource.join("; "),
|
|
599
|
+
params: scriptParams,
|
|
600
|
+
lang: "painless"
|
|
601
|
+
}
|
|
602
|
+
},
|
|
603
|
+
refresh: true
|
|
604
|
+
});
|
|
605
|
+
} catch (error) {
|
|
606
|
+
throw new MastraError({
|
|
607
|
+
id: createVectorErrorId("OPENSEARCH", "UPDATE_VECTOR_BY_FILTER", "FAILED"),
|
|
608
|
+
domain: ErrorDomain.STORAGE,
|
|
609
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
610
|
+
details: {
|
|
611
|
+
indexName,
|
|
612
|
+
filter: JSON.stringify(filter)
|
|
613
|
+
}
|
|
614
|
+
}, error);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
/**
|
|
618
|
+
* Deletes a vector by its ID.
|
|
619
|
+
* @param indexName - The name of the index containing the vector.
|
|
620
|
+
* @param id - The ID of the vector to delete.
|
|
621
|
+
* @returns A promise that resolves when the deletion is complete.
|
|
622
|
+
* @throws Will throw an error if the deletion operation fails.
|
|
623
|
+
*/
|
|
624
|
+
async deleteVector({ indexName, id }) {
|
|
625
|
+
try {
|
|
626
|
+
await this.client.delete({
|
|
627
|
+
index: indexName,
|
|
628
|
+
id,
|
|
629
|
+
refresh: true
|
|
630
|
+
});
|
|
631
|
+
} catch (error) {
|
|
632
|
+
if (error && typeof error === "object" && "statusCode" in error && error.statusCode === 404) return;
|
|
633
|
+
throw new MastraError({
|
|
634
|
+
id: createVectorErrorId("OPENSEARCH", "DELETE_VECTOR", "FAILED"),
|
|
635
|
+
domain: ErrorDomain.STORAGE,
|
|
636
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
637
|
+
details: {
|
|
638
|
+
indexName,
|
|
639
|
+
...id && { id }
|
|
640
|
+
}
|
|
641
|
+
}, error);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
async deleteVectors({ indexName, filter, ids }) {
|
|
645
|
+
if (ids && filter) throw new MastraError({
|
|
646
|
+
id: createVectorErrorId("OPENSEARCH", "DELETE_VECTORS", "MUTUALLY_EXCLUSIVE"),
|
|
647
|
+
domain: ErrorDomain.STORAGE,
|
|
648
|
+
category: ErrorCategory.USER,
|
|
649
|
+
text: "ids and filter are mutually exclusive",
|
|
650
|
+
details: { indexName }
|
|
651
|
+
});
|
|
652
|
+
if (!ids && !filter) throw new MastraError({
|
|
653
|
+
id: createVectorErrorId("OPENSEARCH", "DELETE_VECTORS", "NO_TARGET"),
|
|
654
|
+
domain: ErrorDomain.STORAGE,
|
|
655
|
+
category: ErrorCategory.USER,
|
|
656
|
+
text: "Either filter or ids must be provided",
|
|
657
|
+
details: { indexName }
|
|
658
|
+
});
|
|
659
|
+
if (ids && ids.length === 0) throw new MastraError({
|
|
660
|
+
id: createVectorErrorId("OPENSEARCH", "DELETE_VECTORS", "EMPTY_IDS"),
|
|
661
|
+
domain: ErrorDomain.STORAGE,
|
|
662
|
+
category: ErrorCategory.USER,
|
|
663
|
+
text: "Cannot delete with empty ids array",
|
|
664
|
+
details: { indexName }
|
|
665
|
+
});
|
|
666
|
+
if (filter && Object.keys(filter).length === 0) throw new MastraError({
|
|
667
|
+
id: createVectorErrorId("OPENSEARCH", "DELETE_VECTORS", "EMPTY_FILTER"),
|
|
668
|
+
domain: ErrorDomain.STORAGE,
|
|
669
|
+
category: ErrorCategory.USER,
|
|
670
|
+
text: "Cannot delete with empty filter",
|
|
671
|
+
details: { indexName }
|
|
672
|
+
});
|
|
673
|
+
try {
|
|
674
|
+
if (ids) {
|
|
675
|
+
const bulkBody = ids.flatMap((id) => [{ delete: {
|
|
676
|
+
_index: indexName,
|
|
677
|
+
_id: id
|
|
678
|
+
} }]);
|
|
679
|
+
await this.client.bulk({
|
|
680
|
+
body: bulkBody,
|
|
681
|
+
refresh: true
|
|
682
|
+
});
|
|
683
|
+
} else if (filter) {
|
|
684
|
+
const translatedFilter = new OpenSearchFilterTranslator().translate(filter);
|
|
685
|
+
await this.client.deleteByQuery({
|
|
686
|
+
index: indexName,
|
|
687
|
+
body: { query: translatedFilter || { match_all: {} } },
|
|
688
|
+
refresh: true
|
|
689
|
+
});
|
|
690
|
+
}
|
|
691
|
+
} catch (error) {
|
|
692
|
+
if (error instanceof MastraError) throw error;
|
|
693
|
+
throw new MastraError({
|
|
694
|
+
id: createVectorErrorId("OPENSEARCH", "DELETE_VECTORS", "FAILED"),
|
|
695
|
+
domain: ErrorDomain.STORAGE,
|
|
696
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
697
|
+
details: {
|
|
698
|
+
indexName,
|
|
699
|
+
...filter && { filter: JSON.stringify(filter) },
|
|
700
|
+
...ids && { idsCount: ids.length }
|
|
701
|
+
}
|
|
702
|
+
}, error);
|
|
703
|
+
}
|
|
704
|
+
}
|
|
907
705
|
};
|
|
908
|
-
|
|
706
|
+
//#endregion
|
|
909
707
|
export { OpenSearchVector };
|
|
910
|
-
|
|
708
|
+
|
|
911
709
|
//# sourceMappingURL=index.js.map
|