@mastra/opensearch 0.10.3 → 0.10.4-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +24 -0
- package/dist/_tsup-dts-rollup.d.cts +18 -5
- package/dist/_tsup-dts-rollup.d.ts +18 -5
- package/dist/index.cjs +128 -53
- package/dist/index.js +120 -45
- package/package.json +4 -4
- package/src/vector/filter.test.ts +55 -54
- package/src/vector/filter.ts +23 -5
- package/src/vector/index.test.ts +4 -5
- package/src/vector/index.ts +134 -55
package/src/vector/index.ts
CHANGED
|
@@ -9,10 +9,11 @@ import type {
|
|
|
9
9
|
UpdateVectorParams,
|
|
10
10
|
UpsertVectorParams,
|
|
11
11
|
} from '@mastra/core';
|
|
12
|
+
import { MastraError, ErrorDomain, ErrorCategory } from '@mastra/core/error';
|
|
12
13
|
import { MastraVector } from '@mastra/core/vector';
|
|
13
|
-
import type { VectorFilter } from '@mastra/core/vector/filter';
|
|
14
14
|
import { Client as OpenSearchClient } from '@opensearch-project/opensearch';
|
|
15
15
|
import { OpenSearchFilterTranslator } from './filter';
|
|
16
|
+
import type { OpenSearchVectorFilter } from './filter';
|
|
16
17
|
|
|
17
18
|
const METRIC_MAPPING = {
|
|
18
19
|
cosine: 'cosinesimil',
|
|
@@ -26,7 +27,9 @@ const REVERSE_METRIC_MAPPING = {
|
|
|
26
27
|
innerproduct: 'dotproduct',
|
|
27
28
|
} as const;
|
|
28
29
|
|
|
29
|
-
|
|
30
|
+
type OpenSearchVectorParams = QueryVectorParams<OpenSearchVectorFilter>;
|
|
31
|
+
|
|
32
|
+
export class OpenSearchVector extends MastraVector<OpenSearchVectorFilter> {
|
|
30
33
|
private client: OpenSearchClient;
|
|
31
34
|
|
|
32
35
|
/**
|
|
@@ -49,7 +52,13 @@ export class OpenSearchVector extends MastraVector {
|
|
|
49
52
|
*/
|
|
50
53
|
async createIndex({ indexName, dimension, metric = 'cosine' }: CreateIndexParams): Promise<void> {
|
|
51
54
|
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
52
|
-
throw new
|
|
55
|
+
throw new MastraError({
|
|
56
|
+
id: 'STORAGE_OPENSEARCH_VECTOR_CREATE_INDEX_INVALID_ARGS',
|
|
57
|
+
domain: ErrorDomain.STORAGE,
|
|
58
|
+
category: ErrorCategory.USER,
|
|
59
|
+
text: 'Dimension must be a positive integer',
|
|
60
|
+
details: { indexName, dimension },
|
|
61
|
+
});
|
|
53
62
|
}
|
|
54
63
|
|
|
55
64
|
try {
|
|
@@ -82,8 +91,15 @@ export class OpenSearchVector extends MastraVector {
|
|
|
82
91
|
await this.validateExistingIndex(indexName, dimension, metric);
|
|
83
92
|
return;
|
|
84
93
|
}
|
|
85
|
-
|
|
86
|
-
|
|
94
|
+
throw new MastraError(
|
|
95
|
+
{
|
|
96
|
+
id: 'STORAGE_OPENSEARCH_VECTOR_CREATE_INDEX_FAILED',
|
|
97
|
+
domain: ErrorDomain.STORAGE,
|
|
98
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
99
|
+
details: { indexName, dimension, metric },
|
|
100
|
+
},
|
|
101
|
+
error,
|
|
102
|
+
);
|
|
87
103
|
}
|
|
88
104
|
}
|
|
89
105
|
|
|
@@ -100,9 +116,15 @@ export class OpenSearchVector extends MastraVector {
|
|
|
100
116
|
.filter((index: string | undefined) => index !== undefined);
|
|
101
117
|
|
|
102
118
|
return indexes;
|
|
103
|
-
} catch (error
|
|
104
|
-
|
|
105
|
-
|
|
119
|
+
} catch (error) {
|
|
120
|
+
throw new MastraError(
|
|
121
|
+
{
|
|
122
|
+
id: 'STORAGE_OPENSEARCH_VECTOR_LIST_INDEXES_FAILED',
|
|
123
|
+
domain: ErrorDomain.STORAGE,
|
|
124
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
125
|
+
},
|
|
126
|
+
error,
|
|
127
|
+
);
|
|
106
128
|
}
|
|
107
129
|
}
|
|
108
130
|
|
|
@@ -137,7 +159,17 @@ export class OpenSearchVector extends MastraVector {
|
|
|
137
159
|
try {
|
|
138
160
|
await this.client.indices.delete({ index: indexName });
|
|
139
161
|
} catch (error) {
|
|
140
|
-
|
|
162
|
+
const mastraError = new MastraError(
|
|
163
|
+
{
|
|
164
|
+
id: 'STORAGE_OPENSEARCH_VECTOR_DELETE_INDEX_FAILED',
|
|
165
|
+
domain: ErrorDomain.STORAGE,
|
|
166
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
167
|
+
details: { indexName },
|
|
168
|
+
},
|
|
169
|
+
error,
|
|
170
|
+
);
|
|
171
|
+
this.logger?.error(mastraError.toString());
|
|
172
|
+
this.logger?.trackException(mastraError);
|
|
141
173
|
}
|
|
142
174
|
}
|
|
143
175
|
|
|
@@ -154,39 +186,46 @@ export class OpenSearchVector extends MastraVector {
|
|
|
154
186
|
const vectorIds = ids || vectors.map(() => crypto.randomUUID());
|
|
155
187
|
const operations = [];
|
|
156
188
|
|
|
157
|
-
|
|
158
|
-
|
|
189
|
+
try {
|
|
190
|
+
// Get index stats to check dimension
|
|
191
|
+
const indexInfo = await this.describeIndex({ indexName });
|
|
159
192
|
|
|
160
|
-
|
|
161
|
-
|
|
193
|
+
// Validate vector dimensions
|
|
194
|
+
this.validateVectorDimensions(vectors, indexInfo.dimension);
|
|
162
195
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
196
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
197
|
+
const operation = {
|
|
198
|
+
index: {
|
|
199
|
+
_index: indexName,
|
|
200
|
+
_id: vectorIds[i],
|
|
201
|
+
},
|
|
202
|
+
};
|
|
170
203
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
204
|
+
const document = {
|
|
205
|
+
id: vectorIds[i],
|
|
206
|
+
embedding: vectors[i],
|
|
207
|
+
metadata: metadata[i] || {},
|
|
208
|
+
};
|
|
176
209
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
210
|
+
operations.push(operation);
|
|
211
|
+
operations.push(document);
|
|
212
|
+
}
|
|
180
213
|
|
|
181
|
-
try {
|
|
182
214
|
if (operations.length > 0) {
|
|
183
215
|
await this.client.bulk({ body: operations, refresh: true });
|
|
184
216
|
}
|
|
185
217
|
|
|
186
218
|
return vectorIds;
|
|
187
219
|
} catch (error) {
|
|
188
|
-
|
|
189
|
-
|
|
220
|
+
throw new MastraError(
|
|
221
|
+
{
|
|
222
|
+
id: 'STORAGE_OPENSEARCH_VECTOR_UPSERT_FAILED',
|
|
223
|
+
domain: ErrorDomain.STORAGE,
|
|
224
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
225
|
+
details: { indexName, vectorCount: vectors?.length || 0 },
|
|
226
|
+
},
|
|
227
|
+
error,
|
|
228
|
+
);
|
|
190
229
|
}
|
|
191
230
|
}
|
|
192
231
|
|
|
@@ -206,7 +245,7 @@ export class OpenSearchVector extends MastraVector {
|
|
|
206
245
|
filter,
|
|
207
246
|
topK = 10,
|
|
208
247
|
includeVector = false,
|
|
209
|
-
}:
|
|
248
|
+
}: OpenSearchVectorParams): Promise<QueryResult[]> {
|
|
210
249
|
try {
|
|
211
250
|
const translatedFilter = this.transformFilter(filter);
|
|
212
251
|
|
|
@@ -234,9 +273,16 @@ export class OpenSearchVector extends MastraVector {
|
|
|
234
273
|
});
|
|
235
274
|
|
|
236
275
|
return results;
|
|
237
|
-
} catch (error
|
|
238
|
-
|
|
239
|
-
|
|
276
|
+
} catch (error) {
|
|
277
|
+
throw new MastraError(
|
|
278
|
+
{
|
|
279
|
+
id: 'STORAGE_OPENSEARCH_VECTOR_QUERY_FAILED',
|
|
280
|
+
domain: ErrorDomain.STORAGE,
|
|
281
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
282
|
+
details: { indexName, topK },
|
|
283
|
+
},
|
|
284
|
+
error,
|
|
285
|
+
);
|
|
240
286
|
}
|
|
241
287
|
}
|
|
242
288
|
|
|
@@ -256,10 +302,10 @@ export class OpenSearchVector extends MastraVector {
|
|
|
256
302
|
/**
|
|
257
303
|
* Transforms the filter to the OpenSearch DSL.
|
|
258
304
|
*
|
|
259
|
-
* @param {
|
|
305
|
+
* @param {OpenSearchVectorFilter} filter - The filter to transform.
|
|
260
306
|
* @returns {Record<string, any>} The transformed filter.
|
|
261
307
|
*/
|
|
262
|
-
private transformFilter(filter?:
|
|
308
|
+
private transformFilter(filter?: OpenSearchVectorFilter): any {
|
|
263
309
|
const translator = new OpenSearchFilterTranslator();
|
|
264
310
|
return translator.translate(filter);
|
|
265
311
|
}
|
|
@@ -275,13 +321,15 @@ export class OpenSearchVector extends MastraVector {
|
|
|
275
321
|
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
276
322
|
*/
|
|
277
323
|
async updateVector({ indexName, id, update }: UpdateVectorParams): Promise<void> {
|
|
278
|
-
|
|
279
|
-
throw new Error('No updates provided');
|
|
280
|
-
}
|
|
281
|
-
|
|
324
|
+
let existingDoc;
|
|
282
325
|
try {
|
|
326
|
+
if (!update.vector && !update.metadata) {
|
|
327
|
+
throw new Error('No updates provided');
|
|
328
|
+
}
|
|
329
|
+
|
|
283
330
|
// First get the current document to merge with updates
|
|
284
|
-
const { body
|
|
331
|
+
const { body } = await this.client
|
|
332
|
+
|
|
285
333
|
.get({
|
|
286
334
|
index: indexName,
|
|
287
335
|
id: id,
|
|
@@ -290,25 +338,40 @@ export class OpenSearchVector extends MastraVector {
|
|
|
290
338
|
throw new Error(`Document with ID ${id} not found in index ${indexName}`);
|
|
291
339
|
});
|
|
292
340
|
|
|
293
|
-
if (!
|
|
341
|
+
if (!body || !body._source) {
|
|
294
342
|
throw new Error(`Document with ID ${id} has no source data in index ${indexName}`);
|
|
295
343
|
}
|
|
344
|
+
existingDoc = body;
|
|
345
|
+
} catch (error) {
|
|
346
|
+
throw new MastraError(
|
|
347
|
+
{
|
|
348
|
+
id: 'STORAGE_OPENSEARCH_VECTOR_UPDATE_VECTOR_FAILED',
|
|
349
|
+
domain: ErrorDomain.STORAGE,
|
|
350
|
+
category: ErrorCategory.USER,
|
|
351
|
+
details: { indexName, id },
|
|
352
|
+
},
|
|
353
|
+
error,
|
|
354
|
+
);
|
|
355
|
+
}
|
|
296
356
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
357
|
+
const source = existingDoc._source;
|
|
358
|
+
const updatedDoc: Record<string, any> = {
|
|
359
|
+
id: source?.id || id,
|
|
360
|
+
};
|
|
301
361
|
|
|
362
|
+
try {
|
|
302
363
|
// Update vector if provided
|
|
303
364
|
if (update.vector) {
|
|
304
365
|
// Get index stats to check dimension
|
|
366
|
+
console.log(`1`);
|
|
305
367
|
const indexInfo = await this.describeIndex({ indexName });
|
|
306
368
|
|
|
307
369
|
// Validate vector dimensions
|
|
370
|
+
console.log(`2`);
|
|
308
371
|
this.validateVectorDimensions([update.vector], indexInfo.dimension);
|
|
309
372
|
|
|
310
373
|
updatedDoc.embedding = update.vector;
|
|
311
|
-
} else if (source
|
|
374
|
+
} else if (source?.embedding) {
|
|
312
375
|
updatedDoc.embedding = source.embedding;
|
|
313
376
|
}
|
|
314
377
|
|
|
@@ -316,10 +379,11 @@ export class OpenSearchVector extends MastraVector {
|
|
|
316
379
|
if (update.metadata) {
|
|
317
380
|
updatedDoc.metadata = update.metadata;
|
|
318
381
|
} else {
|
|
319
|
-
updatedDoc.metadata = source
|
|
382
|
+
updatedDoc.metadata = source?.metadata || {};
|
|
320
383
|
}
|
|
321
384
|
|
|
322
385
|
// Update the document
|
|
386
|
+
console.log(`3`);
|
|
323
387
|
await this.client.index({
|
|
324
388
|
index: indexName,
|
|
325
389
|
id: id,
|
|
@@ -327,8 +391,15 @@ export class OpenSearchVector extends MastraVector {
|
|
|
327
391
|
refresh: true,
|
|
328
392
|
});
|
|
329
393
|
} catch (error) {
|
|
330
|
-
|
|
331
|
-
|
|
394
|
+
throw new MastraError(
|
|
395
|
+
{
|
|
396
|
+
id: 'STORAGE_OPENSEARCH_VECTOR_UPDATE_VECTOR_FAILED',
|
|
397
|
+
domain: ErrorDomain.STORAGE,
|
|
398
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
399
|
+
details: { indexName, id },
|
|
400
|
+
},
|
|
401
|
+
error,
|
|
402
|
+
);
|
|
332
403
|
}
|
|
333
404
|
}
|
|
334
405
|
|
|
@@ -347,11 +418,19 @@ export class OpenSearchVector extends MastraVector {
|
|
|
347
418
|
refresh: true,
|
|
348
419
|
});
|
|
349
420
|
} catch (error: unknown) {
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
throw error;
|
|
421
|
+
// Don't throw error if document doesn't exist (404)
|
|
422
|
+
if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) {
|
|
423
|
+
return;
|
|
354
424
|
}
|
|
425
|
+
throw new MastraError(
|
|
426
|
+
{
|
|
427
|
+
id: 'STORAGE_OPENSEARCH_VECTOR_DELETE_VECTOR_FAILED',
|
|
428
|
+
domain: ErrorDomain.STORAGE,
|
|
429
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
430
|
+
details: { indexName, id },
|
|
431
|
+
},
|
|
432
|
+
error,
|
|
433
|
+
);
|
|
355
434
|
}
|
|
356
435
|
}
|
|
357
436
|
}
|