@mastra/pg 0.11.0 → 0.11.1-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +24 -0
- package/dist/_tsup-dts-rollup.d.cts +11 -0
- package/dist/_tsup-dts-rollup.d.ts +11 -0
- package/dist/index.cjs +809 -247
- package/dist/index.js +785 -223
- package/package.json +3 -3
- package/src/storage/index.test.ts +242 -12
- package/src/storage/index.ts +606 -191
- package/src/vector/index.ts +279 -86
package/src/vector/index.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ErrorCategory, ErrorDomain, MastraError } from '@mastra/core/error';
|
|
1
2
|
import { parseSqlIdentifier } from '@mastra/core/utils';
|
|
2
3
|
import { MastraVector } from '@mastra/core/vector';
|
|
3
4
|
import type {
|
|
@@ -75,47 +76,61 @@ export class PgVector extends MastraVector {
|
|
|
75
76
|
schemaName?: string;
|
|
76
77
|
pgPoolOptions?: Omit<pg.PoolConfig, 'connectionString'>;
|
|
77
78
|
}) {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
79
|
+
try {
|
|
80
|
+
if (!connectionString || connectionString.trim() === '') {
|
|
81
|
+
throw new Error(
|
|
82
|
+
'PgVector: connectionString must be provided and cannot be empty. Passing an empty string may cause fallback to local Postgres defaults.',
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
super();
|
|
84
86
|
|
|
85
|
-
|
|
87
|
+
this.schema = schemaName;
|
|
86
88
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
const telemetry = this.__getTelemetry();
|
|
89
|
+
const basePool = new pg.Pool({
|
|
90
|
+
connectionString,
|
|
91
|
+
max: 20, // Maximum number of clients in the pool
|
|
92
|
+
idleTimeoutMillis: 30000, // Close idle connections after 30 seconds
|
|
93
|
+
connectionTimeoutMillis: 2000, // Fail fast if can't connect
|
|
94
|
+
...pgPoolOptions,
|
|
95
|
+
});
|
|
96
96
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
const
|
|
110
|
-
|
|
111
|
-
indexName
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
97
|
+
const telemetry = this.__getTelemetry();
|
|
98
|
+
|
|
99
|
+
this.pool =
|
|
100
|
+
telemetry?.traceClass(basePool, {
|
|
101
|
+
spanNamePrefix: 'pg-vector',
|
|
102
|
+
attributes: {
|
|
103
|
+
'vector.type': 'postgres',
|
|
104
|
+
},
|
|
105
|
+
}) ?? basePool;
|
|
106
|
+
|
|
107
|
+
void (async () => {
|
|
108
|
+
// warm the created indexes cache so we don't need to check if indexes exist every time
|
|
109
|
+
const existingIndexes = await this.listIndexes();
|
|
110
|
+
void existingIndexes.map(async indexName => {
|
|
111
|
+
const info = await this.getIndexInfo({ indexName });
|
|
112
|
+
const key = await this.getIndexCacheKey({
|
|
113
|
+
indexName,
|
|
114
|
+
metric: info.metric,
|
|
115
|
+
dimension: info.dimension,
|
|
116
|
+
type: info.type,
|
|
117
|
+
});
|
|
118
|
+
this.createdIndexes.set(indexName, key);
|
|
115
119
|
});
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
120
|
+
})();
|
|
121
|
+
} catch (error) {
|
|
122
|
+
throw new MastraError(
|
|
123
|
+
{
|
|
124
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_INITIALIZATION_FAILED',
|
|
125
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
126
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
127
|
+
details: {
|
|
128
|
+
schemaName: schemaName ?? '',
|
|
129
|
+
},
|
|
130
|
+
},
|
|
131
|
+
error,
|
|
132
|
+
);
|
|
133
|
+
}
|
|
119
134
|
}
|
|
120
135
|
|
|
121
136
|
private getMutexByName(indexName: string) {
|
|
@@ -160,11 +175,27 @@ export class PgVector extends MastraVector {
|
|
|
160
175
|
ef,
|
|
161
176
|
probes,
|
|
162
177
|
}: PgQueryVectorParams): Promise<QueryResult[]> {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
178
|
+
try {
|
|
179
|
+
if (!Number.isInteger(topK) || topK <= 0) {
|
|
180
|
+
throw new Error('topK must be a positive integer');
|
|
181
|
+
}
|
|
182
|
+
if (!Array.isArray(queryVector) || !queryVector.every(x => typeof x === 'number' && Number.isFinite(x))) {
|
|
183
|
+
throw new Error('queryVector must be an array of finite numbers');
|
|
184
|
+
}
|
|
185
|
+
} catch (error) {
|
|
186
|
+
const mastraError = new MastraError(
|
|
187
|
+
{
|
|
188
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_QUERY_INVALID_INPUT',
|
|
189
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
190
|
+
category: ErrorCategory.USER,
|
|
191
|
+
details: {
|
|
192
|
+
indexName,
|
|
193
|
+
},
|
|
194
|
+
},
|
|
195
|
+
error,
|
|
196
|
+
);
|
|
197
|
+
this.logger?.trackException(mastraError);
|
|
198
|
+
throw mastraError;
|
|
168
199
|
}
|
|
169
200
|
|
|
170
201
|
const client = await this.pool.connect();
|
|
@@ -213,6 +244,20 @@ export class PgVector extends MastraVector {
|
|
|
213
244
|
metadata,
|
|
214
245
|
...(includeVector && embedding && { vector: JSON.parse(embedding) }),
|
|
215
246
|
}));
|
|
247
|
+
} catch (error) {
|
|
248
|
+
const mastraError = new MastraError(
|
|
249
|
+
{
|
|
250
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_QUERY_FAILED',
|
|
251
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
252
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
253
|
+
details: {
|
|
254
|
+
indexName,
|
|
255
|
+
},
|
|
256
|
+
},
|
|
257
|
+
error,
|
|
258
|
+
);
|
|
259
|
+
this.logger?.trackException(mastraError);
|
|
260
|
+
throw mastraError;
|
|
216
261
|
} finally {
|
|
217
262
|
client.release();
|
|
218
263
|
}
|
|
@@ -249,13 +294,40 @@ export class PgVector extends MastraVector {
|
|
|
249
294
|
const match = error.message.match(/expected (\d+) dimensions, not (\d+)/);
|
|
250
295
|
if (match) {
|
|
251
296
|
const [, expected, actual] = match;
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
297
|
+
const mastraError = new MastraError(
|
|
298
|
+
{
|
|
299
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_UPSERT_INVALID_INPUT',
|
|
300
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
301
|
+
category: ErrorCategory.USER,
|
|
302
|
+
text:
|
|
303
|
+
`Vector dimension mismatch: Index "${indexName}" expects ${expected} dimensions but got ${actual} dimensions. ` +
|
|
304
|
+
`Either use a matching embedding model or delete and recreate the index with the new dimension.`,
|
|
305
|
+
details: {
|
|
306
|
+
indexName,
|
|
307
|
+
expected: expected ?? '',
|
|
308
|
+
actual: actual ?? '',
|
|
309
|
+
},
|
|
310
|
+
},
|
|
311
|
+
error,
|
|
255
312
|
);
|
|
313
|
+
this.logger?.trackException(mastraError);
|
|
314
|
+
throw mastraError;
|
|
256
315
|
}
|
|
257
316
|
}
|
|
258
|
-
|
|
317
|
+
|
|
318
|
+
const mastraError = new MastraError(
|
|
319
|
+
{
|
|
320
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_UPSERT_FAILED',
|
|
321
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
322
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
323
|
+
details: {
|
|
324
|
+
indexName,
|
|
325
|
+
},
|
|
326
|
+
},
|
|
327
|
+
error,
|
|
328
|
+
);
|
|
329
|
+
this.logger?.trackException(mastraError);
|
|
330
|
+
throw mastraError;
|
|
259
331
|
} finally {
|
|
260
332
|
client.release();
|
|
261
333
|
}
|
|
@@ -336,11 +408,27 @@ export class PgVector extends MastraVector {
|
|
|
336
408
|
const { tableName } = this.getTableName(indexName);
|
|
337
409
|
|
|
338
410
|
// Validate inputs
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
411
|
+
try {
|
|
412
|
+
if (!indexName.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
|
|
413
|
+
throw new Error('Invalid index name format');
|
|
414
|
+
}
|
|
415
|
+
if (!Number.isInteger(dimension) || dimension <= 0) {
|
|
416
|
+
throw new Error('Dimension must be a positive integer');
|
|
417
|
+
}
|
|
418
|
+
} catch (error) {
|
|
419
|
+
const mastraError = new MastraError(
|
|
420
|
+
{
|
|
421
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_CREATE_INDEX_INVALID_INPUT',
|
|
422
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
423
|
+
category: ErrorCategory.USER,
|
|
424
|
+
details: {
|
|
425
|
+
indexName,
|
|
426
|
+
},
|
|
427
|
+
},
|
|
428
|
+
error,
|
|
429
|
+
);
|
|
430
|
+
this.logger?.trackException(mastraError);
|
|
431
|
+
throw mastraError;
|
|
344
432
|
}
|
|
345
433
|
|
|
346
434
|
const indexCacheKey = await this.getIndexCacheKey({ indexName, dimension, type: indexConfig.type, metric });
|
|
@@ -351,21 +439,22 @@ export class PgVector extends MastraVector {
|
|
|
351
439
|
|
|
352
440
|
const mutex = this.getMutexByName(`create-${indexName}`);
|
|
353
441
|
// Use async-mutex instead of advisory lock for perf (over 2x as fast)
|
|
354
|
-
await mutex
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
442
|
+
await mutex
|
|
443
|
+
.runExclusive(async () => {
|
|
444
|
+
if (this.cachedIndexExists(indexName, indexCacheKey)) {
|
|
445
|
+
// this may have been created while we were waiting to acquire a lock
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
359
448
|
|
|
360
|
-
|
|
449
|
+
const client = await this.pool.connect();
|
|
361
450
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
451
|
+
try {
|
|
452
|
+
// Setup schema if needed
|
|
453
|
+
await this.setupSchema(client);
|
|
365
454
|
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
455
|
+
// Install vector extension first (needs to be in public schema)
|
|
456
|
+
await this.installVectorExtension(client);
|
|
457
|
+
await client.query(`
|
|
369
458
|
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
370
459
|
id SERIAL PRIMARY KEY,
|
|
371
460
|
vector_id TEXT UNIQUE NOT NULL,
|
|
@@ -373,24 +462,53 @@ export class PgVector extends MastraVector {
|
|
|
373
462
|
metadata JSONB DEFAULT '{}'::jsonb
|
|
374
463
|
);
|
|
375
464
|
`);
|
|
376
|
-
|
|
465
|
+
this.createdIndexes.set(indexName, indexCacheKey);
|
|
377
466
|
|
|
378
|
-
|
|
379
|
-
|
|
467
|
+
if (buildIndex) {
|
|
468
|
+
await this.setupIndex({ indexName, metric, indexConfig }, client);
|
|
469
|
+
}
|
|
470
|
+
} catch (error: any) {
|
|
471
|
+
this.createdIndexes.delete(indexName);
|
|
472
|
+
throw error;
|
|
473
|
+
} finally {
|
|
474
|
+
client.release();
|
|
380
475
|
}
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
476
|
+
})
|
|
477
|
+
.catch(error => {
|
|
478
|
+
const mastraError = new MastraError(
|
|
479
|
+
{
|
|
480
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_CREATE_INDEX_FAILED',
|
|
481
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
482
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
483
|
+
details: {
|
|
484
|
+
indexName,
|
|
485
|
+
},
|
|
486
|
+
},
|
|
487
|
+
error,
|
|
488
|
+
);
|
|
489
|
+
this.logger?.trackException(mastraError);
|
|
490
|
+
throw mastraError;
|
|
491
|
+
});
|
|
388
492
|
}
|
|
389
493
|
|
|
390
494
|
async buildIndex({ indexName, metric = 'cosine', indexConfig }: PgDefineIndexParams): Promise<void> {
|
|
391
495
|
const client = await this.pool.connect();
|
|
392
496
|
try {
|
|
393
497
|
await this.setupIndex({ indexName, metric, indexConfig }, client);
|
|
498
|
+
} catch (error: any) {
|
|
499
|
+
const mastraError = new MastraError(
|
|
500
|
+
{
|
|
501
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_BUILD_INDEX_FAILED',
|
|
502
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
503
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
504
|
+
details: {
|
|
505
|
+
indexName,
|
|
506
|
+
},
|
|
507
|
+
},
|
|
508
|
+
error,
|
|
509
|
+
);
|
|
510
|
+
this.logger?.trackException(mastraError);
|
|
511
|
+
throw mastraError;
|
|
394
512
|
} finally {
|
|
395
513
|
client.release();
|
|
396
514
|
}
|
|
@@ -512,6 +630,17 @@ export class PgVector extends MastraVector {
|
|
|
512
630
|
`;
|
|
513
631
|
const vectorTables = await client.query(vectorTablesQuery, [this.schema || 'public']);
|
|
514
632
|
return vectorTables.rows.map(row => row.table_name);
|
|
633
|
+
} catch (e) {
|
|
634
|
+
const mastraError = new MastraError(
|
|
635
|
+
{
|
|
636
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_LIST_INDEXES_FAILED',
|
|
637
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
638
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
639
|
+
},
|
|
640
|
+
e,
|
|
641
|
+
);
|
|
642
|
+
this.logger?.trackException(mastraError);
|
|
643
|
+
throw mastraError;
|
|
515
644
|
} finally {
|
|
516
645
|
client.release();
|
|
517
646
|
}
|
|
@@ -613,7 +742,19 @@ export class PgVector extends MastraVector {
|
|
|
613
742
|
};
|
|
614
743
|
} catch (e: any) {
|
|
615
744
|
await client.query('ROLLBACK');
|
|
616
|
-
|
|
745
|
+
const mastraError = new MastraError(
|
|
746
|
+
{
|
|
747
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_DESCRIBE_INDEX_FAILED',
|
|
748
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
749
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
750
|
+
details: {
|
|
751
|
+
indexName,
|
|
752
|
+
},
|
|
753
|
+
},
|
|
754
|
+
e,
|
|
755
|
+
);
|
|
756
|
+
this.logger?.trackException(mastraError);
|
|
757
|
+
throw mastraError;
|
|
617
758
|
} finally {
|
|
618
759
|
client.release();
|
|
619
760
|
}
|
|
@@ -628,7 +769,19 @@ export class PgVector extends MastraVector {
|
|
|
628
769
|
this.createdIndexes.delete(indexName);
|
|
629
770
|
} catch (error: any) {
|
|
630
771
|
await client.query('ROLLBACK');
|
|
631
|
-
|
|
772
|
+
const mastraError = new MastraError(
|
|
773
|
+
{
|
|
774
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_DELETE_INDEX_FAILED',
|
|
775
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
776
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
777
|
+
details: {
|
|
778
|
+
indexName,
|
|
779
|
+
},
|
|
780
|
+
},
|
|
781
|
+
error,
|
|
782
|
+
);
|
|
783
|
+
this.logger?.trackException(mastraError);
|
|
784
|
+
throw mastraError;
|
|
632
785
|
} finally {
|
|
633
786
|
client.release();
|
|
634
787
|
}
|
|
@@ -641,7 +794,19 @@ export class PgVector extends MastraVector {
|
|
|
641
794
|
await client.query(`TRUNCATE ${tableName}`);
|
|
642
795
|
} catch (e: any) {
|
|
643
796
|
await client.query('ROLLBACK');
|
|
644
|
-
|
|
797
|
+
const mastraError = new MastraError(
|
|
798
|
+
{
|
|
799
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_TRUNCATE_INDEX_FAILED',
|
|
800
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
801
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
802
|
+
details: {
|
|
803
|
+
indexName,
|
|
804
|
+
},
|
|
805
|
+
},
|
|
806
|
+
e,
|
|
807
|
+
);
|
|
808
|
+
this.logger?.trackException(mastraError);
|
|
809
|
+
throw mastraError;
|
|
645
810
|
} finally {
|
|
646
811
|
client.release();
|
|
647
812
|
}
|
|
@@ -662,12 +827,13 @@ export class PgVector extends MastraVector {
|
|
|
662
827
|
* @throws Will throw an error if no updates are provided or if the update operation fails.
|
|
663
828
|
*/
|
|
664
829
|
async updateVector({ indexName, id, update }: UpdateVectorParams): Promise<void> {
|
|
665
|
-
|
|
666
|
-
throw new Error('No updates provided');
|
|
667
|
-
}
|
|
668
|
-
|
|
669
|
-
const client = await this.pool.connect();
|
|
830
|
+
let client;
|
|
670
831
|
try {
|
|
832
|
+
if (!update.vector && !update.metadata) {
|
|
833
|
+
throw new Error('No updates provided');
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
client = await this.pool.connect();
|
|
671
837
|
let updateParts = [];
|
|
672
838
|
let values = [id];
|
|
673
839
|
let valueIndex = 2;
|
|
@@ -699,9 +865,22 @@ export class PgVector extends MastraVector {
|
|
|
699
865
|
|
|
700
866
|
await client.query(query, values);
|
|
701
867
|
} catch (error: any) {
|
|
702
|
-
|
|
868
|
+
const mastraError = new MastraError(
|
|
869
|
+
{
|
|
870
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_UPDATE_VECTOR_FAILED',
|
|
871
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
872
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
873
|
+
details: {
|
|
874
|
+
indexName,
|
|
875
|
+
id,
|
|
876
|
+
},
|
|
877
|
+
},
|
|
878
|
+
error,
|
|
879
|
+
);
|
|
880
|
+
this.logger?.trackException(mastraError);
|
|
881
|
+
throw mastraError;
|
|
703
882
|
} finally {
|
|
704
|
-
client
|
|
883
|
+
client?.release();
|
|
705
884
|
}
|
|
706
885
|
}
|
|
707
886
|
|
|
@@ -713,8 +892,9 @@ export class PgVector extends MastraVector {
|
|
|
713
892
|
* @throws Will throw an error if the deletion operation fails.
|
|
714
893
|
*/
|
|
715
894
|
async deleteVector({ indexName, id }: DeleteVectorParams): Promise<void> {
|
|
716
|
-
|
|
895
|
+
let client;
|
|
717
896
|
try {
|
|
897
|
+
client = await this.pool.connect();
|
|
718
898
|
const { tableName } = this.getTableName(indexName);
|
|
719
899
|
const query = `
|
|
720
900
|
DELETE FROM ${tableName}
|
|
@@ -722,9 +902,22 @@ export class PgVector extends MastraVector {
|
|
|
722
902
|
`;
|
|
723
903
|
await client.query(query, [id]);
|
|
724
904
|
} catch (error: any) {
|
|
725
|
-
|
|
905
|
+
const mastraError = new MastraError(
|
|
906
|
+
{
|
|
907
|
+
id: 'MASTRA_STORAGE_PG_VECTOR_DELETE_VECTOR_FAILED',
|
|
908
|
+
domain: ErrorDomain.MASTRA_VECTOR,
|
|
909
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
910
|
+
details: {
|
|
911
|
+
indexName,
|
|
912
|
+
id,
|
|
913
|
+
},
|
|
914
|
+
},
|
|
915
|
+
error,
|
|
916
|
+
);
|
|
917
|
+
this.logger?.trackException(mastraError);
|
|
918
|
+
throw mastraError;
|
|
726
919
|
} finally {
|
|
727
|
-
client
|
|
920
|
+
client?.release();
|
|
728
921
|
}
|
|
729
922
|
}
|
|
730
923
|
}
|