@mastra/pg 0.2.6-alpha.3 → 0.2.6-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +6 -0
- package/dist/_tsup-dts-rollup.d.cts +1 -0
- package/dist/_tsup-dts-rollup.d.ts +1 -0
- package/dist/index.cjs +44 -41
- package/dist/index.js +44 -41
- package/package.json +1 -1
- package/src/vector/index.ts +53 -49
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
|
|
2
|
-
> @mastra/pg@0.2.6-alpha.
|
|
2
|
+
> @mastra/pg@0.2.6-alpha.4 build /home/runner/work/mastra/mastra/stores/pg
|
|
3
3
|
> tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
|
|
4
4
|
|
|
5
5
|
[34mCLI[39m Building entry: src/index.ts
|
|
6
6
|
[34mCLI[39m Using tsconfig: tsconfig.json
|
|
7
7
|
[34mCLI[39m tsup v8.4.0
|
|
8
8
|
[34mTSC[39m Build start
|
|
9
|
-
[32mTSC[39m ⚡️ Build success in
|
|
9
|
+
[32mTSC[39m ⚡️ Build success in 10085ms
|
|
10
10
|
[34mDTS[39m Build start
|
|
11
11
|
[34mCLI[39m Target: es2022
|
|
12
12
|
Analysis will use the bundled TypeScript version 5.8.2
|
|
13
13
|
[36mWriting package typings: /home/runner/work/mastra/mastra/stores/pg/dist/_tsup-dts-rollup.d.ts[39m
|
|
14
14
|
Analysis will use the bundled TypeScript version 5.8.2
|
|
15
15
|
[36mWriting package typings: /home/runner/work/mastra/mastra/stores/pg/dist/_tsup-dts-rollup.d.cts[39m
|
|
16
|
-
[32mDTS[39m ⚡️ Build success in
|
|
16
|
+
[32mDTS[39m ⚡️ Build success in 10753ms
|
|
17
17
|
[34mCLI[39m Cleaning output folder
|
|
18
18
|
[34mESM[39m Build start
|
|
19
19
|
[34mCJS[39m Build start
|
|
20
|
-
[
|
|
21
|
-
[
|
|
22
|
-
[
|
|
23
|
-
[
|
|
20
|
+
[32mESM[39m [1mdist/index.js [22m[32m36.43 KB[39m
|
|
21
|
+
[32mESM[39m ⚡️ Build success in 1230ms
|
|
22
|
+
[32mCJS[39m [1mdist/index.cjs [22m[32m36.85 KB[39m
|
|
23
|
+
[32mCJS[39m ⚡️ Build success in 1230ms
|
package/CHANGELOG.md
CHANGED
|
@@ -211,6 +211,7 @@ declare class PgVector extends MastraVector {
|
|
|
211
211
|
*/
|
|
212
212
|
defineIndex(indexName: string, metric: "cosine" | "euclidean" | "dotproduct" | undefined, indexConfig: IndexConfig): Promise<void>;
|
|
213
213
|
buildIndex(...args: ParamsToArgs<PgDefineIndexParams> | PgDefineIndexArgs): Promise<void>;
|
|
214
|
+
private setupIndex;
|
|
214
215
|
listIndexes(): Promise<string[]>;
|
|
215
216
|
describeIndex(indexName: string): Promise<PGIndexStats>;
|
|
216
217
|
deleteIndex(indexName: string): Promise<void>;
|
|
@@ -211,6 +211,7 @@ declare class PgVector extends MastraVector {
|
|
|
211
211
|
*/
|
|
212
212
|
defineIndex(indexName: string, metric: "cosine" | "euclidean" | "dotproduct" | undefined, indexConfig: IndexConfig): Promise<void>;
|
|
213
213
|
buildIndex(...args: ParamsToArgs<PgDefineIndexParams> | PgDefineIndexArgs): Promise<void>;
|
|
214
|
+
private setupIndex;
|
|
214
215
|
listIndexes(): Promise<string[]>;
|
|
215
216
|
describeIndex(indexName: string): Promise<PGIndexStats>;
|
|
216
217
|
deleteIndex(indexName: string): Promise<void>;
|
package/dist/index.cjs
CHANGED
|
@@ -454,7 +454,7 @@ var PgVector = class extends vector.MastraVector {
|
|
|
454
454
|
await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
|
|
455
455
|
}
|
|
456
456
|
if (buildIndex) {
|
|
457
|
-
await this.
|
|
457
|
+
await this.setupIndex({ indexName, metric, indexConfig }, client);
|
|
458
458
|
}
|
|
459
459
|
} catch (error) {
|
|
460
460
|
console.error("Failed to create vector table:", error);
|
|
@@ -477,38 +477,44 @@ var PgVector = class extends vector.MastraVector {
|
|
|
477
477
|
const { indexName, metric = "cosine", indexConfig } = params;
|
|
478
478
|
const client = await this.pool.connect();
|
|
479
479
|
try {
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
480
|
+
await this.setupIndex({ indexName, metric, indexConfig }, client);
|
|
481
|
+
} finally {
|
|
482
|
+
client.release();
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
async setupIndex({ indexName, metric, indexConfig }, client) {
|
|
486
|
+
const hash = crypto$1.createHash("sha256").update("build:" + indexName).digest("hex");
|
|
487
|
+
const lockId = BigInt("0x" + hash.slice(0, 8)) % BigInt(2 ** 31);
|
|
488
|
+
const acquired = await client.query("SELECT pg_try_advisory_lock($1)", [lockId]);
|
|
489
|
+
if (!acquired.rows[0].pg_try_advisory_lock) {
|
|
490
|
+
const exists = await client.query(
|
|
491
|
+
`
|
|
486
492
|
SELECT 1 FROM pg_class c
|
|
487
493
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
488
494
|
WHERE c.relname = $1
|
|
489
495
|
AND n.nspname = 'public'
|
|
490
496
|
`,
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
}
|
|
498
|
-
await client.query("SELECT pg_advisory_lock($1)", [lockId]);
|
|
497
|
+
[`${indexName}_vector_idx`]
|
|
498
|
+
);
|
|
499
|
+
if (exists.rows.length > 0) {
|
|
500
|
+
console.log(`Index ${indexName}_vector_idx already exists, skipping creation`);
|
|
501
|
+
this.indexCache.delete(indexName);
|
|
502
|
+
return;
|
|
499
503
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
504
|
+
await client.query("SELECT pg_advisory_lock($1)", [lockId]);
|
|
505
|
+
}
|
|
506
|
+
try {
|
|
507
|
+
await client.query(`DROP INDEX IF EXISTS ${indexName}_vector_idx`);
|
|
508
|
+
if (indexConfig.type === "flat") {
|
|
509
|
+
this.indexCache.delete(indexName);
|
|
510
|
+
return;
|
|
511
|
+
}
|
|
512
|
+
const metricOp = metric === "cosine" ? "vector_cosine_ops" : metric === "euclidean" ? "vector_l2_ops" : "vector_ip_ops";
|
|
513
|
+
let indexSQL;
|
|
514
|
+
if (indexConfig.type === "hnsw") {
|
|
515
|
+
const m = indexConfig.hnsw?.m ?? 8;
|
|
516
|
+
const efConstruction = indexConfig.hnsw?.efConstruction ?? 32;
|
|
517
|
+
indexSQL = `
|
|
512
518
|
CREATE INDEX IF NOT EXISTS ${indexName}_vector_idx
|
|
513
519
|
ON ${indexName}
|
|
514
520
|
USING hnsw (embedding ${metricOp})
|
|
@@ -517,28 +523,25 @@ var PgVector = class extends vector.MastraVector {
|
|
|
517
523
|
ef_construction = ${efConstruction}
|
|
518
524
|
)
|
|
519
525
|
`;
|
|
526
|
+
} else {
|
|
527
|
+
let lists;
|
|
528
|
+
if (indexConfig.ivf?.lists) {
|
|
529
|
+
lists = indexConfig.ivf.lists;
|
|
520
530
|
} else {
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
const size = (await client.query(`SELECT COUNT(*) FROM ${indexName}`)).rows[0].count;
|
|
526
|
-
lists = Math.max(100, Math.min(4e3, Math.floor(Math.sqrt(size) * 2)));
|
|
527
|
-
}
|
|
528
|
-
indexSQL = `
|
|
531
|
+
const size = (await client.query(`SELECT COUNT(*) FROM ${indexName}`)).rows[0].count;
|
|
532
|
+
lists = Math.max(100, Math.min(4e3, Math.floor(Math.sqrt(size) * 2)));
|
|
533
|
+
}
|
|
534
|
+
indexSQL = `
|
|
529
535
|
CREATE INDEX IF NOT EXISTS ${indexName}_vector_idx
|
|
530
536
|
ON ${indexName}
|
|
531
537
|
USING ivfflat (embedding ${metricOp})
|
|
532
538
|
WITH (lists = ${lists});
|
|
533
539
|
`;
|
|
534
|
-
}
|
|
535
|
-
await client.query(indexSQL);
|
|
536
|
-
this.indexCache.delete(indexName);
|
|
537
|
-
} finally {
|
|
538
|
-
await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
|
|
539
540
|
}
|
|
541
|
+
await client.query(indexSQL);
|
|
542
|
+
this.indexCache.delete(indexName);
|
|
540
543
|
} finally {
|
|
541
|
-
client.
|
|
544
|
+
await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
|
|
542
545
|
}
|
|
543
546
|
}
|
|
544
547
|
async listIndexes() {
|
package/dist/index.js
CHANGED
|
@@ -447,7 +447,7 @@ var PgVector = class extends MastraVector {
|
|
|
447
447
|
await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
|
|
448
448
|
}
|
|
449
449
|
if (buildIndex) {
|
|
450
|
-
await this.
|
|
450
|
+
await this.setupIndex({ indexName, metric, indexConfig }, client);
|
|
451
451
|
}
|
|
452
452
|
} catch (error) {
|
|
453
453
|
console.error("Failed to create vector table:", error);
|
|
@@ -470,38 +470,44 @@ var PgVector = class extends MastraVector {
|
|
|
470
470
|
const { indexName, metric = "cosine", indexConfig } = params;
|
|
471
471
|
const client = await this.pool.connect();
|
|
472
472
|
try {
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
473
|
+
await this.setupIndex({ indexName, metric, indexConfig }, client);
|
|
474
|
+
} finally {
|
|
475
|
+
client.release();
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
async setupIndex({ indexName, metric, indexConfig }, client) {
|
|
479
|
+
const hash = createHash("sha256").update("build:" + indexName).digest("hex");
|
|
480
|
+
const lockId = BigInt("0x" + hash.slice(0, 8)) % BigInt(2 ** 31);
|
|
481
|
+
const acquired = await client.query("SELECT pg_try_advisory_lock($1)", [lockId]);
|
|
482
|
+
if (!acquired.rows[0].pg_try_advisory_lock) {
|
|
483
|
+
const exists = await client.query(
|
|
484
|
+
`
|
|
479
485
|
SELECT 1 FROM pg_class c
|
|
480
486
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
481
487
|
WHERE c.relname = $1
|
|
482
488
|
AND n.nspname = 'public'
|
|
483
489
|
`,
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
}
|
|
491
|
-
await client.query("SELECT pg_advisory_lock($1)", [lockId]);
|
|
490
|
+
[`${indexName}_vector_idx`]
|
|
491
|
+
);
|
|
492
|
+
if (exists.rows.length > 0) {
|
|
493
|
+
console.log(`Index ${indexName}_vector_idx already exists, skipping creation`);
|
|
494
|
+
this.indexCache.delete(indexName);
|
|
495
|
+
return;
|
|
492
496
|
}
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
497
|
+
await client.query("SELECT pg_advisory_lock($1)", [lockId]);
|
|
498
|
+
}
|
|
499
|
+
try {
|
|
500
|
+
await client.query(`DROP INDEX IF EXISTS ${indexName}_vector_idx`);
|
|
501
|
+
if (indexConfig.type === "flat") {
|
|
502
|
+
this.indexCache.delete(indexName);
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
const metricOp = metric === "cosine" ? "vector_cosine_ops" : metric === "euclidean" ? "vector_l2_ops" : "vector_ip_ops";
|
|
506
|
+
let indexSQL;
|
|
507
|
+
if (indexConfig.type === "hnsw") {
|
|
508
|
+
const m = indexConfig.hnsw?.m ?? 8;
|
|
509
|
+
const efConstruction = indexConfig.hnsw?.efConstruction ?? 32;
|
|
510
|
+
indexSQL = `
|
|
505
511
|
CREATE INDEX IF NOT EXISTS ${indexName}_vector_idx
|
|
506
512
|
ON ${indexName}
|
|
507
513
|
USING hnsw (embedding ${metricOp})
|
|
@@ -510,28 +516,25 @@ var PgVector = class extends MastraVector {
|
|
|
510
516
|
ef_construction = ${efConstruction}
|
|
511
517
|
)
|
|
512
518
|
`;
|
|
519
|
+
} else {
|
|
520
|
+
let lists;
|
|
521
|
+
if (indexConfig.ivf?.lists) {
|
|
522
|
+
lists = indexConfig.ivf.lists;
|
|
513
523
|
} else {
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
const size = (await client.query(`SELECT COUNT(*) FROM ${indexName}`)).rows[0].count;
|
|
519
|
-
lists = Math.max(100, Math.min(4e3, Math.floor(Math.sqrt(size) * 2)));
|
|
520
|
-
}
|
|
521
|
-
indexSQL = `
|
|
524
|
+
const size = (await client.query(`SELECT COUNT(*) FROM ${indexName}`)).rows[0].count;
|
|
525
|
+
lists = Math.max(100, Math.min(4e3, Math.floor(Math.sqrt(size) * 2)));
|
|
526
|
+
}
|
|
527
|
+
indexSQL = `
|
|
522
528
|
CREATE INDEX IF NOT EXISTS ${indexName}_vector_idx
|
|
523
529
|
ON ${indexName}
|
|
524
530
|
USING ivfflat (embedding ${metricOp})
|
|
525
531
|
WITH (lists = ${lists});
|
|
526
532
|
`;
|
|
527
|
-
}
|
|
528
|
-
await client.query(indexSQL);
|
|
529
|
-
this.indexCache.delete(indexName);
|
|
530
|
-
} finally {
|
|
531
|
-
await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
|
|
532
533
|
}
|
|
534
|
+
await client.query(indexSQL);
|
|
535
|
+
this.indexCache.delete(indexName);
|
|
533
536
|
} finally {
|
|
534
|
-
client.
|
|
537
|
+
await client.query("SELECT pg_advisory_unlock($1)", [lockId]);
|
|
535
538
|
}
|
|
536
539
|
}
|
|
537
540
|
async listIndexes() {
|
package/package.json
CHANGED
package/src/vector/index.ts
CHANGED
|
@@ -262,7 +262,7 @@ export class PgVector extends MastraVector {
|
|
|
262
262
|
}
|
|
263
263
|
|
|
264
264
|
if (buildIndex) {
|
|
265
|
-
await this.
|
|
265
|
+
await this.setupIndex({ indexName, metric, indexConfig }, client);
|
|
266
266
|
}
|
|
267
267
|
} catch (error: any) {
|
|
268
268
|
console.error('Failed to create vector table:', error);
|
|
@@ -293,52 +293,59 @@ export class PgVector extends MastraVector {
|
|
|
293
293
|
|
|
294
294
|
const client = await this.pool.connect();
|
|
295
295
|
try {
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
const acquired = await client.query('SELECT pg_try_advisory_lock($1)', [lockId]);
|
|
296
|
+
await this.setupIndex({ indexName, metric, indexConfig }, client);
|
|
297
|
+
} finally {
|
|
298
|
+
client.release();
|
|
299
|
+
}
|
|
300
|
+
}
|
|
302
301
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
302
|
+
private async setupIndex({ indexName, metric, indexConfig }: PgDefineIndexParams, client: pg.PoolClient) {
|
|
303
|
+
// Use a different hash prefix for buildIndex locks to avoid conflicts with createIndex
|
|
304
|
+
const hash = createHash('sha256')
|
|
305
|
+
.update('build:' + indexName)
|
|
306
|
+
.digest('hex');
|
|
307
|
+
const lockId = BigInt('0x' + hash.slice(0, 8)) % BigInt(2 ** 31);
|
|
308
|
+
const acquired = await client.query('SELECT pg_try_advisory_lock($1)', [lockId]);
|
|
309
|
+
|
|
310
|
+
if (!acquired.rows[0].pg_try_advisory_lock) {
|
|
311
|
+
// Check if index already exists
|
|
312
|
+
const exists = await client.query(
|
|
313
|
+
`
|
|
307
314
|
SELECT 1 FROM pg_class c
|
|
308
315
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
309
316
|
WHERE c.relname = $1
|
|
310
317
|
AND n.nspname = 'public'
|
|
311
318
|
`,
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
if (exists.rows.length > 0) {
|
|
316
|
-
console.log(`Index ${indexName}_vector_idx already exists, skipping creation`);
|
|
317
|
-
this.indexCache.delete(indexName); // Still clear cache since we checked
|
|
318
|
-
return;
|
|
319
|
-
}
|
|
319
|
+
[`${indexName}_vector_idx`],
|
|
320
|
+
);
|
|
320
321
|
|
|
321
|
-
|
|
322
|
-
|
|
322
|
+
if (exists.rows.length > 0) {
|
|
323
|
+
console.log(`Index ${indexName}_vector_idx already exists, skipping creation`);
|
|
324
|
+
this.indexCache.delete(indexName); // Still clear cache since we checked
|
|
325
|
+
return;
|
|
323
326
|
}
|
|
324
327
|
|
|
325
|
-
|
|
326
|
-
|
|
328
|
+
// Index doesn't exist, wait for lock
|
|
329
|
+
await client.query('SELECT pg_advisory_lock($1)', [lockId]);
|
|
330
|
+
}
|
|
327
331
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
+
try {
|
|
333
|
+
await client.query(`DROP INDEX IF EXISTS ${indexName}_vector_idx`);
|
|
334
|
+
|
|
335
|
+
if (indexConfig.type === 'flat') {
|
|
336
|
+
this.indexCache.delete(indexName);
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
332
339
|
|
|
333
|
-
|
|
334
|
-
|
|
340
|
+
const metricOp =
|
|
341
|
+
metric === 'cosine' ? 'vector_cosine_ops' : metric === 'euclidean' ? 'vector_l2_ops' : 'vector_ip_ops';
|
|
335
342
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
343
|
+
let indexSQL: string;
|
|
344
|
+
if (indexConfig.type === 'hnsw') {
|
|
345
|
+
const m = indexConfig.hnsw?.m ?? 8;
|
|
346
|
+
const efConstruction = indexConfig.hnsw?.efConstruction ?? 32;
|
|
340
347
|
|
|
341
|
-
|
|
348
|
+
indexSQL = `
|
|
342
349
|
CREATE INDEX IF NOT EXISTS ${indexName}_vector_idx
|
|
343
350
|
ON ${indexName}
|
|
344
351
|
USING hnsw (embedding ${metricOp})
|
|
@@ -347,29 +354,26 @@ export class PgVector extends MastraVector {
|
|
|
347
354
|
ef_construction = ${efConstruction}
|
|
348
355
|
)
|
|
349
356
|
`;
|
|
357
|
+
} else {
|
|
358
|
+
let lists: number;
|
|
359
|
+
if (indexConfig.ivf?.lists) {
|
|
360
|
+
lists = indexConfig.ivf.lists;
|
|
350
361
|
} else {
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
const size = (await client.query(`SELECT COUNT(*) FROM ${indexName}`)).rows[0].count;
|
|
356
|
-
lists = Math.max(100, Math.min(4000, Math.floor(Math.sqrt(size) * 2)));
|
|
357
|
-
}
|
|
358
|
-
indexSQL = `
|
|
362
|
+
const size = (await client.query(`SELECT COUNT(*) FROM ${indexName}`)).rows[0].count;
|
|
363
|
+
lists = Math.max(100, Math.min(4000, Math.floor(Math.sqrt(size) * 2)));
|
|
364
|
+
}
|
|
365
|
+
indexSQL = `
|
|
359
366
|
CREATE INDEX IF NOT EXISTS ${indexName}_vector_idx
|
|
360
367
|
ON ${indexName}
|
|
361
368
|
USING ivfflat (embedding ${metricOp})
|
|
362
369
|
WITH (lists = ${lists});
|
|
363
370
|
`;
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
await client.query(indexSQL);
|
|
367
|
-
this.indexCache.delete(indexName);
|
|
368
|
-
} finally {
|
|
369
|
-
await client.query('SELECT pg_advisory_unlock($1)', [lockId]);
|
|
370
371
|
}
|
|
372
|
+
|
|
373
|
+
await client.query(indexSQL);
|
|
374
|
+
this.indexCache.delete(indexName);
|
|
371
375
|
} finally {
|
|
372
|
-
client.
|
|
376
|
+
await client.query('SELECT pg_advisory_unlock($1)', [lockId]);
|
|
373
377
|
}
|
|
374
378
|
}
|
|
375
379
|
|