@mastra/pg 0.1.6-alpha.0 → 0.1.6-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +11 -6
- package/CHANGELOG.md +39 -0
- package/README.md +22 -17
- package/dist/_tsup-dts-rollup.d.cts +329 -0
- package/dist/_tsup-dts-rollup.d.ts +58 -29
- package/dist/index.cjs +1050 -0
- package/dist/index.d.cts +4 -0
- package/dist/index.js +35 -22
- package/eslint.config.js +6 -0
- package/package.json +12 -5
- package/src/storage/index.test.ts +3 -2
- package/src/storage/index.ts +11 -24
- package/src/vector/filter.ts +5 -4
- package/src/vector/index.test.ts +698 -316
- package/src/vector/index.ts +69 -37
- package/src/vector/performance.helpers.ts +3 -3
- package/src/vector/sql-builder.ts +17 -13
- package/src/vector/vector.performance.test.ts +16 -20
package/dist/index.d.cts
ADDED
package/dist/index.js
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
import { MastraVector } from '@mastra/core/vector';
|
|
2
2
|
import pg from 'pg';
|
|
3
|
-
import { BaseFilterTranslator } from '@mastra/core/filter';
|
|
4
|
-
import '@mastra/core/memory';
|
|
3
|
+
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
5
4
|
import { MastraStorage } from '@mastra/core/storage';
|
|
6
|
-
import '@mastra/core/workflows';
|
|
7
5
|
import pgPromise from 'pg-promise';
|
|
8
6
|
|
|
9
7
|
// src/vector/index.ts
|
|
@@ -73,6 +71,8 @@ var PGFilterTranslator = class extends BaseFilterTranslator {
|
|
|
73
71
|
return { $regex: flags ? `(?${flags})${pattern}` : pattern };
|
|
74
72
|
}
|
|
75
73
|
};
|
|
74
|
+
|
|
75
|
+
// src/vector/sql-builder.ts
|
|
76
76
|
var createBasicOperator = (symbol) => {
|
|
77
77
|
return (key, paramIndex) => ({
|
|
78
78
|
sql: `CASE
|
|
@@ -264,7 +264,7 @@ function buildFilterQuery(filter, minScore) {
|
|
|
264
264
|
}
|
|
265
265
|
const joinOperator = key === "$or" || key === "$nor" ? "OR" : "AND";
|
|
266
266
|
const conditions2 = value.map((f) => {
|
|
267
|
-
const entries = Object.entries(f);
|
|
267
|
+
const entries = Object.entries(f || {});
|
|
268
268
|
if (entries.length === 0) return "";
|
|
269
269
|
const [firstKey, firstValue] = entries[0] || [];
|
|
270
270
|
if (["$and", "$or", "$not", "$nor"].includes(firstKey)) {
|
|
@@ -307,9 +307,8 @@ var PgVector = class extends MastraVector {
|
|
|
307
307
|
}) ?? basePool;
|
|
308
308
|
}
|
|
309
309
|
transformFilter(filter) {
|
|
310
|
-
const
|
|
311
|
-
|
|
312
|
-
return translatedFilter;
|
|
310
|
+
const translator = new PGFilterTranslator();
|
|
311
|
+
return translator.translate(filter);
|
|
313
312
|
}
|
|
314
313
|
async getIndexInfo(indexName) {
|
|
315
314
|
if (!this.indexCache.has(indexName)) {
|
|
@@ -317,7 +316,9 @@ var PgVector = class extends MastraVector {
|
|
|
317
316
|
}
|
|
318
317
|
return this.indexCache.get(indexName);
|
|
319
318
|
}
|
|
320
|
-
async query(
|
|
319
|
+
async query(...args) {
|
|
320
|
+
const params = this.normalizeArgs("query", args, ["minScore", "ef", "probes"]);
|
|
321
|
+
const { indexName, queryVector, topK = 10, filter, includeVector = false, minScore = 0, ef, probes } = params;
|
|
321
322
|
const client = await this.pool.connect();
|
|
322
323
|
try {
|
|
323
324
|
const vectorStr = `[${queryVector.join(",")}]`;
|
|
@@ -325,12 +326,12 @@ var PgVector = class extends MastraVector {
|
|
|
325
326
|
const { sql: filterQuery, values: filterValues } = buildFilterQuery(translatedFilter, minScore);
|
|
326
327
|
const indexInfo = await this.getIndexInfo(indexName);
|
|
327
328
|
if (indexInfo.type === "hnsw") {
|
|
328
|
-
const calculatedEf =
|
|
329
|
+
const calculatedEf = ef ?? Math.max(topK, (indexInfo?.config?.m ?? 16) * topK);
|
|
329
330
|
const searchEf = Math.min(1e3, Math.max(1, calculatedEf));
|
|
330
331
|
await client.query(`SET LOCAL hnsw.ef_search = ${searchEf}`);
|
|
331
332
|
}
|
|
332
|
-
if (indexInfo.type === "ivfflat" &&
|
|
333
|
-
await client.query(`SET LOCAL ivfflat.probes = ${
|
|
333
|
+
if (indexInfo.type === "ivfflat" && probes) {
|
|
334
|
+
await client.query(`SET LOCAL ivfflat.probes = ${probes}`);
|
|
334
335
|
}
|
|
335
336
|
const query = `
|
|
336
337
|
WITH vector_scores AS (
|
|
@@ -358,7 +359,9 @@ var PgVector = class extends MastraVector {
|
|
|
358
359
|
client.release();
|
|
359
360
|
}
|
|
360
361
|
}
|
|
361
|
-
async upsert(
|
|
362
|
+
async upsert(...args) {
|
|
363
|
+
const params = this.normalizeArgs("upsert", args);
|
|
364
|
+
const { indexName, vectors, metadata, ids } = params;
|
|
362
365
|
const client = await this.pool.connect();
|
|
363
366
|
try {
|
|
364
367
|
await client.query("BEGIN");
|
|
@@ -384,7 +387,9 @@ var PgVector = class extends MastraVector {
|
|
|
384
387
|
client.release();
|
|
385
388
|
}
|
|
386
389
|
}
|
|
387
|
-
async createIndex(
|
|
390
|
+
async createIndex(...args) {
|
|
391
|
+
const params = this.normalizeArgs("createIndex", args, ["indexConfig", "buildIndex"]);
|
|
392
|
+
const { indexName, dimension, metric = "cosine", indexConfig = {}, buildIndex = true } = params;
|
|
388
393
|
const client = await this.pool.connect();
|
|
389
394
|
try {
|
|
390
395
|
if (!indexName.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
|
|
@@ -410,8 +415,8 @@ var PgVector = class extends MastraVector {
|
|
|
410
415
|
metadata JSONB DEFAULT '{}'::jsonb
|
|
411
416
|
);
|
|
412
417
|
`);
|
|
413
|
-
if (
|
|
414
|
-
await this.
|
|
418
|
+
if (buildIndex) {
|
|
419
|
+
await this.buildIndex({ indexName, metric, indexConfig });
|
|
415
420
|
}
|
|
416
421
|
} catch (error) {
|
|
417
422
|
console.error("Failed to create vector table:", error);
|
|
@@ -420,7 +425,15 @@ var PgVector = class extends MastraVector {
|
|
|
420
425
|
client.release();
|
|
421
426
|
}
|
|
422
427
|
}
|
|
428
|
+
/**
|
|
429
|
+
* @deprecated This function is deprecated. Use buildIndex instead
|
|
430
|
+
*/
|
|
423
431
|
async defineIndex(indexName, metric = "cosine", indexConfig) {
|
|
432
|
+
return this.buildIndex({ indexName, metric, indexConfig });
|
|
433
|
+
}
|
|
434
|
+
async buildIndex(...args) {
|
|
435
|
+
const params = this.normalizeArgs("buildIndex", args, ["metric", "indexConfig"]);
|
|
436
|
+
const { indexName, metric = "cosine", indexConfig } = params;
|
|
424
437
|
const client = await this.pool.connect();
|
|
425
438
|
try {
|
|
426
439
|
await client.query(`DROP INDEX IF EXISTS ${indexName}_vector_idx`);
|
|
@@ -576,7 +589,7 @@ var PostgresStore = class extends MastraStorage {
|
|
|
576
589
|
}
|
|
577
590
|
);
|
|
578
591
|
}
|
|
579
|
-
getEvalsByAgentName(
|
|
592
|
+
getEvalsByAgentName(_agentName, _type) {
|
|
580
593
|
throw new Error("Method not implemented.");
|
|
581
594
|
}
|
|
582
595
|
async batchInsert({ tableName, records }) {
|
|
@@ -874,11 +887,11 @@ var PostgresStore = class extends MastraStorage {
|
|
|
874
887
|
WITH ordered_messages AS (
|
|
875
888
|
SELECT
|
|
876
889
|
*,
|
|
877
|
-
ROW_NUMBER() OVER (ORDER BY "createdAt") as row_num
|
|
890
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" DESC) as row_num
|
|
878
891
|
FROM "${MastraStorage.TABLE_MESSAGES}"
|
|
879
892
|
WHERE thread_id = $1
|
|
880
893
|
)
|
|
881
|
-
SELECT
|
|
894
|
+
SELECT
|
|
882
895
|
m.id,
|
|
883
896
|
m.content,
|
|
884
897
|
m.role,
|
|
@@ -892,13 +905,13 @@ var PostgresStore = class extends MastraStorage {
|
|
|
892
905
|
WHERE target.id = ANY($2)
|
|
893
906
|
AND (
|
|
894
907
|
-- Get previous messages based on the max withPreviousMessages
|
|
895
|
-
(m.row_num
|
|
908
|
+
(m.row_num <= target.row_num + $3 AND m.row_num > target.row_num)
|
|
896
909
|
OR
|
|
897
910
|
-- Get next messages based on the max withNextMessages
|
|
898
|
-
(m.row_num
|
|
911
|
+
(m.row_num >= target.row_num - $4 AND m.row_num < target.row_num)
|
|
899
912
|
)
|
|
900
913
|
)
|
|
901
|
-
ORDER BY m.
|
|
914
|
+
ORDER BY m."createdAt" DESC
|
|
902
915
|
`,
|
|
903
916
|
[
|
|
904
917
|
threadId,
|
|
@@ -932,7 +945,7 @@ var PostgresStore = class extends MastraStorage {
|
|
|
932
945
|
if (typeof message.content === "string") {
|
|
933
946
|
try {
|
|
934
947
|
message.content = JSON.parse(message.content);
|
|
935
|
-
} catch
|
|
948
|
+
} catch {
|
|
936
949
|
}
|
|
937
950
|
}
|
|
938
951
|
});
|
package/eslint.config.js
ADDED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mastra/pg",
|
|
3
|
-
"version": "0.1.6-alpha.
|
|
3
|
+
"version": "0.1.6-alpha.3",
|
|
4
4
|
"description": "Postgres provider for Mastra - includes both vector and db storage capabilities",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -10,6 +10,10 @@
|
|
|
10
10
|
"import": {
|
|
11
11
|
"types": "./dist/index.d.ts",
|
|
12
12
|
"default": "./dist/index.js"
|
|
13
|
+
},
|
|
14
|
+
"require": {
|
|
15
|
+
"types": "./dist/index.d.cts",
|
|
16
|
+
"default": "./dist/index.cjs"
|
|
13
17
|
}
|
|
14
18
|
},
|
|
15
19
|
"./package.json": "./package.json"
|
|
@@ -17,7 +21,7 @@
|
|
|
17
21
|
"dependencies": {
|
|
18
22
|
"pg": "^8.13.1",
|
|
19
23
|
"pg-promise": "^11.5.4",
|
|
20
|
-
"@mastra/core": "^0.4.3-alpha.
|
|
24
|
+
"@mastra/core": "^0.4.3-alpha.3"
|
|
21
25
|
},
|
|
22
26
|
"devDependencies": {
|
|
23
27
|
"@microsoft/api-extractor": "^7.49.2",
|
|
@@ -25,10 +29,12 @@
|
|
|
25
29
|
"@types/pg": "^8.11.10",
|
|
26
30
|
"tsup": "^8.0.1",
|
|
27
31
|
"typescript": "^5.7.3",
|
|
28
|
-
"vitest": "^3.0.4"
|
|
32
|
+
"vitest": "^3.0.4",
|
|
33
|
+
"eslint": "^9.20.1",
|
|
34
|
+
"@internal/lint": "0.0.0"
|
|
29
35
|
},
|
|
30
36
|
"scripts": {
|
|
31
|
-
"build": "tsup src/index.ts --format esm --experimental-dts --clean --treeshake",
|
|
37
|
+
"build": "tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake",
|
|
32
38
|
"build:watch": "pnpm build --watch",
|
|
33
39
|
"pretest": "docker compose up -d && (for i in $(seq 1 30); do docker compose exec -T db pg_isready -U postgres && break || (sleep 1; [ $i -eq 30 ] && exit 1); done)",
|
|
34
40
|
"test": "vitest run",
|
|
@@ -38,6 +44,7 @@
|
|
|
38
44
|
"posttest:perf": "docker compose -f docker-compose.perf.yaml down -v",
|
|
39
45
|
"pretest:watch": "docker compose up -d",
|
|
40
46
|
"test:watch": "vitest watch",
|
|
41
|
-
"posttest:watch": "docker compose down -v"
|
|
47
|
+
"posttest:watch": "docker compose down -v",
|
|
48
|
+
"lint": "eslint ."
|
|
42
49
|
}
|
|
43
50
|
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { WorkflowRunState } from '@mastra/core/workflows';
|
|
2
1
|
import { randomUUID } from 'crypto';
|
|
2
|
+
import type { WorkflowRunState } from '@mastra/core/workflows';
|
|
3
3
|
import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest';
|
|
4
4
|
|
|
5
|
-
import { PostgresStore
|
|
5
|
+
import { PostgresStore } from '.';
|
|
6
|
+
import type { PostgresConfig } from '.';
|
|
6
7
|
|
|
7
8
|
const TEST_CONFIG: PostgresConfig = {
|
|
8
9
|
host: process.env.POSTGRES_HOST || 'localhost',
|
package/src/storage/index.ts
CHANGED
|
@@ -1,22 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
type StorageColumn,
|
|
6
|
-
type StorageGetMessagesArg,
|
|
7
|
-
type TABLE_NAMES,
|
|
8
|
-
} from '@mastra/core/storage';
|
|
9
|
-
import { type WorkflowRunState } from '@mastra/core/workflows';
|
|
1
|
+
import type { MessageType, StorageThreadType } from '@mastra/core/memory';
|
|
2
|
+
import { MastraStorage } from '@mastra/core/storage';
|
|
3
|
+
import type { EvalRow, StorageColumn, StorageGetMessagesArg, TABLE_NAMES } from '@mastra/core/storage';
|
|
4
|
+
import type { WorkflowRunState } from '@mastra/core/workflows';
|
|
10
5
|
import pgPromise from 'pg-promise';
|
|
11
6
|
|
|
12
|
-
function safelyParseJSON(json: string): any {
|
|
13
|
-
try {
|
|
14
|
-
return JSON.parse(json);
|
|
15
|
-
} catch (e) {
|
|
16
|
-
return {};
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
|
|
20
7
|
export type PostgresConfig =
|
|
21
8
|
| {
|
|
22
9
|
host: string;
|
|
@@ -49,7 +36,7 @@ export class PostgresStore extends MastraStorage {
|
|
|
49
36
|
);
|
|
50
37
|
}
|
|
51
38
|
|
|
52
|
-
getEvalsByAgentName(
|
|
39
|
+
getEvalsByAgentName(_agentName: string, _type?: 'test' | 'live'): Promise<EvalRow[]> {
|
|
53
40
|
throw new Error('Method not implemented.');
|
|
54
41
|
}
|
|
55
42
|
|
|
@@ -424,11 +411,11 @@ export class PostgresStore extends MastraStorage {
|
|
|
424
411
|
WITH ordered_messages AS (
|
|
425
412
|
SELECT
|
|
426
413
|
*,
|
|
427
|
-
ROW_NUMBER() OVER (ORDER BY "createdAt") as row_num
|
|
414
|
+
ROW_NUMBER() OVER (ORDER BY "createdAt" DESC) as row_num
|
|
428
415
|
FROM "${MastraStorage.TABLE_MESSAGES}"
|
|
429
416
|
WHERE thread_id = $1
|
|
430
417
|
)
|
|
431
|
-
SELECT
|
|
418
|
+
SELECT
|
|
432
419
|
m.id,
|
|
433
420
|
m.content,
|
|
434
421
|
m.role,
|
|
@@ -442,13 +429,13 @@ export class PostgresStore extends MastraStorage {
|
|
|
442
429
|
WHERE target.id = ANY($2)
|
|
443
430
|
AND (
|
|
444
431
|
-- Get previous messages based on the max withPreviousMessages
|
|
445
|
-
(m.row_num
|
|
432
|
+
(m.row_num <= target.row_num + $3 AND m.row_num > target.row_num)
|
|
446
433
|
OR
|
|
447
434
|
-- Get next messages based on the max withNextMessages
|
|
448
|
-
(m.row_num
|
|
435
|
+
(m.row_num >= target.row_num - $4 AND m.row_num < target.row_num)
|
|
449
436
|
)
|
|
450
437
|
)
|
|
451
|
-
ORDER BY m.
|
|
438
|
+
ORDER BY m."createdAt" DESC
|
|
452
439
|
`,
|
|
453
440
|
[
|
|
454
441
|
threadId,
|
|
@@ -490,7 +477,7 @@ export class PostgresStore extends MastraStorage {
|
|
|
490
477
|
if (typeof message.content === 'string') {
|
|
491
478
|
try {
|
|
492
479
|
message.content = JSON.parse(message.content);
|
|
493
|
-
} catch
|
|
480
|
+
} catch {
|
|
494
481
|
// If parsing fails, leave as string
|
|
495
482
|
}
|
|
496
483
|
}
|
package/src/vector/filter.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { BaseFilterTranslator
|
|
1
|
+
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
2
|
+
import type { FieldCondition, VectorFilter, OperatorSupport } from '@mastra/core/vector/filter';
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
5
|
* Translates MongoDB-style filters to PG compatible filters.
|
|
@@ -18,7 +19,7 @@ export class PGFilterTranslator extends BaseFilterTranslator {
|
|
|
18
19
|
};
|
|
19
20
|
}
|
|
20
21
|
|
|
21
|
-
translate(filter
|
|
22
|
+
translate(filter?: VectorFilter): VectorFilter {
|
|
22
23
|
if (this.isEmpty(filter)) {
|
|
23
24
|
return filter;
|
|
24
25
|
}
|
|
@@ -26,7 +27,7 @@ export class PGFilterTranslator extends BaseFilterTranslator {
|
|
|
26
27
|
return this.translateNode(filter);
|
|
27
28
|
}
|
|
28
29
|
|
|
29
|
-
private translateNode(node:
|
|
30
|
+
private translateNode(node: VectorFilter | FieldCondition, currentPath: string = ''): any {
|
|
30
31
|
// Helper to wrap result with path if needed
|
|
31
32
|
const withPath = (result: any) => (currentPath ? { [currentPath]: result } : result);
|
|
32
33
|
|
|
@@ -67,7 +68,7 @@ export class PGFilterTranslator extends BaseFilterTranslator {
|
|
|
67
68
|
|
|
68
69
|
if (this.isLogicalOperator(key)) {
|
|
69
70
|
result[key] = Array.isArray(value)
|
|
70
|
-
? value.map((filter:
|
|
71
|
+
? value.map((filter: VectorFilter) => this.translateNode(filter))
|
|
71
72
|
: this.translateNode(value);
|
|
72
73
|
} else if (this.isOperator(key)) {
|
|
73
74
|
if (this.isArrayOperator(key) && !Array.isArray(value) && key !== '$elemMatch') {
|