@mastra/lance 0.1.3-alpha.1 → 0.1.3-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +12 -0
- package/dist/_tsup-dts-rollup.d.cts +18 -4
- package/dist/_tsup-dts-rollup.d.ts +18 -4
- package/dist/index.cjs +16 -7
- package/dist/index.js +16 -7
- package/package.json +4 -4
- package/src/storage/index.test.ts +71 -1
- package/src/storage/index.ts +25 -7
- package/src/vector/filter.test.ts +3 -3
- package/src/vector/filter.ts +24 -4
- package/src/vector/index.ts +4 -4
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
|
|
2
|
-
> @mastra/lance@0.1.3-alpha.
|
|
2
|
+
> @mastra/lance@0.1.3-alpha.2 build /home/runner/work/mastra/mastra/stores/lance
|
|
3
3
|
> tsup src/index.ts --format esm,cjs --experimental-dts --clean --treeshake=smallest --splitting
|
|
4
4
|
|
|
5
5
|
[34mCLI[39m Building entry: src/index.ts
|
|
6
6
|
[34mCLI[39m Using tsconfig: tsconfig.json
|
|
7
7
|
[34mCLI[39m tsup v8.5.0
|
|
8
8
|
[34mTSC[39m Build start
|
|
9
|
-
[32mTSC[39m ⚡️ Build success in
|
|
9
|
+
[32mTSC[39m ⚡️ Build success in 10342ms
|
|
10
10
|
[34mDTS[39m Build start
|
|
11
11
|
[34mCLI[39m Target: es2022
|
|
12
12
|
Analysis will use the bundled TypeScript version 5.8.3
|
|
13
13
|
[36mWriting package typings: /home/runner/work/mastra/mastra/stores/lance/dist/_tsup-dts-rollup.d.ts[39m
|
|
14
14
|
Analysis will use the bundled TypeScript version 5.8.3
|
|
15
15
|
[36mWriting package typings: /home/runner/work/mastra/mastra/stores/lance/dist/_tsup-dts-rollup.d.cts[39m
|
|
16
|
-
[32mDTS[39m ⚡️ Build success in
|
|
16
|
+
[32mDTS[39m ⚡️ Build success in 13286ms
|
|
17
17
|
[34mCLI[39m Cleaning output folder
|
|
18
18
|
[34mESM[39m Build start
|
|
19
19
|
[34mCJS[39m Build start
|
|
20
|
-
[
|
|
21
|
-
[
|
|
22
|
-
[
|
|
23
|
-
[
|
|
20
|
+
[32mCJS[39m [1mdist/index.cjs [22m[32m72.48 KB[39m
|
|
21
|
+
[32mCJS[39m ⚡️ Build success in 1677ms
|
|
22
|
+
[32mESM[39m [1mdist/index.js [22m[32m70.94 KB[39m
|
|
23
|
+
[32mESM[39m ⚡️ Build success in 1693ms
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# @mastra/lance
|
|
2
2
|
|
|
3
|
+
## 0.1.3-alpha.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 0fb9d64: [MASTRA-4018] Update saveMessages in storage adapters to upsert messages
|
|
8
|
+
- 144eb0b: [MASTRA-3669] Metadata Filter Types
|
|
9
|
+
- Updated dependencies [15e9d26]
|
|
10
|
+
- Updated dependencies [07d6d88]
|
|
11
|
+
- Updated dependencies [5d74aab]
|
|
12
|
+
- Updated dependencies [144eb0b]
|
|
13
|
+
- @mastra/core@0.10.7-alpha.2
|
|
14
|
+
|
|
3
15
|
## 0.1.3-alpha.1
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ArrayOperator } from '@mastra/core/vector/filter';
|
|
2
2
|
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
3
3
|
import { BasicOperator } from '@mastra/core/vector/filter';
|
|
4
|
+
import type { BlacklistedRootOperators } from '@mastra/core/vector/filter';
|
|
4
5
|
import type { ConnectionOptions } from '@lancedb/lancedb';
|
|
5
6
|
import type { CreateIndexParams } from '@mastra/core';
|
|
6
7
|
import type { CreateTableOptions } from '@lancedb/lancedb';
|
|
@@ -10,12 +11,14 @@ import type { DescribeIndexParams } from '@mastra/core';
|
|
|
10
11
|
import type { EvalRow } from '@mastra/core/storage';
|
|
11
12
|
import type { IndexStats } from '@mastra/core';
|
|
12
13
|
import { LogicalOperator } from '@mastra/core/vector/filter';
|
|
14
|
+
import type { LogicalOperatorValueMap } from '@mastra/core/vector/filter';
|
|
13
15
|
import type { MastraMessageContentV2 } from '@mastra/core/agent';
|
|
14
16
|
import type { MastraMessageV1 } from '@mastra/core/memory';
|
|
15
17
|
import type { MastraMessageV2 } from '@mastra/core/memory';
|
|
16
18
|
import { MastraStorage } from '@mastra/core/storage';
|
|
17
19
|
import { MastraVector } from '@mastra/core/vector';
|
|
18
20
|
import { NumericOperator } from '@mastra/core/vector/filter';
|
|
21
|
+
import type { OperatorValueMap } from '@mastra/core/vector/filter';
|
|
19
22
|
import type { PaginationInfo } from '@mastra/core/storage';
|
|
20
23
|
import type { QueryResult } from '@mastra/core';
|
|
21
24
|
import type { QueryVectorParams } from '@mastra/core';
|
|
@@ -53,13 +56,15 @@ declare interface IVFConfig {
|
|
|
53
56
|
lists?: number;
|
|
54
57
|
}
|
|
55
58
|
|
|
59
|
+
declare type LanceBlacklisted = BlacklistedRootOperators | '$like' | '$notLike' | '$contains';
|
|
60
|
+
|
|
56
61
|
declare interface LanceCreateIndexParams extends CreateIndexParams {
|
|
57
62
|
indexConfig?: LanceIndexConfig;
|
|
58
63
|
tableName?: string;
|
|
59
64
|
}
|
|
60
65
|
|
|
61
|
-
export declare class LanceFilterTranslator extends BaseFilterTranslator {
|
|
62
|
-
translate(filter:
|
|
66
|
+
export declare class LanceFilterTranslator extends BaseFilterTranslator<LanceVectorFilter, string> {
|
|
67
|
+
translate(filter: LanceVectorFilter): string;
|
|
63
68
|
private processFilter;
|
|
64
69
|
private processLogicalOperator;
|
|
65
70
|
private processNestedObject;
|
|
@@ -94,7 +99,13 @@ declare interface LanceIndexConfig extends IndexConfig {
|
|
|
94
99
|
numSubVectors?: number;
|
|
95
100
|
}
|
|
96
101
|
|
|
97
|
-
declare
|
|
102
|
+
declare type LanceOperatorValueMap = OperatorValueMap & {
|
|
103
|
+
$like: string;
|
|
104
|
+
$notLike: string;
|
|
105
|
+
$contains: string;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
declare interface LanceQueryVectorParams extends QueryVectorParams<LanceVectorFilter> {
|
|
98
109
|
tableName: string;
|
|
99
110
|
columns?: string[];
|
|
100
111
|
includeAllColumns?: boolean;
|
|
@@ -125,6 +136,7 @@ declare class LanceStorage extends MastraStorage {
|
|
|
125
136
|
* ```
|
|
126
137
|
*/
|
|
127
138
|
static create(name: string, uri: string, options?: ConnectionOptions): Promise<LanceStorage>;
|
|
139
|
+
private getPrimaryKeys;
|
|
128
140
|
/**
|
|
129
141
|
* @internal
|
|
130
142
|
* Private constructor to enforce using the create factory method
|
|
@@ -327,7 +339,9 @@ declare interface LanceUpsertVectorParams extends UpsertVectorParams {
|
|
|
327
339
|
tableName: string;
|
|
328
340
|
}
|
|
329
341
|
|
|
330
|
-
declare
|
|
342
|
+
export declare type LanceVectorFilter = VectorFilter<keyof LanceOperatorValueMap, LanceOperatorValueMap, LogicalOperatorValueMap, LanceBlacklisted>;
|
|
343
|
+
|
|
344
|
+
declare class LanceVectorStore extends MastraVector<LanceVectorFilter> {
|
|
331
345
|
private lanceClient;
|
|
332
346
|
/**
|
|
333
347
|
* Creates a new instance of LanceVectorStore
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ArrayOperator } from '@mastra/core/vector/filter';
|
|
2
2
|
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
3
3
|
import { BasicOperator } from '@mastra/core/vector/filter';
|
|
4
|
+
import type { BlacklistedRootOperators } from '@mastra/core/vector/filter';
|
|
4
5
|
import type { ConnectionOptions } from '@lancedb/lancedb';
|
|
5
6
|
import type { CreateIndexParams } from '@mastra/core';
|
|
6
7
|
import type { CreateTableOptions } from '@lancedb/lancedb';
|
|
@@ -10,12 +11,14 @@ import type { DescribeIndexParams } from '@mastra/core';
|
|
|
10
11
|
import type { EvalRow } from '@mastra/core/storage';
|
|
11
12
|
import type { IndexStats } from '@mastra/core';
|
|
12
13
|
import { LogicalOperator } from '@mastra/core/vector/filter';
|
|
14
|
+
import type { LogicalOperatorValueMap } from '@mastra/core/vector/filter';
|
|
13
15
|
import type { MastraMessageContentV2 } from '@mastra/core/agent';
|
|
14
16
|
import type { MastraMessageV1 } from '@mastra/core/memory';
|
|
15
17
|
import type { MastraMessageV2 } from '@mastra/core/memory';
|
|
16
18
|
import { MastraStorage } from '@mastra/core/storage';
|
|
17
19
|
import { MastraVector } from '@mastra/core/vector';
|
|
18
20
|
import { NumericOperator } from '@mastra/core/vector/filter';
|
|
21
|
+
import type { OperatorValueMap } from '@mastra/core/vector/filter';
|
|
19
22
|
import type { PaginationInfo } from '@mastra/core/storage';
|
|
20
23
|
import type { QueryResult } from '@mastra/core';
|
|
21
24
|
import type { QueryVectorParams } from '@mastra/core';
|
|
@@ -53,13 +56,15 @@ declare interface IVFConfig {
|
|
|
53
56
|
lists?: number;
|
|
54
57
|
}
|
|
55
58
|
|
|
59
|
+
declare type LanceBlacklisted = BlacklistedRootOperators | '$like' | '$notLike' | '$contains';
|
|
60
|
+
|
|
56
61
|
declare interface LanceCreateIndexParams extends CreateIndexParams {
|
|
57
62
|
indexConfig?: LanceIndexConfig;
|
|
58
63
|
tableName?: string;
|
|
59
64
|
}
|
|
60
65
|
|
|
61
|
-
export declare class LanceFilterTranslator extends BaseFilterTranslator {
|
|
62
|
-
translate(filter:
|
|
66
|
+
export declare class LanceFilterTranslator extends BaseFilterTranslator<LanceVectorFilter, string> {
|
|
67
|
+
translate(filter: LanceVectorFilter): string;
|
|
63
68
|
private processFilter;
|
|
64
69
|
private processLogicalOperator;
|
|
65
70
|
private processNestedObject;
|
|
@@ -94,7 +99,13 @@ declare interface LanceIndexConfig extends IndexConfig {
|
|
|
94
99
|
numSubVectors?: number;
|
|
95
100
|
}
|
|
96
101
|
|
|
97
|
-
declare
|
|
102
|
+
declare type LanceOperatorValueMap = OperatorValueMap & {
|
|
103
|
+
$like: string;
|
|
104
|
+
$notLike: string;
|
|
105
|
+
$contains: string;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
declare interface LanceQueryVectorParams extends QueryVectorParams<LanceVectorFilter> {
|
|
98
109
|
tableName: string;
|
|
99
110
|
columns?: string[];
|
|
100
111
|
includeAllColumns?: boolean;
|
|
@@ -125,6 +136,7 @@ declare class LanceStorage extends MastraStorage {
|
|
|
125
136
|
* ```
|
|
126
137
|
*/
|
|
127
138
|
static create(name: string, uri: string, options?: ConnectionOptions): Promise<LanceStorage>;
|
|
139
|
+
private getPrimaryKeys;
|
|
128
140
|
/**
|
|
129
141
|
* @internal
|
|
130
142
|
* Private constructor to enforce using the create factory method
|
|
@@ -327,7 +339,9 @@ declare interface LanceUpsertVectorParams extends UpsertVectorParams {
|
|
|
327
339
|
tableName: string;
|
|
328
340
|
}
|
|
329
341
|
|
|
330
|
-
declare
|
|
342
|
+
export declare type LanceVectorFilter = VectorFilter<keyof LanceOperatorValueMap, LanceOperatorValueMap, LogicalOperatorValueMap, LanceBlacklisted>;
|
|
343
|
+
|
|
344
|
+
declare class LanceVectorStore extends MastraVector<LanceVectorFilter> {
|
|
331
345
|
private lanceClient;
|
|
332
346
|
/**
|
|
333
347
|
* Creates a new instance of LanceVectorStore
|
package/dist/index.cjs
CHANGED
|
@@ -51,6 +51,15 @@ var LanceStorage = class _LanceStorage extends storage.MastraStorage {
|
|
|
51
51
|
);
|
|
52
52
|
}
|
|
53
53
|
}
|
|
54
|
+
getPrimaryKeys(tableName) {
|
|
55
|
+
let primaryId = ["id"];
|
|
56
|
+
if (tableName === storage.TABLE_WORKFLOW_SNAPSHOT) {
|
|
57
|
+
primaryId = ["workflow_name", "run_id"];
|
|
58
|
+
} else if (tableName === storage.TABLE_EVALS) {
|
|
59
|
+
primaryId = ["agent_name", "metric_name", "run_id"];
|
|
60
|
+
}
|
|
61
|
+
return primaryId;
|
|
62
|
+
}
|
|
54
63
|
/**
|
|
55
64
|
* @internal
|
|
56
65
|
* Private constructor to enforce using the create factory method
|
|
@@ -377,6 +386,7 @@ var LanceStorage = class _LanceStorage extends storage.MastraStorage {
|
|
|
377
386
|
}
|
|
378
387
|
try {
|
|
379
388
|
const table = await this.lanceClient.openTable(tableName);
|
|
389
|
+
const primaryId = this.getPrimaryKeys(tableName);
|
|
380
390
|
const processedRecord = { ...record };
|
|
381
391
|
for (const key in processedRecord) {
|
|
382
392
|
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
@@ -384,7 +394,7 @@ var LanceStorage = class _LanceStorage extends storage.MastraStorage {
|
|
|
384
394
|
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
385
395
|
}
|
|
386
396
|
}
|
|
387
|
-
await table.
|
|
397
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([processedRecord]);
|
|
388
398
|
} catch (error$1) {
|
|
389
399
|
throw new error.MastraError(
|
|
390
400
|
{
|
|
@@ -427,6 +437,7 @@ var LanceStorage = class _LanceStorage extends storage.MastraStorage {
|
|
|
427
437
|
}
|
|
428
438
|
try {
|
|
429
439
|
const table = await this.lanceClient.openTable(tableName);
|
|
440
|
+
const primaryId = this.getPrimaryKeys(tableName);
|
|
430
441
|
const processedRecords = records.map((record) => {
|
|
431
442
|
const processedRecord = { ...record };
|
|
432
443
|
for (const key in processedRecord) {
|
|
@@ -437,7 +448,7 @@ var LanceStorage = class _LanceStorage extends storage.MastraStorage {
|
|
|
437
448
|
}
|
|
438
449
|
return processedRecord;
|
|
439
450
|
});
|
|
440
|
-
await table.
|
|
451
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(processedRecords);
|
|
441
452
|
} catch (error$1) {
|
|
442
453
|
throw new error.MastraError(
|
|
443
454
|
{
|
|
@@ -651,7 +662,7 @@ var LanceStorage = class _LanceStorage extends storage.MastraStorage {
|
|
|
651
662
|
try {
|
|
652
663
|
const record = { id, title, metadata: JSON.stringify(metadata) };
|
|
653
664
|
const table = await this.lanceClient.openTable(storage.TABLE_THREADS);
|
|
654
|
-
await table.
|
|
665
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
655
666
|
const query = table.query().where(`id = '${id}'`);
|
|
656
667
|
const records = await query.toArray();
|
|
657
668
|
return this.processResultWithTypeConversion(
|
|
@@ -805,7 +816,7 @@ var LanceStorage = class _LanceStorage extends storage.MastraStorage {
|
|
|
805
816
|
content: JSON.stringify(message.content)
|
|
806
817
|
}));
|
|
807
818
|
const table = await this.lanceClient.openTable(storage.TABLE_MESSAGES);
|
|
808
|
-
await table.
|
|
819
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(transformedMessages);
|
|
809
820
|
const list = new agent.MessageList().add(messages, "memory");
|
|
810
821
|
if (format === `v2`) return list.get.all.v2();
|
|
811
822
|
return list.get.all.v1();
|
|
@@ -1068,10 +1079,8 @@ var LanceStorage = class _LanceStorage extends storage.MastraStorage {
|
|
|
1068
1079
|
const records = await query.toArray();
|
|
1069
1080
|
let createdAt;
|
|
1070
1081
|
const now = Date.now();
|
|
1071
|
-
let mode = "append";
|
|
1072
1082
|
if (records.length > 0) {
|
|
1073
1083
|
createdAt = records[0].createdAt ?? now;
|
|
1074
|
-
mode = "overwrite";
|
|
1075
1084
|
} else {
|
|
1076
1085
|
createdAt = now;
|
|
1077
1086
|
}
|
|
@@ -1082,7 +1091,7 @@ var LanceStorage = class _LanceStorage extends storage.MastraStorage {
|
|
|
1082
1091
|
createdAt,
|
|
1083
1092
|
updatedAt: now
|
|
1084
1093
|
};
|
|
1085
|
-
await table.
|
|
1094
|
+
await table.mergeInsert(["workflow_name", "run_id"]).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
1086
1095
|
} catch (error$1) {
|
|
1087
1096
|
throw new error.MastraError(
|
|
1088
1097
|
{
|
package/dist/index.js
CHANGED
|
@@ -49,6 +49,15 @@ var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
|
49
49
|
);
|
|
50
50
|
}
|
|
51
51
|
}
|
|
52
|
+
getPrimaryKeys(tableName) {
|
|
53
|
+
let primaryId = ["id"];
|
|
54
|
+
if (tableName === TABLE_WORKFLOW_SNAPSHOT) {
|
|
55
|
+
primaryId = ["workflow_name", "run_id"];
|
|
56
|
+
} else if (tableName === TABLE_EVALS) {
|
|
57
|
+
primaryId = ["agent_name", "metric_name", "run_id"];
|
|
58
|
+
}
|
|
59
|
+
return primaryId;
|
|
60
|
+
}
|
|
52
61
|
/**
|
|
53
62
|
* @internal
|
|
54
63
|
* Private constructor to enforce using the create factory method
|
|
@@ -375,6 +384,7 @@ var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
|
375
384
|
}
|
|
376
385
|
try {
|
|
377
386
|
const table = await this.lanceClient.openTable(tableName);
|
|
387
|
+
const primaryId = this.getPrimaryKeys(tableName);
|
|
378
388
|
const processedRecord = { ...record };
|
|
379
389
|
for (const key in processedRecord) {
|
|
380
390
|
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
@@ -382,7 +392,7 @@ var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
|
382
392
|
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
383
393
|
}
|
|
384
394
|
}
|
|
385
|
-
await table.
|
|
395
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([processedRecord]);
|
|
386
396
|
} catch (error) {
|
|
387
397
|
throw new MastraError(
|
|
388
398
|
{
|
|
@@ -425,6 +435,7 @@ var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
|
425
435
|
}
|
|
426
436
|
try {
|
|
427
437
|
const table = await this.lanceClient.openTable(tableName);
|
|
438
|
+
const primaryId = this.getPrimaryKeys(tableName);
|
|
428
439
|
const processedRecords = records.map((record) => {
|
|
429
440
|
const processedRecord = { ...record };
|
|
430
441
|
for (const key in processedRecord) {
|
|
@@ -435,7 +446,7 @@ var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
|
435
446
|
}
|
|
436
447
|
return processedRecord;
|
|
437
448
|
});
|
|
438
|
-
await table.
|
|
449
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(processedRecords);
|
|
439
450
|
} catch (error) {
|
|
440
451
|
throw new MastraError(
|
|
441
452
|
{
|
|
@@ -649,7 +660,7 @@ var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
|
649
660
|
try {
|
|
650
661
|
const record = { id, title, metadata: JSON.stringify(metadata) };
|
|
651
662
|
const table = await this.lanceClient.openTable(TABLE_THREADS);
|
|
652
|
-
await table.
|
|
663
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
653
664
|
const query = table.query().where(`id = '${id}'`);
|
|
654
665
|
const records = await query.toArray();
|
|
655
666
|
return this.processResultWithTypeConversion(
|
|
@@ -803,7 +814,7 @@ var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
|
803
814
|
content: JSON.stringify(message.content)
|
|
804
815
|
}));
|
|
805
816
|
const table = await this.lanceClient.openTable(TABLE_MESSAGES);
|
|
806
|
-
await table.
|
|
817
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(transformedMessages);
|
|
807
818
|
const list = new MessageList().add(messages, "memory");
|
|
808
819
|
if (format === `v2`) return list.get.all.v2();
|
|
809
820
|
return list.get.all.v1();
|
|
@@ -1066,10 +1077,8 @@ var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
|
1066
1077
|
const records = await query.toArray();
|
|
1067
1078
|
let createdAt;
|
|
1068
1079
|
const now = Date.now();
|
|
1069
|
-
let mode = "append";
|
|
1070
1080
|
if (records.length > 0) {
|
|
1071
1081
|
createdAt = records[0].createdAt ?? now;
|
|
1072
|
-
mode = "overwrite";
|
|
1073
1082
|
} else {
|
|
1074
1083
|
createdAt = now;
|
|
1075
1084
|
}
|
|
@@ -1080,7 +1089,7 @@ var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
|
1080
1089
|
createdAt,
|
|
1081
1090
|
updatedAt: now
|
|
1082
1091
|
};
|
|
1083
|
-
await table.
|
|
1092
|
+
await table.mergeInsert(["workflow_name", "run_id"]).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
1084
1093
|
} catch (error) {
|
|
1085
1094
|
throw new MastraError(
|
|
1086
1095
|
{
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mastra/lance",
|
|
3
|
-
"version": "0.1.3-alpha.
|
|
3
|
+
"version": "0.1.3-alpha.2",
|
|
4
4
|
"description": "Lance provider for Mastra - includes both vector and db storage capabilities",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -25,13 +25,13 @@
|
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"@microsoft/api-extractor": "^7.52.8",
|
|
27
27
|
"@types/node": "^20.19.0",
|
|
28
|
-
"eslint": "^9.
|
|
28
|
+
"eslint": "^9.29.0",
|
|
29
29
|
"tsup": "^8.5.0",
|
|
30
30
|
"typescript": "^5.8.3",
|
|
31
31
|
"vitest": "^3.2.3",
|
|
32
32
|
"@internal/lint": "0.0.13",
|
|
33
|
-
"@
|
|
34
|
-
"@
|
|
33
|
+
"@mastra/core": "^0.10.7-alpha.2",
|
|
34
|
+
"@internal/storage-test-utils": "0.0.9"
|
|
35
35
|
},
|
|
36
36
|
"peerDependencies": {
|
|
37
37
|
"@mastra/core": ">=0.10.4-0 <0.11.0"
|
|
@@ -54,6 +54,7 @@ function generateMessageRecords(count: number, threadId?: string): MastraMessage
|
|
|
54
54
|
toolCallIds: [],
|
|
55
55
|
toolCallArgs: [],
|
|
56
56
|
toolNames: [],
|
|
57
|
+
type: 'v2',
|
|
57
58
|
}));
|
|
58
59
|
}
|
|
59
60
|
|
|
@@ -563,7 +564,6 @@ describe('LanceStorage tests', async () => {
|
|
|
563
564
|
expect(message.threadId).toEqual(threadId);
|
|
564
565
|
expect(message.id.toString()).toEqual(messages[index].id);
|
|
565
566
|
expect(message.content).toEqual(messages[index].content);
|
|
566
|
-
expect(new Date(message.createdAt)).toEqual(new Date(messages[index].createdAt));
|
|
567
567
|
expect(message.role).toEqual(messages[index].role);
|
|
568
568
|
expect(message.resourceId).toEqual(messages[index].resourceId);
|
|
569
569
|
expect(message.type).toEqual(messages[index].type);
|
|
@@ -708,6 +708,76 @@ describe('LanceStorage tests', async () => {
|
|
|
708
708
|
expect(currentDate).toBeLessThanOrEqual(nextDate);
|
|
709
709
|
}
|
|
710
710
|
});
|
|
711
|
+
|
|
712
|
+
it('should upsert messages: duplicate id+threadId results in update, not duplicate row', async () => {
|
|
713
|
+
const thread = 'thread-1';
|
|
714
|
+
const baseMessage = generateMessageRecords(1, thread)[0];
|
|
715
|
+
|
|
716
|
+
// Insert the message for the first time
|
|
717
|
+
await storage.saveMessages({ messages: [baseMessage], format: 'v2' });
|
|
718
|
+
|
|
719
|
+
// Insert again with the same id and threadId but different content
|
|
720
|
+
const updatedMessage: MastraMessageV2 = {
|
|
721
|
+
...generateMessageRecords(1, thread)[0],
|
|
722
|
+
id: baseMessage.id,
|
|
723
|
+
content: { format: 2, parts: [{ type: 'text', text: 'Updated' }] },
|
|
724
|
+
};
|
|
725
|
+
|
|
726
|
+
await storage.saveMessages({ messages: [updatedMessage], format: 'v2' });
|
|
727
|
+
|
|
728
|
+
// Retrieve messages for the thread
|
|
729
|
+
const retrievedMessages = await storage.getMessages({ threadId: thread, format: 'v2' });
|
|
730
|
+
// Only one message should exist for that id+threadId
|
|
731
|
+
expect(retrievedMessages.filter(m => m.id.toString() === baseMessage.id)).toHaveLength(1);
|
|
732
|
+
|
|
733
|
+
// The content should be the updated one
|
|
734
|
+
expect(retrievedMessages.find(m => m.id.toString() === baseMessage.id)?.content.parts[0].text).toBe('Updated');
|
|
735
|
+
});
|
|
736
|
+
|
|
737
|
+
it('should upsert messages: duplicate id and different threadid', async () => {
|
|
738
|
+
const thread1 = 'thread-1';
|
|
739
|
+
const thread2 = 'thread-2';
|
|
740
|
+
const thread3 = 'thread-3';
|
|
741
|
+
|
|
742
|
+
const message = generateMessageRecords(1, thread1)[0];
|
|
743
|
+
|
|
744
|
+
// Insert message into thread1
|
|
745
|
+
await storage.saveMessages({ messages: [message], format: 'v2' });
|
|
746
|
+
|
|
747
|
+
// Attempt to insert a message with the same id but different threadId
|
|
748
|
+
const conflictingMessage: MastraMessageV2 = {
|
|
749
|
+
...generateMessageRecords(1, thread2)[0],
|
|
750
|
+
id: message.id,
|
|
751
|
+
content: { format: 2, parts: [{ type: 'text', text: 'Thread2 Content' }] },
|
|
752
|
+
};
|
|
753
|
+
|
|
754
|
+
const differentMessage: MastraMessageV2 = {
|
|
755
|
+
...generateMessageRecords(1, thread3)[0],
|
|
756
|
+
id: '2',
|
|
757
|
+
content: { format: 2, parts: [{ type: 'text', text: 'Another Message Content' }] },
|
|
758
|
+
};
|
|
759
|
+
|
|
760
|
+
// Save should move the message to the new thread
|
|
761
|
+
await storage.saveMessages({ messages: [conflictingMessage], format: 'v2' });
|
|
762
|
+
|
|
763
|
+
await storage.saveMessages({ messages: [differentMessage], format: 'v2' });
|
|
764
|
+
|
|
765
|
+
// Retrieve messages for both threads
|
|
766
|
+
const thread1Messages = await storage.getMessages({ threadId: thread1, format: 'v2' });
|
|
767
|
+
const thread2Messages = await storage.getMessages({ threadId: thread2, format: 'v2' });
|
|
768
|
+
const thread3Messages = await storage.getMessages({ threadId: thread3, format: 'v2' });
|
|
769
|
+
|
|
770
|
+
// Thread 1 should NOT have the message with that id
|
|
771
|
+
expect(thread1Messages.find(m => m.id.toString() === message.id)).toBeUndefined();
|
|
772
|
+
|
|
773
|
+
// Thread 2 should have the message with that id
|
|
774
|
+
expect(thread2Messages.find(m => m.id.toString() === message.id)?.content.parts[0].text).toBe('Thread2 Content');
|
|
775
|
+
|
|
776
|
+
// Thread 2 should have the other message
|
|
777
|
+
expect(thread3Messages.find(m => m.id.toString() === differentMessage.id)?.content.parts[0].text).toBe(
|
|
778
|
+
'Another Message Content',
|
|
779
|
+
);
|
|
780
|
+
});
|
|
711
781
|
});
|
|
712
782
|
|
|
713
783
|
describe('Trace operations', () => {
|
package/src/storage/index.ts
CHANGED
|
@@ -71,6 +71,17 @@ export class LanceStorage extends MastraStorage {
|
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
private getPrimaryKeys(tableName: TABLE_NAMES): string[] {
|
|
75
|
+
let primaryId: string[] = ['id'];
|
|
76
|
+
if (tableName === TABLE_WORKFLOW_SNAPSHOT) {
|
|
77
|
+
primaryId = ['workflow_name', 'run_id'];
|
|
78
|
+
} else if (tableName === TABLE_EVALS) {
|
|
79
|
+
primaryId = ['agent_name', 'metric_name', 'run_id'];
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return primaryId;
|
|
83
|
+
}
|
|
84
|
+
|
|
74
85
|
/**
|
|
75
86
|
* @internal
|
|
76
87
|
* Private constructor to enforce using the create factory method
|
|
@@ -436,6 +447,8 @@ export class LanceStorage extends MastraStorage {
|
|
|
436
447
|
try {
|
|
437
448
|
const table = await this.lanceClient.openTable(tableName);
|
|
438
449
|
|
|
450
|
+
const primaryId = this.getPrimaryKeys(tableName as TABLE_NAMES);
|
|
451
|
+
|
|
439
452
|
const processedRecord = { ...record };
|
|
440
453
|
|
|
441
454
|
for (const key in processedRecord) {
|
|
@@ -449,7 +462,7 @@ export class LanceStorage extends MastraStorage {
|
|
|
449
462
|
}
|
|
450
463
|
}
|
|
451
464
|
|
|
452
|
-
await table.
|
|
465
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([processedRecord]);
|
|
453
466
|
} catch (error: any) {
|
|
454
467
|
throw new MastraError(
|
|
455
468
|
{
|
|
@@ -495,6 +508,8 @@ export class LanceStorage extends MastraStorage {
|
|
|
495
508
|
try {
|
|
496
509
|
const table = await this.lanceClient.openTable(tableName);
|
|
497
510
|
|
|
511
|
+
const primaryId = this.getPrimaryKeys(tableName as TABLE_NAMES);
|
|
512
|
+
|
|
498
513
|
const processedRecords = records.map(record => {
|
|
499
514
|
const processedRecord = { ...record };
|
|
500
515
|
|
|
@@ -515,7 +530,7 @@ export class LanceStorage extends MastraStorage {
|
|
|
515
530
|
return processedRecord;
|
|
516
531
|
});
|
|
517
532
|
|
|
518
|
-
await table.
|
|
533
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(processedRecords);
|
|
519
534
|
} catch (error: any) {
|
|
520
535
|
throw new MastraError(
|
|
521
536
|
{
|
|
@@ -788,7 +803,7 @@ export class LanceStorage extends MastraStorage {
|
|
|
788
803
|
try {
|
|
789
804
|
const record = { id, title, metadata: JSON.stringify(metadata) };
|
|
790
805
|
const table = await this.lanceClient.openTable(TABLE_THREADS);
|
|
791
|
-
await table.
|
|
806
|
+
await table.mergeInsert('id').whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
792
807
|
|
|
793
808
|
const query = table.query().where(`id = '${id}'`);
|
|
794
809
|
|
|
@@ -1005,7 +1020,8 @@ export class LanceStorage extends MastraStorage {
|
|
|
1005
1020
|
}));
|
|
1006
1021
|
|
|
1007
1022
|
const table = await this.lanceClient.openTable(TABLE_MESSAGES);
|
|
1008
|
-
await table.
|
|
1023
|
+
await table.mergeInsert('id').whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(transformedMessages);
|
|
1024
|
+
|
|
1009
1025
|
const list = new MessageList().add(messages, 'memory');
|
|
1010
1026
|
if (format === `v2`) return list.get.all.v2();
|
|
1011
1027
|
return list.get.all.v1();
|
|
@@ -1322,11 +1338,9 @@ export class LanceStorage extends MastraStorage {
|
|
|
1322
1338
|
const records = await query.toArray();
|
|
1323
1339
|
let createdAt: number;
|
|
1324
1340
|
const now = Date.now();
|
|
1325
|
-
let mode: 'append' | 'overwrite' = 'append';
|
|
1326
1341
|
|
|
1327
1342
|
if (records.length > 0) {
|
|
1328
1343
|
createdAt = records[0].createdAt ?? now;
|
|
1329
|
-
mode = 'overwrite';
|
|
1330
1344
|
} else {
|
|
1331
1345
|
createdAt = now;
|
|
1332
1346
|
}
|
|
@@ -1339,7 +1353,11 @@ export class LanceStorage extends MastraStorage {
|
|
|
1339
1353
|
updatedAt: now,
|
|
1340
1354
|
};
|
|
1341
1355
|
|
|
1342
|
-
await table
|
|
1356
|
+
await table
|
|
1357
|
+
.mergeInsert(['workflow_name', 'run_id'])
|
|
1358
|
+
.whenMatchedUpdateAll()
|
|
1359
|
+
.whenNotMatchedInsertAll()
|
|
1360
|
+
.execute([record]);
|
|
1343
1361
|
} catch (error: any) {
|
|
1344
1362
|
throw new MastraError(
|
|
1345
1363
|
{
|
|
@@ -13,8 +13,8 @@ describe('LanceFilterTranslator', () => {
|
|
|
13
13
|
describe('basic operations', () => {
|
|
14
14
|
it('handles empty filters', () => {
|
|
15
15
|
expect(translator.translate({})).toEqual('');
|
|
16
|
-
expect(translator.translate(null
|
|
17
|
-
expect(translator.translate(undefined
|
|
16
|
+
expect(translator.translate(null)).toEqual('');
|
|
17
|
+
expect(translator.translate(undefined)).toEqual('');
|
|
18
18
|
});
|
|
19
19
|
|
|
20
20
|
it('translates equality operation', () => {
|
|
@@ -244,7 +244,7 @@ describe('LanceFilterTranslator', () => {
|
|
|
244
244
|
});
|
|
245
245
|
|
|
246
246
|
it('throws error for invalid operators at top level', () => {
|
|
247
|
-
const invalidFilters = [{ $gt: 100 }, { $in: ['value1', 'value2'] }, { $like: '%pattern%' }];
|
|
247
|
+
const invalidFilters: any = [{ $gt: 100 }, { $in: ['value1', 'value2'] }, { $like: '%pattern%' }];
|
|
248
248
|
|
|
249
249
|
invalidFilters.forEach(filter => {
|
|
250
250
|
expect(() => translator.translate(filter)).toThrow(/Invalid top-level operator/);
|
package/src/vector/filter.ts
CHANGED
|
@@ -1,8 +1,28 @@
|
|
|
1
|
-
import type { VectorFilter } from '@mastra/core/vector/filter';
|
|
2
1
|
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
import type {
|
|
3
|
+
VectorFilter,
|
|
4
|
+
OperatorValueMap,
|
|
5
|
+
LogicalOperatorValueMap,
|
|
6
|
+
BlacklistedRootOperators,
|
|
7
|
+
} from '@mastra/core/vector/filter';
|
|
8
|
+
|
|
9
|
+
type LanceOperatorValueMap = OperatorValueMap & {
|
|
10
|
+
$like: string;
|
|
11
|
+
$notLike: string;
|
|
12
|
+
$contains: string;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
type LanceBlacklisted = BlacklistedRootOperators | '$like' | '$notLike' | '$contains';
|
|
16
|
+
|
|
17
|
+
export type LanceVectorFilter = VectorFilter<
|
|
18
|
+
keyof LanceOperatorValueMap,
|
|
19
|
+
LanceOperatorValueMap,
|
|
20
|
+
LogicalOperatorValueMap,
|
|
21
|
+
LanceBlacklisted
|
|
22
|
+
>;
|
|
23
|
+
|
|
24
|
+
export class LanceFilterTranslator extends BaseFilterTranslator<LanceVectorFilter, string> {
|
|
25
|
+
translate(filter: LanceVectorFilter): string {
|
|
6
26
|
if (!filter || Object.keys(filter).length === 0) {
|
|
7
27
|
return '';
|
|
8
28
|
}
|
package/src/vector/index.ts
CHANGED
|
@@ -15,7 +15,7 @@ import type {
|
|
|
15
15
|
import { ErrorCategory, ErrorDomain, MastraError } from '@mastra/core/error';
|
|
16
16
|
|
|
17
17
|
import { MastraVector } from '@mastra/core/vector';
|
|
18
|
-
import type {
|
|
18
|
+
import type { LanceVectorFilter } from './filter';
|
|
19
19
|
import { LanceFilterTranslator } from './filter';
|
|
20
20
|
import type { IndexConfig } from './types';
|
|
21
21
|
|
|
@@ -33,13 +33,13 @@ interface LanceUpsertVectorParams extends UpsertVectorParams {
|
|
|
33
33
|
tableName: string;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
interface LanceQueryVectorParams extends QueryVectorParams {
|
|
36
|
+
interface LanceQueryVectorParams extends QueryVectorParams<LanceVectorFilter> {
|
|
37
37
|
tableName: string;
|
|
38
38
|
columns?: string[];
|
|
39
39
|
includeAllColumns?: boolean;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
export class LanceVectorStore extends MastraVector {
|
|
42
|
+
export class LanceVectorStore extends MastraVector<LanceVectorFilter> {
|
|
43
43
|
private lanceClient!: Connection;
|
|
44
44
|
|
|
45
45
|
/**
|
|
@@ -204,7 +204,7 @@ export class LanceVectorStore extends MastraVector {
|
|
|
204
204
|
}
|
|
205
205
|
}
|
|
206
206
|
|
|
207
|
-
private filterTranslator(filter:
|
|
207
|
+
private filterTranslator(filter: LanceVectorFilter): string {
|
|
208
208
|
// Add metadata_ prefix to filter keys if they don't already have it
|
|
209
209
|
const processFilterKeys = (filterObj: Record<string, any>): Record<string, any> => {
|
|
210
210
|
const result: Record<string, any> = {};
|