@mastra/lance 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1570 -3
- package/LICENSE.md +15 -0
- package/README.md +61 -4
- package/dist/docs/SKILL.md +27 -0
- package/dist/docs/assets/SOURCE_MAP.json +6 -0
- package/dist/docs/references/docs-rag-vector-databases.md +645 -0
- package/dist/docs/references/reference-storage-lance.md +131 -0
- package/dist/docs/references/reference-vectors-lance.md +220 -0
- package/dist/index.cjs +1719 -1786
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1717 -1787
- package/dist/index.js.map +1 -1
- package/dist/storage/{domains/operations → db}/index.d.ts +21 -2
- package/dist/storage/db/index.d.ts.map +1 -0
- package/dist/storage/db/utils.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +21 -46
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/dist/storage/domains/scores/index.d.ts +24 -27
- package/dist/storage/domains/scores/index.d.ts.map +1 -1
- package/dist/storage/domains/workflows/index.d.ts +21 -24
- package/dist/storage/domains/workflows/index.d.ts.map +1 -1
- package/dist/storage/index.d.ts +103 -246
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/vector/filter.d.ts +5 -5
- package/dist/vector/index.d.ts +29 -11
- package/dist/vector/index.d.ts.map +1 -1
- package/package.json +17 -13
- package/dist/storage/domains/legacy-evals/index.d.ts +0 -25
- package/dist/storage/domains/legacy-evals/index.d.ts.map +0 -1
- package/dist/storage/domains/operations/index.d.ts.map +0 -1
- package/dist/storage/domains/traces/index.d.ts +0 -34
- package/dist/storage/domains/traces/index.d.ts.map +0 -1
- package/dist/storage/domains/utils.d.ts.map +0 -1
- /package/dist/storage/{domains → db}/utils.d.ts +0 -0
package/dist/index.js
CHANGED
|
@@ -1,133 +1,18 @@
|
|
|
1
1
|
import { connect, Index } from '@lancedb/lancedb';
|
|
2
2
|
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
|
-
import {
|
|
3
|
+
import { MemoryStorage, TABLE_SCHEMAS, TABLE_THREADS, TABLE_MESSAGES, TABLE_RESOURCES, createStorageErrorId, normalizePerPage, calculatePagination, ScoresStorage, SCORERS_SCHEMA, TABLE_SCORERS, WorkflowsStorage, ensureDate, TABLE_WORKFLOW_SNAPSHOT, MastraCompositeStore, createVectorErrorId, getDefaultValue } from '@mastra/core/storage';
|
|
4
4
|
import { MessageList } from '@mastra/core/agent';
|
|
5
|
+
import { MastraBase } from '@mastra/core/base';
|
|
5
6
|
import { Utf8, Float64, Binary, Float32, Int32, Field, Schema } from 'apache-arrow';
|
|
6
|
-
import { saveScorePayloadSchema } from '@mastra/core/
|
|
7
|
+
import { saveScorePayloadSchema } from '@mastra/core/evals';
|
|
7
8
|
import { MastraVector } from '@mastra/core/vector';
|
|
8
9
|
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
9
10
|
|
|
10
11
|
// src/storage/index.ts
|
|
11
|
-
var StoreLegacyEvalsLance = class extends LegacyEvalsStorage {
|
|
12
|
-
client;
|
|
13
|
-
constructor({ client }) {
|
|
14
|
-
super();
|
|
15
|
-
this.client = client;
|
|
16
|
-
}
|
|
17
|
-
async getEvalsByAgentName(agentName, type) {
|
|
18
|
-
try {
|
|
19
|
-
const table = await this.client.openTable(TABLE_EVALS);
|
|
20
|
-
const query = table.query().where(`agent_name = '${agentName}'`);
|
|
21
|
-
const records = await query.toArray();
|
|
22
|
-
let filteredRecords = records;
|
|
23
|
-
if (type === "live") {
|
|
24
|
-
filteredRecords = records.filter((record) => record.test_info === null);
|
|
25
|
-
} else if (type === "test") {
|
|
26
|
-
filteredRecords = records.filter((record) => record.test_info !== null);
|
|
27
|
-
}
|
|
28
|
-
return filteredRecords.map((record) => {
|
|
29
|
-
return {
|
|
30
|
-
id: record.id,
|
|
31
|
-
input: record.input,
|
|
32
|
-
output: record.output,
|
|
33
|
-
agentName: record.agent_name,
|
|
34
|
-
metricName: record.metric_name,
|
|
35
|
-
result: JSON.parse(record.result),
|
|
36
|
-
instructions: record.instructions,
|
|
37
|
-
testInfo: record.test_info ? JSON.parse(record.test_info) : null,
|
|
38
|
-
globalRunId: record.global_run_id,
|
|
39
|
-
runId: record.run_id,
|
|
40
|
-
createdAt: new Date(record.created_at).toString()
|
|
41
|
-
};
|
|
42
|
-
});
|
|
43
|
-
} catch (error) {
|
|
44
|
-
throw new MastraError(
|
|
45
|
-
{
|
|
46
|
-
id: "LANCE_STORE_GET_EVALS_BY_AGENT_NAME_FAILED",
|
|
47
|
-
domain: ErrorDomain.STORAGE,
|
|
48
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
49
|
-
details: { agentName }
|
|
50
|
-
},
|
|
51
|
-
error
|
|
52
|
-
);
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
async getEvals(options) {
|
|
56
|
-
try {
|
|
57
|
-
const table = await this.client.openTable(TABLE_EVALS);
|
|
58
|
-
const conditions = [];
|
|
59
|
-
if (options.agentName) {
|
|
60
|
-
conditions.push(`agent_name = '${options.agentName}'`);
|
|
61
|
-
}
|
|
62
|
-
if (options.type === "live") {
|
|
63
|
-
conditions.push("length(test_info) = 0");
|
|
64
|
-
} else if (options.type === "test") {
|
|
65
|
-
conditions.push("length(test_info) > 0");
|
|
66
|
-
}
|
|
67
|
-
const startDate = options.dateRange?.start || options.fromDate;
|
|
68
|
-
const endDate = options.dateRange?.end || options.toDate;
|
|
69
|
-
if (startDate) {
|
|
70
|
-
conditions.push(`\`created_at\` >= ${startDate.getTime()}`);
|
|
71
|
-
}
|
|
72
|
-
if (endDate) {
|
|
73
|
-
conditions.push(`\`created_at\` <= ${endDate.getTime()}`);
|
|
74
|
-
}
|
|
75
|
-
let total = 0;
|
|
76
|
-
if (conditions.length > 0) {
|
|
77
|
-
total = await table.countRows(conditions.join(" AND "));
|
|
78
|
-
} else {
|
|
79
|
-
total = await table.countRows();
|
|
80
|
-
}
|
|
81
|
-
const query = table.query();
|
|
82
|
-
if (conditions.length > 0) {
|
|
83
|
-
const whereClause = conditions.join(" AND ");
|
|
84
|
-
query.where(whereClause);
|
|
85
|
-
}
|
|
86
|
-
const records = await query.toArray();
|
|
87
|
-
const evals = records.sort((a, b) => b.created_at - a.created_at).map((record) => {
|
|
88
|
-
return {
|
|
89
|
-
id: record.id,
|
|
90
|
-
input: record.input,
|
|
91
|
-
output: record.output,
|
|
92
|
-
agentName: record.agent_name,
|
|
93
|
-
metricName: record.metric_name,
|
|
94
|
-
result: JSON.parse(record.result),
|
|
95
|
-
instructions: record.instructions,
|
|
96
|
-
testInfo: record.test_info ? JSON.parse(record.test_info) : null,
|
|
97
|
-
globalRunId: record.global_run_id,
|
|
98
|
-
runId: record.run_id,
|
|
99
|
-
createdAt: new Date(record.created_at).toISOString()
|
|
100
|
-
};
|
|
101
|
-
});
|
|
102
|
-
const page = options.page || 0;
|
|
103
|
-
const perPage = options.perPage || 10;
|
|
104
|
-
const pagedEvals = evals.slice(page * perPage, (page + 1) * perPage);
|
|
105
|
-
return {
|
|
106
|
-
evals: pagedEvals,
|
|
107
|
-
total,
|
|
108
|
-
page,
|
|
109
|
-
perPage,
|
|
110
|
-
hasMore: total > (page + 1) * perPage
|
|
111
|
-
};
|
|
112
|
-
} catch (error) {
|
|
113
|
-
throw new MastraError(
|
|
114
|
-
{
|
|
115
|
-
id: "LANCE_STORE_GET_EVALS_FAILED",
|
|
116
|
-
domain: ErrorDomain.STORAGE,
|
|
117
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
118
|
-
details: { agentName: options.agentName ?? "" }
|
|
119
|
-
},
|
|
120
|
-
error
|
|
121
|
-
);
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
};
|
|
125
12
|
function getPrimaryKeys(tableName) {
|
|
126
13
|
let primaryId = ["id"];
|
|
127
14
|
if (tableName === TABLE_WORKFLOW_SNAPSHOT) {
|
|
128
15
|
primaryId = ["workflow_name", "run_id"];
|
|
129
|
-
} else if (tableName === TABLE_EVALS) {
|
|
130
|
-
primaryId = ["agent_name", "metric_name", "run_id"];
|
|
131
16
|
}
|
|
132
17
|
return primaryId;
|
|
133
18
|
}
|
|
@@ -205,7 +90,7 @@ async function getTableSchema({
|
|
|
205
90
|
} catch (validationError) {
|
|
206
91
|
throw new MastraError(
|
|
207
92
|
{
|
|
208
|
-
id: "
|
|
93
|
+
id: createStorageErrorId("LANCE", "GET_TABLE_SCHEMA", "INVALID_ARGS"),
|
|
209
94
|
domain: ErrorDomain.STORAGE,
|
|
210
95
|
category: ErrorCategory.USER,
|
|
211
96
|
text: validationError.message,
|
|
@@ -228,7 +113,7 @@ async function getTableSchema({
|
|
|
228
113
|
} catch (error) {
|
|
229
114
|
throw new MastraError(
|
|
230
115
|
{
|
|
231
|
-
id: "
|
|
116
|
+
id: createStorageErrorId("LANCE", "GET_TABLE_SCHEMA", "FAILED"),
|
|
232
117
|
domain: ErrorDomain.STORAGE,
|
|
233
118
|
category: ErrorCategory.THIRD_PARTY,
|
|
234
119
|
details: { tableName }
|
|
@@ -238,1163 +123,1236 @@ async function getTableSchema({
|
|
|
238
123
|
}
|
|
239
124
|
}
|
|
240
125
|
|
|
241
|
-
// src/storage/
|
|
242
|
-
|
|
126
|
+
// src/storage/db/index.ts
|
|
127
|
+
function resolveLanceConfig(config) {
|
|
128
|
+
return config.client;
|
|
129
|
+
}
|
|
130
|
+
var LanceDB = class extends MastraBase {
|
|
243
131
|
client;
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
super();
|
|
132
|
+
constructor({ client }) {
|
|
133
|
+
super({ name: "lance-db" });
|
|
247
134
|
this.client = client;
|
|
248
|
-
this.operations = operations;
|
|
249
135
|
}
|
|
250
|
-
|
|
136
|
+
getDefaultValue(type) {
|
|
137
|
+
switch (type) {
|
|
138
|
+
case "text":
|
|
139
|
+
return "''";
|
|
140
|
+
case "timestamp":
|
|
141
|
+
return "CURRENT_TIMESTAMP";
|
|
142
|
+
case "integer":
|
|
143
|
+
case "bigint":
|
|
144
|
+
return "0";
|
|
145
|
+
case "jsonb":
|
|
146
|
+
return "'{}'";
|
|
147
|
+
case "uuid":
|
|
148
|
+
return "''";
|
|
149
|
+
default:
|
|
150
|
+
return getDefaultValue(type);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
async hasColumn(tableName, columnName) {
|
|
154
|
+
const table = await this.client.openTable(tableName);
|
|
155
|
+
const schema = await table.schema();
|
|
156
|
+
return schema.fields.some((field) => field.name === columnName);
|
|
157
|
+
}
|
|
158
|
+
translateSchema(schema) {
|
|
159
|
+
const fields = Object.entries(schema).map(([name, column]) => {
|
|
160
|
+
let arrowType;
|
|
161
|
+
switch (column.type.toLowerCase()) {
|
|
162
|
+
case "text":
|
|
163
|
+
case "uuid":
|
|
164
|
+
arrowType = new Utf8();
|
|
165
|
+
break;
|
|
166
|
+
case "int":
|
|
167
|
+
case "integer":
|
|
168
|
+
arrowType = new Int32();
|
|
169
|
+
break;
|
|
170
|
+
case "bigint":
|
|
171
|
+
arrowType = new Float64();
|
|
172
|
+
break;
|
|
173
|
+
case "float":
|
|
174
|
+
arrowType = new Float32();
|
|
175
|
+
break;
|
|
176
|
+
case "jsonb":
|
|
177
|
+
case "json":
|
|
178
|
+
arrowType = new Utf8();
|
|
179
|
+
break;
|
|
180
|
+
case "binary":
|
|
181
|
+
arrowType = new Binary();
|
|
182
|
+
break;
|
|
183
|
+
case "timestamp":
|
|
184
|
+
arrowType = new Float64();
|
|
185
|
+
break;
|
|
186
|
+
default:
|
|
187
|
+
arrowType = new Utf8();
|
|
188
|
+
}
|
|
189
|
+
return new Field(name, arrowType, column.nullable ?? true);
|
|
190
|
+
});
|
|
191
|
+
return new Schema(fields);
|
|
192
|
+
}
|
|
193
|
+
async createTable({
|
|
194
|
+
tableName,
|
|
195
|
+
schema
|
|
196
|
+
}) {
|
|
251
197
|
try {
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
198
|
+
if (!this.client) {
|
|
199
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
200
|
+
}
|
|
201
|
+
if (!tableName) {
|
|
202
|
+
throw new Error("tableName is required for createTable.");
|
|
203
|
+
}
|
|
204
|
+
if (!schema) {
|
|
205
|
+
throw new Error("schema is required for createTable.");
|
|
255
206
|
}
|
|
256
|
-
return {
|
|
257
|
-
...thread,
|
|
258
|
-
createdAt: new Date(thread.createdAt),
|
|
259
|
-
updatedAt: new Date(thread.updatedAt)
|
|
260
|
-
};
|
|
261
207
|
} catch (error) {
|
|
262
208
|
throw new MastraError(
|
|
263
209
|
{
|
|
264
|
-
id: "
|
|
210
|
+
id: createStorageErrorId("LANCE", "CREATE_TABLE", "INVALID_ARGS"),
|
|
265
211
|
domain: ErrorDomain.STORAGE,
|
|
266
|
-
category: ErrorCategory.
|
|
212
|
+
category: ErrorCategory.USER,
|
|
213
|
+
details: { tableName }
|
|
267
214
|
},
|
|
268
215
|
error
|
|
269
216
|
);
|
|
270
217
|
}
|
|
271
|
-
}
|
|
272
|
-
async getThreadsByResourceId({ resourceId }) {
|
|
273
218
|
try {
|
|
274
|
-
const
|
|
275
|
-
|
|
276
|
-
const records = await query.toArray();
|
|
277
|
-
return processResultWithTypeConversion(
|
|
278
|
-
records,
|
|
279
|
-
await getTableSchema({ tableName: TABLE_THREADS, client: this.client })
|
|
280
|
-
);
|
|
219
|
+
const arrowSchema = this.translateSchema(schema);
|
|
220
|
+
await this.client.createEmptyTable(tableName, arrowSchema);
|
|
281
221
|
} catch (error) {
|
|
222
|
+
if (error.message?.includes("already exists")) {
|
|
223
|
+
this.logger.debug(`Table '${tableName}' already exists, skipping create`);
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
282
226
|
throw new MastraError(
|
|
283
227
|
{
|
|
284
|
-
id: "
|
|
228
|
+
id: createStorageErrorId("LANCE", "CREATE_TABLE", "FAILED"),
|
|
285
229
|
domain: ErrorDomain.STORAGE,
|
|
286
|
-
category: ErrorCategory.THIRD_PARTY
|
|
230
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
231
|
+
details: { tableName }
|
|
287
232
|
},
|
|
288
233
|
error
|
|
289
234
|
);
|
|
290
235
|
}
|
|
291
236
|
}
|
|
292
|
-
|
|
293
|
-
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
294
|
-
* @param thread - The thread to save
|
|
295
|
-
* @returns The saved thread
|
|
296
|
-
*/
|
|
297
|
-
async saveThread({ thread }) {
|
|
237
|
+
async dropTable({ tableName }) {
|
|
298
238
|
try {
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
239
|
+
if (!this.client) {
|
|
240
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
241
|
+
}
|
|
242
|
+
if (!tableName) {
|
|
243
|
+
throw new Error("tableName is required for dropTable.");
|
|
244
|
+
}
|
|
245
|
+
} catch (validationError) {
|
|
304
246
|
throw new MastraError(
|
|
305
247
|
{
|
|
306
|
-
id: "
|
|
248
|
+
id: createStorageErrorId("LANCE", "DROP_TABLE", "INVALID_ARGS"),
|
|
307
249
|
domain: ErrorDomain.STORAGE,
|
|
308
|
-
category: ErrorCategory.
|
|
250
|
+
category: ErrorCategory.USER,
|
|
251
|
+
text: validationError.message,
|
|
252
|
+
details: { tableName }
|
|
309
253
|
},
|
|
310
|
-
|
|
254
|
+
validationError
|
|
311
255
|
);
|
|
312
256
|
}
|
|
313
|
-
}
|
|
314
|
-
async updateThread({
|
|
315
|
-
id,
|
|
316
|
-
title,
|
|
317
|
-
metadata
|
|
318
|
-
}) {
|
|
319
|
-
const maxRetries = 5;
|
|
320
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
321
|
-
try {
|
|
322
|
-
const current = await this.getThreadById({ threadId: id });
|
|
323
|
-
if (!current) {
|
|
324
|
-
throw new Error(`Thread with id ${id} not found`);
|
|
325
|
-
}
|
|
326
|
-
const mergedMetadata = { ...current.metadata, ...metadata };
|
|
327
|
-
const record = {
|
|
328
|
-
id,
|
|
329
|
-
title,
|
|
330
|
-
metadata: JSON.stringify(mergedMetadata),
|
|
331
|
-
updatedAt: (/* @__PURE__ */ new Date()).getTime()
|
|
332
|
-
};
|
|
333
|
-
const table = await this.client.openTable(TABLE_THREADS);
|
|
334
|
-
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
335
|
-
const updatedThread = await this.getThreadById({ threadId: id });
|
|
336
|
-
if (!updatedThread) {
|
|
337
|
-
throw new Error(`Failed to retrieve updated thread ${id}`);
|
|
338
|
-
}
|
|
339
|
-
return updatedThread;
|
|
340
|
-
} catch (error) {
|
|
341
|
-
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
342
|
-
const delay = Math.pow(2, attempt) * 10;
|
|
343
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
344
|
-
continue;
|
|
345
|
-
}
|
|
346
|
-
throw new MastraError(
|
|
347
|
-
{
|
|
348
|
-
id: "LANCE_STORE_UPDATE_THREAD_FAILED",
|
|
349
|
-
domain: ErrorDomain.STORAGE,
|
|
350
|
-
category: ErrorCategory.THIRD_PARTY
|
|
351
|
-
},
|
|
352
|
-
error
|
|
353
|
-
);
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
|
-
throw new MastraError(
|
|
357
|
-
{
|
|
358
|
-
id: "LANCE_STORE_UPDATE_THREAD_FAILED",
|
|
359
|
-
domain: ErrorDomain.STORAGE,
|
|
360
|
-
category: ErrorCategory.THIRD_PARTY
|
|
361
|
-
},
|
|
362
|
-
new Error("All retries exhausted")
|
|
363
|
-
);
|
|
364
|
-
}
|
|
365
|
-
async deleteThread({ threadId }) {
|
|
366
257
|
try {
|
|
367
|
-
|
|
368
|
-
await table.delete(`id = '${threadId}'`);
|
|
369
|
-
const messagesTable = await this.client.openTable(TABLE_MESSAGES);
|
|
370
|
-
await messagesTable.delete(`thread_id = '${threadId}'`);
|
|
258
|
+
await this.client.dropTable(tableName);
|
|
371
259
|
} catch (error) {
|
|
260
|
+
if (error.toString().includes("was not found") || error.message?.includes("Table not found")) {
|
|
261
|
+
this.logger.debug(`Table '${tableName}' does not exist, skipping drop`);
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
372
264
|
throw new MastraError(
|
|
373
265
|
{
|
|
374
|
-
id: "
|
|
266
|
+
id: createStorageErrorId("LANCE", "DROP_TABLE", "FAILED"),
|
|
375
267
|
domain: ErrorDomain.STORAGE,
|
|
376
|
-
category: ErrorCategory.THIRD_PARTY
|
|
268
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
269
|
+
details: { tableName }
|
|
377
270
|
},
|
|
378
271
|
error
|
|
379
272
|
);
|
|
380
273
|
}
|
|
381
274
|
}
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
content: typeof message.content === "string" ? (() => {
|
|
388
|
-
try {
|
|
389
|
-
return JSON.parse(message.content);
|
|
390
|
-
} catch {
|
|
391
|
-
return message.content;
|
|
392
|
-
}
|
|
393
|
-
})() : message.content
|
|
394
|
-
};
|
|
395
|
-
}
|
|
396
|
-
async getMessages({
|
|
397
|
-
threadId,
|
|
398
|
-
resourceId,
|
|
399
|
-
selectBy,
|
|
400
|
-
format,
|
|
401
|
-
threadConfig
|
|
402
|
-
}) {
|
|
275
|
+
async alterTable({
|
|
276
|
+
tableName,
|
|
277
|
+
schema,
|
|
278
|
+
ifNotExists
|
|
279
|
+
}) {
|
|
403
280
|
try {
|
|
404
|
-
if (!
|
|
405
|
-
|
|
406
|
-
throw new Error("ThreadConfig is not supported by LanceDB storage");
|
|
281
|
+
if (!this.client) {
|
|
282
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
407
283
|
}
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
let allRecords = [];
|
|
411
|
-
if (selectBy?.include && selectBy.include.length > 0) {
|
|
412
|
-
const threadIds = [...new Set(selectBy.include.map((item) => item.threadId))];
|
|
413
|
-
for (const threadId2 of threadIds) {
|
|
414
|
-
const threadQuery = table.query().where(`thread_id = '${threadId2}'`);
|
|
415
|
-
let threadRecords = await threadQuery.toArray();
|
|
416
|
-
allRecords.push(...threadRecords);
|
|
417
|
-
}
|
|
418
|
-
} else {
|
|
419
|
-
let query = table.query().where(`\`thread_id\` = '${threadId}'`);
|
|
420
|
-
allRecords = await query.toArray();
|
|
284
|
+
if (!tableName) {
|
|
285
|
+
throw new Error("tableName is required for alterTable.");
|
|
421
286
|
}
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
const dateB = new Date(b.createdAt).getTime();
|
|
425
|
-
return dateA - dateB;
|
|
426
|
-
});
|
|
427
|
-
if (selectBy?.include && selectBy.include.length > 0) {
|
|
428
|
-
allRecords = this.processMessagesWithContext(allRecords, selectBy.include);
|
|
287
|
+
if (!schema) {
|
|
288
|
+
throw new Error("schema is required for alterTable.");
|
|
429
289
|
}
|
|
430
|
-
if (
|
|
431
|
-
|
|
290
|
+
if (!ifNotExists || ifNotExists.length === 0) {
|
|
291
|
+
this.logger.debug("No columns specified to add in alterTable, skipping.");
|
|
292
|
+
return;
|
|
432
293
|
}
|
|
433
|
-
|
|
434
|
-
allRecords,
|
|
435
|
-
await getTableSchema({ tableName: TABLE_MESSAGES, client: this.client })
|
|
436
|
-
);
|
|
437
|
-
const list = new MessageList({ threadId, resourceId }).add(messages.map(this.normalizeMessage), "memory");
|
|
438
|
-
if (format === "v2") return list.get.all.v2();
|
|
439
|
-
return list.get.all.v1();
|
|
440
|
-
} catch (error) {
|
|
294
|
+
} catch (validationError) {
|
|
441
295
|
throw new MastraError(
|
|
442
296
|
{
|
|
443
|
-
id: "
|
|
297
|
+
id: createStorageErrorId("LANCE", "ALTER_TABLE", "INVALID_ARGS"),
|
|
444
298
|
domain: ErrorDomain.STORAGE,
|
|
445
|
-
category: ErrorCategory.
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
resourceId: resourceId ?? ""
|
|
449
|
-
}
|
|
299
|
+
category: ErrorCategory.USER,
|
|
300
|
+
text: validationError.message,
|
|
301
|
+
details: { tableName }
|
|
450
302
|
},
|
|
451
|
-
|
|
303
|
+
validationError
|
|
452
304
|
);
|
|
453
305
|
}
|
|
454
|
-
}
|
|
455
|
-
async getMessagesById({
|
|
456
|
-
messageIds,
|
|
457
|
-
format
|
|
458
|
-
}) {
|
|
459
|
-
if (messageIds.length === 0) return [];
|
|
460
306
|
try {
|
|
461
|
-
const table = await this.client.openTable(
|
|
462
|
-
const
|
|
463
|
-
const
|
|
464
|
-
const
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
307
|
+
const table = await this.client.openTable(tableName);
|
|
308
|
+
const currentSchema = await table.schema();
|
|
309
|
+
const existingFields = new Set(currentSchema.fields.map((f) => f.name));
|
|
310
|
+
const typeMap = {
|
|
311
|
+
text: "string",
|
|
312
|
+
integer: "int",
|
|
313
|
+
bigint: "bigint",
|
|
314
|
+
timestamp: "timestamp",
|
|
315
|
+
jsonb: "string",
|
|
316
|
+
uuid: "string"
|
|
317
|
+
};
|
|
318
|
+
const columnsToAdd = ifNotExists.filter((col) => schema[col] && !existingFields.has(col)).map((col) => {
|
|
319
|
+
const colDef = schema[col];
|
|
320
|
+
return {
|
|
321
|
+
name: col,
|
|
322
|
+
valueSql: colDef?.nullable ? `cast(NULL as ${typeMap[colDef.type ?? "text"]})` : `cast(${this.getDefaultValue(colDef?.type ?? "text")} as ${typeMap[colDef?.type ?? "text"]})`
|
|
323
|
+
};
|
|
324
|
+
});
|
|
325
|
+
if (columnsToAdd.length > 0) {
|
|
326
|
+
await table.addColumns(columnsToAdd);
|
|
327
|
+
this.logger?.info?.(`Added columns [${columnsToAdd.map((c) => c.name).join(", ")}] to table ${tableName}`);
|
|
328
|
+
}
|
|
471
329
|
} catch (error) {
|
|
472
330
|
throw new MastraError(
|
|
473
331
|
{
|
|
474
|
-
id: "
|
|
332
|
+
id: createStorageErrorId("LANCE", "ALTER_TABLE", "FAILED"),
|
|
475
333
|
domain: ErrorDomain.STORAGE,
|
|
476
334
|
category: ErrorCategory.THIRD_PARTY,
|
|
477
|
-
details: {
|
|
478
|
-
messageIds: JSON.stringify(messageIds)
|
|
479
|
-
}
|
|
335
|
+
details: { tableName }
|
|
480
336
|
},
|
|
481
337
|
error
|
|
482
338
|
);
|
|
483
339
|
}
|
|
484
340
|
}
|
|
485
|
-
async
|
|
341
|
+
async clearTable({ tableName }) {
|
|
486
342
|
try {
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
return [];
|
|
490
|
-
}
|
|
491
|
-
const threadId = messages[0]?.threadId;
|
|
492
|
-
if (!threadId) {
|
|
493
|
-
throw new Error("Thread ID is required");
|
|
343
|
+
if (!this.client) {
|
|
344
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
494
345
|
}
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
throw new Error("Message ID is required");
|
|
498
|
-
}
|
|
499
|
-
if (!message.threadId) {
|
|
500
|
-
throw new Error("Thread ID is required for all messages");
|
|
501
|
-
}
|
|
502
|
-
if (message.resourceId === null || message.resourceId === void 0) {
|
|
503
|
-
throw new Error("Resource ID cannot be null or undefined");
|
|
504
|
-
}
|
|
505
|
-
if (!message.content) {
|
|
506
|
-
throw new Error("Message content is required");
|
|
507
|
-
}
|
|
346
|
+
if (!tableName) {
|
|
347
|
+
throw new Error("tableName is required for clearTable.");
|
|
508
348
|
}
|
|
509
|
-
|
|
510
|
-
const { threadId: threadId2, type, ...rest } = message;
|
|
511
|
-
return {
|
|
512
|
-
...rest,
|
|
513
|
-
thread_id: threadId2,
|
|
514
|
-
type: type ?? "v2",
|
|
515
|
-
content: JSON.stringify(message.content)
|
|
516
|
-
};
|
|
517
|
-
});
|
|
518
|
-
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
519
|
-
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(transformedMessages);
|
|
520
|
-
const threadsTable = await this.client.openTable(TABLE_THREADS);
|
|
521
|
-
const currentTime = (/* @__PURE__ */ new Date()).getTime();
|
|
522
|
-
const updateRecord = { id: threadId, updatedAt: currentTime };
|
|
523
|
-
await threadsTable.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([updateRecord]);
|
|
524
|
-
const list = new MessageList().add(messages, "memory");
|
|
525
|
-
if (format === `v2`) return list.get.all.v2();
|
|
526
|
-
return list.get.all.v1();
|
|
527
|
-
} catch (error) {
|
|
349
|
+
} catch (validationError) {
|
|
528
350
|
throw new MastraError(
|
|
529
351
|
{
|
|
530
|
-
id: "
|
|
352
|
+
id: createStorageErrorId("LANCE", "CLEAR_TABLE", "INVALID_ARGS"),
|
|
531
353
|
domain: ErrorDomain.STORAGE,
|
|
532
|
-
category: ErrorCategory.
|
|
354
|
+
category: ErrorCategory.USER,
|
|
355
|
+
text: validationError.message,
|
|
356
|
+
details: { tableName }
|
|
533
357
|
},
|
|
534
|
-
|
|
358
|
+
validationError
|
|
535
359
|
);
|
|
536
360
|
}
|
|
537
|
-
}
|
|
538
|
-
async getThreadsByResourceIdPaginated(args) {
|
|
539
361
|
try {
|
|
540
|
-
const
|
|
541
|
-
|
|
542
|
-
const total = await table.countRows(`\`resourceId\` = '${resourceId}'`);
|
|
543
|
-
const query = table.query().where(`\`resourceId\` = '${resourceId}'`);
|
|
544
|
-
const offset = page * perPage;
|
|
545
|
-
query.limit(perPage);
|
|
546
|
-
if (offset > 0) {
|
|
547
|
-
query.offset(offset);
|
|
548
|
-
}
|
|
549
|
-
const records = await query.toArray();
|
|
550
|
-
records.sort((a, b) => new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime());
|
|
551
|
-
const schema = await getTableSchema({ tableName: TABLE_THREADS, client: this.client });
|
|
552
|
-
const threads = records.map((record) => processResultWithTypeConversion(record, schema));
|
|
553
|
-
return {
|
|
554
|
-
threads,
|
|
555
|
-
total,
|
|
556
|
-
page,
|
|
557
|
-
perPage,
|
|
558
|
-
hasMore: total > (page + 1) * perPage
|
|
559
|
-
};
|
|
362
|
+
const table = await this.client.openTable(tableName);
|
|
363
|
+
await table.delete("1=1");
|
|
560
364
|
} catch (error) {
|
|
561
365
|
throw new MastraError(
|
|
562
366
|
{
|
|
563
|
-
id: "
|
|
367
|
+
id: createStorageErrorId("LANCE", "CLEAR_TABLE", "FAILED"),
|
|
564
368
|
domain: ErrorDomain.STORAGE,
|
|
565
|
-
category: ErrorCategory.THIRD_PARTY
|
|
369
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
370
|
+
details: { tableName }
|
|
566
371
|
},
|
|
567
372
|
error
|
|
568
373
|
);
|
|
569
374
|
}
|
|
570
375
|
}
|
|
571
|
-
|
|
572
|
-
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
573
|
-
* @param records - The sorted array of records to process
|
|
574
|
-
* @param include - The array of include specifications with context parameters
|
|
575
|
-
* @returns The processed array with context messages included
|
|
576
|
-
*/
|
|
577
|
-
processMessagesWithContext(records, include) {
|
|
578
|
-
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
579
|
-
if (messagesWithContext.length === 0) {
|
|
580
|
-
return records;
|
|
581
|
-
}
|
|
582
|
-
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
583
|
-
records.forEach((message, index) => {
|
|
584
|
-
messageIndexMap.set(message.id, index);
|
|
585
|
-
});
|
|
586
|
-
const additionalIndices = /* @__PURE__ */ new Set();
|
|
587
|
-
for (const item of messagesWithContext) {
|
|
588
|
-
const messageIndex = messageIndexMap.get(item.id);
|
|
589
|
-
if (messageIndex !== void 0) {
|
|
590
|
-
if (item.withPreviousMessages) {
|
|
591
|
-
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
592
|
-
for (let i = startIdx; i < messageIndex; i++) {
|
|
593
|
-
additionalIndices.add(i);
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
if (item.withNextMessages) {
|
|
597
|
-
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
598
|
-
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
599
|
-
additionalIndices.add(i);
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
}
|
|
603
|
-
}
|
|
604
|
-
if (additionalIndices.size === 0) {
|
|
605
|
-
return records;
|
|
606
|
-
}
|
|
607
|
-
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
608
|
-
const allIndices = /* @__PURE__ */ new Set();
|
|
609
|
-
records.forEach((record, index) => {
|
|
610
|
-
if (originalMatchIds.has(record.id)) {
|
|
611
|
-
allIndices.add(index);
|
|
612
|
-
}
|
|
613
|
-
});
|
|
614
|
-
additionalIndices.forEach((index) => {
|
|
615
|
-
allIndices.add(index);
|
|
616
|
-
});
|
|
617
|
-
return Array.from(allIndices).sort((a, b) => a - b).map((index) => records[index]);
|
|
618
|
-
}
|
|
619
|
-
async getMessagesPaginated(args) {
|
|
620
|
-
const { threadId, resourceId, selectBy, format = "v1" } = args;
|
|
621
|
-
const page = selectBy?.pagination?.page ?? 0;
|
|
622
|
-
const perPage = selectBy?.pagination?.perPage ?? 10;
|
|
376
|
+
async insert({ tableName, record }) {
|
|
623
377
|
try {
|
|
624
|
-
if (!
|
|
625
|
-
|
|
626
|
-
const fromDate = dateRange?.start;
|
|
627
|
-
const toDate = dateRange?.end;
|
|
628
|
-
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
629
|
-
const messages = [];
|
|
630
|
-
if (selectBy?.include && Array.isArray(selectBy.include)) {
|
|
631
|
-
const threadIds = [...new Set(selectBy.include.map((item) => item.threadId))];
|
|
632
|
-
const allThreadMessages = [];
|
|
633
|
-
for (const threadId2 of threadIds) {
|
|
634
|
-
const threadQuery = table.query().where(`thread_id = '${threadId2}'`);
|
|
635
|
-
let threadRecords = await threadQuery.toArray();
|
|
636
|
-
if (fromDate) threadRecords = threadRecords.filter((m) => m.createdAt >= fromDate.getTime());
|
|
637
|
-
if (toDate) threadRecords = threadRecords.filter((m) => m.createdAt <= toDate.getTime());
|
|
638
|
-
allThreadMessages.push(...threadRecords);
|
|
639
|
-
}
|
|
640
|
-
allThreadMessages.sort((a, b) => a.createdAt - b.createdAt);
|
|
641
|
-
const contextMessages = this.processMessagesWithContext(allThreadMessages, selectBy.include);
|
|
642
|
-
messages.push(...contextMessages);
|
|
643
|
-
}
|
|
644
|
-
const conditions = [`thread_id = '${threadId}'`];
|
|
645
|
-
if (resourceId) {
|
|
646
|
-
conditions.push(`\`resourceId\` = '${resourceId}'`);
|
|
647
|
-
}
|
|
648
|
-
if (fromDate) {
|
|
649
|
-
conditions.push(`\`createdAt\` >= ${fromDate.getTime()}`);
|
|
650
|
-
}
|
|
651
|
-
if (toDate) {
|
|
652
|
-
conditions.push(`\`createdAt\` <= ${toDate.getTime()}`);
|
|
378
|
+
if (!this.client) {
|
|
379
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
653
380
|
}
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
total = await table.countRows(conditions.join(" AND "));
|
|
657
|
-
} else {
|
|
658
|
-
total = await table.countRows();
|
|
381
|
+
if (!tableName) {
|
|
382
|
+
throw new Error("tableName is required for insert.");
|
|
659
383
|
}
|
|
660
|
-
if (
|
|
661
|
-
|
|
662
|
-
messages: [],
|
|
663
|
-
total: 0,
|
|
664
|
-
page,
|
|
665
|
-
perPage,
|
|
666
|
-
hasMore: false
|
|
667
|
-
};
|
|
384
|
+
if (!record || Object.keys(record).length === 0) {
|
|
385
|
+
throw new Error("record is required and cannot be empty for insert.");
|
|
668
386
|
}
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
if (excludeIds.length > 0) {
|
|
690
|
-
records = records.filter((m) => !excludeIds.includes(m.id));
|
|
387
|
+
} catch (validationError) {
|
|
388
|
+
throw new MastraError(
|
|
389
|
+
{
|
|
390
|
+
id: createStorageErrorId("LANCE", "INSERT", "INVALID_ARGS"),
|
|
391
|
+
domain: ErrorDomain.STORAGE,
|
|
392
|
+
category: ErrorCategory.USER,
|
|
393
|
+
text: validationError.message,
|
|
394
|
+
details: { tableName }
|
|
395
|
+
},
|
|
396
|
+
validationError
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
try {
|
|
400
|
+
const table = await this.client.openTable(tableName);
|
|
401
|
+
const primaryId = getPrimaryKeys(tableName);
|
|
402
|
+
const processedRecord = { ...record };
|
|
403
|
+
for (const key in processedRecord) {
|
|
404
|
+
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
405
|
+
this.logger.debug("Converting object to JSON string: ", processedRecord[key]);
|
|
406
|
+
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
691
407
|
}
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
const allMessages = [...messages, ...selectedMessages];
|
|
695
|
-
const seen = /* @__PURE__ */ new Set();
|
|
696
|
-
const dedupedMessages = allMessages.filter((m) => {
|
|
697
|
-
const key = `${m.id}:${m.thread_id}`;
|
|
698
|
-
if (seen.has(key)) return false;
|
|
699
|
-
seen.add(key);
|
|
700
|
-
return true;
|
|
701
|
-
});
|
|
702
|
-
const formattedMessages = dedupedMessages.map((msg) => {
|
|
703
|
-
const { thread_id, ...rest } = msg;
|
|
704
|
-
return {
|
|
705
|
-
...rest,
|
|
706
|
-
threadId: thread_id,
|
|
707
|
-
content: typeof msg.content === "string" ? (() => {
|
|
708
|
-
try {
|
|
709
|
-
return JSON.parse(msg.content);
|
|
710
|
-
} catch {
|
|
711
|
-
return msg.content;
|
|
712
|
-
}
|
|
713
|
-
})() : msg.content
|
|
714
|
-
};
|
|
715
|
-
});
|
|
716
|
-
const list = new MessageList().add(formattedMessages, "memory");
|
|
717
|
-
return {
|
|
718
|
-
messages: format === "v2" ? list.get.all.v2() : list.get.all.v1(),
|
|
719
|
-
total,
|
|
720
|
-
// Total should be the count of messages matching the filters
|
|
721
|
-
page,
|
|
722
|
-
perPage,
|
|
723
|
-
hasMore: total > (page + 1) * perPage
|
|
724
|
-
};
|
|
408
|
+
}
|
|
409
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([processedRecord]);
|
|
725
410
|
} catch (error) {
|
|
726
|
-
|
|
411
|
+
throw new MastraError(
|
|
727
412
|
{
|
|
728
|
-
id: "
|
|
413
|
+
id: createStorageErrorId("LANCE", "INSERT", "FAILED"),
|
|
729
414
|
domain: ErrorDomain.STORAGE,
|
|
730
415
|
category: ErrorCategory.THIRD_PARTY,
|
|
731
|
-
details: {
|
|
732
|
-
threadId,
|
|
733
|
-
resourceId: resourceId ?? ""
|
|
734
|
-
}
|
|
416
|
+
details: { tableName }
|
|
735
417
|
},
|
|
736
418
|
error
|
|
737
419
|
);
|
|
738
|
-
this.logger?.trackException?.(mastraError);
|
|
739
|
-
this.logger?.error?.(mastraError.toString());
|
|
740
|
-
return { messages: [], total: 0, page, perPage, hasMore: false };
|
|
741
420
|
}
|
|
742
421
|
}
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
if (!messages.length) {
|
|
766
|
-
return [];
|
|
422
|
+
async batchInsert({ tableName, records }) {
|
|
423
|
+
try {
|
|
424
|
+
if (!this.client) {
|
|
425
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
426
|
+
}
|
|
427
|
+
if (!tableName) {
|
|
428
|
+
throw new Error("tableName is required for batchInsert.");
|
|
429
|
+
}
|
|
430
|
+
if (!records || records.length === 0) {
|
|
431
|
+
throw new Error("records array is required and cannot be empty for batchInsert.");
|
|
432
|
+
}
|
|
433
|
+
} catch (validationError) {
|
|
434
|
+
throw new MastraError(
|
|
435
|
+
{
|
|
436
|
+
id: createStorageErrorId("LANCE", "BATCH_INSERT", "INVALID_ARGS"),
|
|
437
|
+
domain: ErrorDomain.STORAGE,
|
|
438
|
+
category: ErrorCategory.USER,
|
|
439
|
+
text: validationError.message,
|
|
440
|
+
details: { tableName }
|
|
441
|
+
},
|
|
442
|
+
validationError
|
|
443
|
+
);
|
|
767
444
|
}
|
|
768
|
-
const updatedMessages = [];
|
|
769
|
-
const affectedThreadIds = /* @__PURE__ */ new Set();
|
|
770
445
|
try {
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
continue;
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
const originalThreadId = existingMsg.threadId;
|
|
780
|
-
affectedThreadIds.add(originalThreadId);
|
|
781
|
-
const updatePayload = {};
|
|
782
|
-
if ("role" in updates && updates.role !== void 0) updatePayload.role = updates.role;
|
|
783
|
-
if ("type" in updates && updates.type !== void 0) updatePayload.type = updates.type;
|
|
784
|
-
if ("resourceId" in updates && updates.resourceId !== void 0) updatePayload.resourceId = updates.resourceId;
|
|
785
|
-
if ("threadId" in updates && updates.threadId !== void 0 && updates.threadId !== null) {
|
|
786
|
-
updatePayload.thread_id = updates.threadId;
|
|
787
|
-
affectedThreadIds.add(updates.threadId);
|
|
788
|
-
}
|
|
789
|
-
if (updates.content) {
|
|
790
|
-
const existingContent = existingMsg.content;
|
|
791
|
-
let newContent = { ...existingContent };
|
|
792
|
-
if (updates.content.metadata !== void 0) {
|
|
793
|
-
newContent.metadata = {
|
|
794
|
-
...existingContent.metadata || {},
|
|
795
|
-
...updates.content.metadata || {}
|
|
796
|
-
};
|
|
797
|
-
}
|
|
798
|
-
if (updates.content.content !== void 0) {
|
|
799
|
-
newContent.content = updates.content.content;
|
|
800
|
-
}
|
|
801
|
-
if ("parts" in updates.content && updates.content.parts !== void 0) {
|
|
802
|
-
newContent.parts = updates.content.parts;
|
|
446
|
+
const table = await this.client.openTable(tableName);
|
|
447
|
+
const primaryId = getPrimaryKeys(tableName);
|
|
448
|
+
const processedRecords = records.map((record) => {
|
|
449
|
+
const processedRecord = { ...record };
|
|
450
|
+
for (const key in processedRecord) {
|
|
451
|
+
if (processedRecord[key] == null) continue;
|
|
452
|
+
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
453
|
+
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
803
454
|
}
|
|
804
|
-
updatePayload.content = JSON.stringify(newContent);
|
|
805
455
|
}
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
updatedMessages.push(this.parseMessageData(updatedMessage));
|
|
810
|
-
}
|
|
811
|
-
}
|
|
812
|
-
for (const threadId of affectedThreadIds) {
|
|
813
|
-
await this.operations.insert({
|
|
814
|
-
tableName: TABLE_THREADS,
|
|
815
|
-
record: { id: threadId, updatedAt: Date.now() }
|
|
816
|
-
});
|
|
817
|
-
}
|
|
818
|
-
return updatedMessages;
|
|
456
|
+
return processedRecord;
|
|
457
|
+
});
|
|
458
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(processedRecords);
|
|
819
459
|
} catch (error) {
|
|
820
460
|
throw new MastraError(
|
|
821
461
|
{
|
|
822
|
-
id: "
|
|
462
|
+
id: createStorageErrorId("LANCE", "BATCH_INSERT", "FAILED"),
|
|
823
463
|
domain: ErrorDomain.STORAGE,
|
|
824
464
|
category: ErrorCategory.THIRD_PARTY,
|
|
825
|
-
details: {
|
|
465
|
+
details: { tableName }
|
|
826
466
|
},
|
|
827
467
|
error
|
|
828
468
|
);
|
|
829
469
|
}
|
|
830
470
|
}
|
|
831
|
-
async
|
|
471
|
+
async load({ tableName, keys }) {
|
|
832
472
|
try {
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
return null;
|
|
836
|
-
}
|
|
837
|
-
let createdAt;
|
|
838
|
-
let updatedAt;
|
|
839
|
-
try {
|
|
840
|
-
if (resource.createdAt instanceof Date) {
|
|
841
|
-
createdAt = resource.createdAt;
|
|
842
|
-
} else if (typeof resource.createdAt === "string") {
|
|
843
|
-
createdAt = new Date(resource.createdAt);
|
|
844
|
-
} else if (typeof resource.createdAt === "number") {
|
|
845
|
-
createdAt = new Date(resource.createdAt);
|
|
846
|
-
} else {
|
|
847
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
848
|
-
}
|
|
849
|
-
if (isNaN(createdAt.getTime())) {
|
|
850
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
851
|
-
}
|
|
852
|
-
} catch {
|
|
853
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
854
|
-
}
|
|
855
|
-
try {
|
|
856
|
-
if (resource.updatedAt instanceof Date) {
|
|
857
|
-
updatedAt = resource.updatedAt;
|
|
858
|
-
} else if (typeof resource.updatedAt === "string") {
|
|
859
|
-
updatedAt = new Date(resource.updatedAt);
|
|
860
|
-
} else if (typeof resource.updatedAt === "number") {
|
|
861
|
-
updatedAt = new Date(resource.updatedAt);
|
|
862
|
-
} else {
|
|
863
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
864
|
-
}
|
|
865
|
-
if (isNaN(updatedAt.getTime())) {
|
|
866
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
867
|
-
}
|
|
868
|
-
} catch {
|
|
869
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
473
|
+
if (!this.client) {
|
|
474
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
870
475
|
}
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
workingMemory = void 0;
|
|
874
|
-
} else if (workingMemory === "") {
|
|
875
|
-
workingMemory = "";
|
|
876
|
-
} else if (typeof workingMemory === "object") {
|
|
877
|
-
workingMemory = JSON.stringify(workingMemory);
|
|
476
|
+
if (!tableName) {
|
|
477
|
+
throw new Error("tableName is required for load.");
|
|
878
478
|
}
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
metadata = void 0;
|
|
882
|
-
} else if (typeof metadata === "string") {
|
|
883
|
-
try {
|
|
884
|
-
metadata = JSON.parse(metadata);
|
|
885
|
-
} catch {
|
|
886
|
-
metadata = metadata;
|
|
887
|
-
}
|
|
479
|
+
if (!keys || Object.keys(keys).length === 0) {
|
|
480
|
+
throw new Error("keys are required and cannot be empty for load.");
|
|
888
481
|
}
|
|
889
|
-
|
|
890
|
-
...resource,
|
|
891
|
-
createdAt,
|
|
892
|
-
updatedAt,
|
|
893
|
-
workingMemory,
|
|
894
|
-
metadata
|
|
895
|
-
};
|
|
896
|
-
} catch (error) {
|
|
482
|
+
} catch (validationError) {
|
|
897
483
|
throw new MastraError(
|
|
898
484
|
{
|
|
899
|
-
id: "
|
|
485
|
+
id: createStorageErrorId("LANCE", "LOAD", "INVALID_ARGS"),
|
|
900
486
|
domain: ErrorDomain.STORAGE,
|
|
901
|
-
category: ErrorCategory.
|
|
487
|
+
category: ErrorCategory.USER,
|
|
488
|
+
text: validationError.message,
|
|
489
|
+
details: { tableName }
|
|
902
490
|
},
|
|
903
|
-
|
|
491
|
+
validationError
|
|
904
492
|
);
|
|
905
493
|
}
|
|
906
|
-
}
|
|
907
|
-
async saveResource({ resource }) {
|
|
908
494
|
try {
|
|
909
|
-
const
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
495
|
+
const table = await this.client.openTable(tableName);
|
|
496
|
+
const tableSchema = await getTableSchema({ tableName, client: this.client });
|
|
497
|
+
const query = table.query();
|
|
498
|
+
if (Object.keys(keys).length > 0) {
|
|
499
|
+
validateKeyTypes(keys, tableSchema);
|
|
500
|
+
const filterConditions = Object.entries(keys).map(([key, value]) => {
|
|
501
|
+
const isCamelCase = /^[a-z][a-zA-Z]*$/.test(key) && /[A-Z]/.test(key);
|
|
502
|
+
const quotedKey = isCamelCase ? `\`${key}\`` : key;
|
|
503
|
+
if (typeof value === "string") {
|
|
504
|
+
return `${quotedKey} = '${value}'`;
|
|
505
|
+
} else if (value === null) {
|
|
506
|
+
return `${quotedKey} IS NULL`;
|
|
507
|
+
} else {
|
|
508
|
+
return `${quotedKey} = ${value}`;
|
|
509
|
+
}
|
|
510
|
+
}).join(" AND ");
|
|
511
|
+
this.logger.debug("where clause generated: " + filterConditions);
|
|
512
|
+
query.where(filterConditions);
|
|
513
|
+
}
|
|
514
|
+
const result = await query.limit(1).toArray();
|
|
515
|
+
if (result.length === 0) {
|
|
516
|
+
this.logger.debug("No record found");
|
|
517
|
+
return null;
|
|
518
|
+
}
|
|
519
|
+
return processResultWithTypeConversion(result[0], tableSchema);
|
|
920
520
|
} catch (error) {
|
|
521
|
+
if (error instanceof MastraError) throw error;
|
|
921
522
|
throw new MastraError(
|
|
922
523
|
{
|
|
923
|
-
id: "
|
|
524
|
+
id: createStorageErrorId("LANCE", "LOAD", "FAILED"),
|
|
924
525
|
domain: ErrorDomain.STORAGE,
|
|
925
|
-
category: ErrorCategory.THIRD_PARTY
|
|
526
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
527
|
+
details: { tableName, keyCount: Object.keys(keys).length, firstKey: Object.keys(keys)[0] ?? "" }
|
|
926
528
|
},
|
|
927
529
|
error
|
|
928
530
|
);
|
|
929
531
|
}
|
|
930
532
|
}
|
|
931
|
-
async updateResource({
|
|
932
|
-
resourceId,
|
|
933
|
-
workingMemory,
|
|
934
|
-
metadata
|
|
935
|
-
}) {
|
|
936
|
-
const maxRetries = 3;
|
|
937
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
938
|
-
try {
|
|
939
|
-
const existingResource = await this.getResourceById({ resourceId });
|
|
940
|
-
if (!existingResource) {
|
|
941
|
-
const newResource = {
|
|
942
|
-
id: resourceId,
|
|
943
|
-
workingMemory,
|
|
944
|
-
metadata: metadata || {},
|
|
945
|
-
createdAt: /* @__PURE__ */ new Date(),
|
|
946
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
947
|
-
};
|
|
948
|
-
return this.saveResource({ resource: newResource });
|
|
949
|
-
}
|
|
950
|
-
const updatedResource = {
|
|
951
|
-
...existingResource,
|
|
952
|
-
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
953
|
-
metadata: {
|
|
954
|
-
...existingResource.metadata,
|
|
955
|
-
...metadata
|
|
956
|
-
},
|
|
957
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
958
|
-
};
|
|
959
|
-
const record = {
|
|
960
|
-
id: resourceId,
|
|
961
|
-
workingMemory: updatedResource.workingMemory || "",
|
|
962
|
-
metadata: updatedResource.metadata ? JSON.stringify(updatedResource.metadata) : "",
|
|
963
|
-
updatedAt: updatedResource.updatedAt.getTime()
|
|
964
|
-
// Store as timestamp (milliseconds)
|
|
965
|
-
};
|
|
966
|
-
const table = await this.client.openTable(TABLE_RESOURCES);
|
|
967
|
-
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
968
|
-
return updatedResource;
|
|
969
|
-
} catch (error) {
|
|
970
|
-
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
971
|
-
const delay = Math.pow(2, attempt) * 10;
|
|
972
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
973
|
-
continue;
|
|
974
|
-
}
|
|
975
|
-
throw new MastraError(
|
|
976
|
-
{
|
|
977
|
-
id: "LANCE_STORE_UPDATE_RESOURCE_FAILED",
|
|
978
|
-
domain: ErrorDomain.STORAGE,
|
|
979
|
-
category: ErrorCategory.THIRD_PARTY
|
|
980
|
-
},
|
|
981
|
-
error
|
|
982
|
-
);
|
|
983
|
-
}
|
|
984
|
-
}
|
|
985
|
-
throw new Error("Unexpected end of retry loop");
|
|
986
|
-
}
|
|
987
533
|
};
|
|
988
|
-
|
|
534
|
+
|
|
535
|
+
// src/storage/domains/memory/index.ts
|
|
536
|
+
var StoreMemoryLance = class extends MemoryStorage {
|
|
989
537
|
client;
|
|
990
|
-
|
|
538
|
+
#db;
|
|
539
|
+
constructor(config) {
|
|
991
540
|
super();
|
|
541
|
+
const client = resolveLanceConfig(config);
|
|
992
542
|
this.client = client;
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
case "jsonb":
|
|
1004
|
-
return "'{}'";
|
|
1005
|
-
case "uuid":
|
|
1006
|
-
return "''";
|
|
1007
|
-
default:
|
|
1008
|
-
return super.getDefaultValue(type);
|
|
1009
|
-
}
|
|
1010
|
-
}
|
|
1011
|
-
async hasColumn(tableName, columnName) {
|
|
1012
|
-
const table = await this.client.openTable(tableName);
|
|
1013
|
-
const schema = await table.schema();
|
|
1014
|
-
return schema.fields.some((field) => field.name === columnName);
|
|
1015
|
-
}
|
|
1016
|
-
translateSchema(schema) {
|
|
1017
|
-
const fields = Object.entries(schema).map(([name, column]) => {
|
|
1018
|
-
let arrowType;
|
|
1019
|
-
switch (column.type.toLowerCase()) {
|
|
1020
|
-
case "text":
|
|
1021
|
-
case "uuid":
|
|
1022
|
-
arrowType = new Utf8();
|
|
1023
|
-
break;
|
|
1024
|
-
case "int":
|
|
1025
|
-
case "integer":
|
|
1026
|
-
arrowType = new Int32();
|
|
1027
|
-
break;
|
|
1028
|
-
case "bigint":
|
|
1029
|
-
arrowType = new Float64();
|
|
1030
|
-
break;
|
|
1031
|
-
case "float":
|
|
1032
|
-
arrowType = new Float32();
|
|
1033
|
-
break;
|
|
1034
|
-
case "jsonb":
|
|
1035
|
-
case "json":
|
|
1036
|
-
arrowType = new Utf8();
|
|
1037
|
-
break;
|
|
1038
|
-
case "binary":
|
|
1039
|
-
arrowType = new Binary();
|
|
1040
|
-
break;
|
|
1041
|
-
case "timestamp":
|
|
1042
|
-
arrowType = new Float64();
|
|
1043
|
-
break;
|
|
1044
|
-
default:
|
|
1045
|
-
arrowType = new Utf8();
|
|
1046
|
-
}
|
|
1047
|
-
return new Field(name, arrowType, column.nullable ?? true);
|
|
543
|
+
this.#db = new LanceDB({ client });
|
|
544
|
+
}
|
|
545
|
+
async init() {
|
|
546
|
+
await this.#db.createTable({ tableName: TABLE_THREADS, schema: TABLE_SCHEMAS[TABLE_THREADS] });
|
|
547
|
+
await this.#db.createTable({ tableName: TABLE_MESSAGES, schema: TABLE_SCHEMAS[TABLE_MESSAGES] });
|
|
548
|
+
await this.#db.createTable({ tableName: TABLE_RESOURCES, schema: TABLE_SCHEMAS[TABLE_RESOURCES] });
|
|
549
|
+
await this.#db.alterTable({
|
|
550
|
+
tableName: TABLE_MESSAGES,
|
|
551
|
+
schema: TABLE_SCHEMAS[TABLE_MESSAGES],
|
|
552
|
+
ifNotExists: ["resourceId"]
|
|
1048
553
|
});
|
|
1049
|
-
return new Schema(fields);
|
|
1050
554
|
}
|
|
1051
|
-
async
|
|
1052
|
-
tableName
|
|
1053
|
-
|
|
1054
|
-
|
|
555
|
+
async dangerouslyClearAll() {
|
|
556
|
+
await this.#db.clearTable({ tableName: TABLE_THREADS });
|
|
557
|
+
await this.#db.clearTable({ tableName: TABLE_MESSAGES });
|
|
558
|
+
await this.#db.clearTable({ tableName: TABLE_RESOURCES });
|
|
559
|
+
}
|
|
560
|
+
async deleteMessages(messageIds) {
|
|
561
|
+
if (!messageIds || messageIds.length === 0) {
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
this.logger.debug("Deleting messages", { count: messageIds.length });
|
|
1055
565
|
try {
|
|
1056
|
-
|
|
1057
|
-
|
|
566
|
+
const threadIds = /* @__PURE__ */ new Set();
|
|
567
|
+
for (const messageId of messageIds) {
|
|
568
|
+
const message = await this.#db.load({ tableName: TABLE_MESSAGES, keys: { id: messageId } });
|
|
569
|
+
if (message?.thread_id) {
|
|
570
|
+
threadIds.add(message.thread_id);
|
|
571
|
+
}
|
|
1058
572
|
}
|
|
1059
|
-
|
|
1060
|
-
|
|
573
|
+
const messagesTable = await this.client.openTable(TABLE_MESSAGES);
|
|
574
|
+
const idConditions = messageIds.map((id) => `id = '${this.escapeSql(id)}'`).join(" OR ");
|
|
575
|
+
await messagesTable.delete(idConditions);
|
|
576
|
+
const now = (/* @__PURE__ */ new Date()).getTime();
|
|
577
|
+
const threadsTable = await this.client.openTable(TABLE_THREADS);
|
|
578
|
+
for (const threadId of threadIds) {
|
|
579
|
+
const thread = await this.getThreadById({ threadId });
|
|
580
|
+
if (thread) {
|
|
581
|
+
const record = {
|
|
582
|
+
id: threadId,
|
|
583
|
+
resourceId: thread.resourceId,
|
|
584
|
+
title: thread.title,
|
|
585
|
+
metadata: JSON.stringify(thread.metadata),
|
|
586
|
+
createdAt: new Date(thread.createdAt).getTime(),
|
|
587
|
+
updatedAt: now
|
|
588
|
+
};
|
|
589
|
+
await threadsTable.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
590
|
+
}
|
|
1061
591
|
}
|
|
1062
|
-
|
|
1063
|
-
|
|
592
|
+
} catch (error) {
|
|
593
|
+
throw new MastraError(
|
|
594
|
+
{
|
|
595
|
+
id: createStorageErrorId("LANCE", "DELETE_MESSAGES", "FAILED"),
|
|
596
|
+
domain: ErrorDomain.STORAGE,
|
|
597
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
598
|
+
details: { count: messageIds.length }
|
|
599
|
+
},
|
|
600
|
+
error
|
|
601
|
+
);
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
// Utility to escape single quotes in SQL strings
|
|
605
|
+
escapeSql(str) {
|
|
606
|
+
return str.replace(/'/g, "''");
|
|
607
|
+
}
|
|
608
|
+
async getThreadById({ threadId }) {
|
|
609
|
+
try {
|
|
610
|
+
const thread = await this.#db.load({ tableName: TABLE_THREADS, keys: { id: threadId } });
|
|
611
|
+
if (!thread) {
|
|
612
|
+
return null;
|
|
1064
613
|
}
|
|
614
|
+
return {
|
|
615
|
+
...thread,
|
|
616
|
+
createdAt: new Date(thread.createdAt),
|
|
617
|
+
updatedAt: new Date(thread.updatedAt)
|
|
618
|
+
};
|
|
1065
619
|
} catch (error) {
|
|
1066
620
|
throw new MastraError(
|
|
1067
621
|
{
|
|
1068
|
-
id: "
|
|
622
|
+
id: createStorageErrorId("LANCE", "GET_THREAD_BY_ID", "FAILED"),
|
|
1069
623
|
domain: ErrorDomain.STORAGE,
|
|
1070
|
-
category: ErrorCategory.
|
|
1071
|
-
details: { tableName }
|
|
624
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1072
625
|
},
|
|
1073
626
|
error
|
|
1074
627
|
);
|
|
1075
628
|
}
|
|
629
|
+
}
|
|
630
|
+
/**
|
|
631
|
+
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
632
|
+
* @param thread - The thread to save
|
|
633
|
+
* @returns The saved thread
|
|
634
|
+
*/
|
|
635
|
+
async saveThread({ thread }) {
|
|
1076
636
|
try {
|
|
1077
|
-
const
|
|
1078
|
-
await this.client.
|
|
637
|
+
const record = { ...thread, metadata: JSON.stringify(thread.metadata) };
|
|
638
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
639
|
+
await table.add([record], { mode: "append" });
|
|
640
|
+
return thread;
|
|
1079
641
|
} catch (error) {
|
|
1080
|
-
if (error.message?.includes("already exists")) {
|
|
1081
|
-
this.logger.debug(`Table '${tableName}' already exists, skipping create`);
|
|
1082
|
-
return;
|
|
1083
|
-
}
|
|
1084
642
|
throw new MastraError(
|
|
1085
643
|
{
|
|
1086
|
-
id: "
|
|
644
|
+
id: createStorageErrorId("LANCE", "SAVE_THREAD", "FAILED"),
|
|
1087
645
|
domain: ErrorDomain.STORAGE,
|
|
1088
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1089
|
-
details: { tableName }
|
|
646
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1090
647
|
},
|
|
1091
648
|
error
|
|
1092
649
|
);
|
|
1093
650
|
}
|
|
1094
651
|
}
|
|
1095
|
-
async
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
652
|
+
async updateThread({
|
|
653
|
+
id,
|
|
654
|
+
title,
|
|
655
|
+
metadata
|
|
656
|
+
}) {
|
|
657
|
+
const maxRetries = 5;
|
|
658
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
659
|
+
try {
|
|
660
|
+
const current = await this.getThreadById({ threadId: id });
|
|
661
|
+
if (!current) {
|
|
662
|
+
throw new Error(`Thread with id ${id} not found`);
|
|
663
|
+
}
|
|
664
|
+
const mergedMetadata = { ...current.metadata, ...metadata };
|
|
665
|
+
const record = {
|
|
666
|
+
id,
|
|
667
|
+
title,
|
|
668
|
+
metadata: JSON.stringify(mergedMetadata),
|
|
669
|
+
updatedAt: (/* @__PURE__ */ new Date()).getTime()
|
|
670
|
+
};
|
|
671
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
672
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
673
|
+
const updatedThread = await this.getThreadById({ threadId: id });
|
|
674
|
+
if (!updatedThread) {
|
|
675
|
+
throw new Error(`Failed to retrieve updated thread ${id}`);
|
|
676
|
+
}
|
|
677
|
+
return updatedThread;
|
|
678
|
+
} catch (error) {
|
|
679
|
+
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
680
|
+
const delay = Math.pow(2, attempt) * 10;
|
|
681
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
682
|
+
continue;
|
|
683
|
+
}
|
|
684
|
+
throw new MastraError(
|
|
685
|
+
{
|
|
686
|
+
id: createStorageErrorId("LANCE", "UPDATE_THREAD", "FAILED"),
|
|
687
|
+
domain: ErrorDomain.STORAGE,
|
|
688
|
+
category: ErrorCategory.THIRD_PARTY
|
|
689
|
+
},
|
|
690
|
+
error
|
|
691
|
+
);
|
|
1102
692
|
}
|
|
1103
|
-
}
|
|
693
|
+
}
|
|
694
|
+
throw new MastraError(
|
|
695
|
+
{
|
|
696
|
+
id: createStorageErrorId("LANCE", "UPDATE_THREAD", "FAILED"),
|
|
697
|
+
domain: ErrorDomain.STORAGE,
|
|
698
|
+
category: ErrorCategory.THIRD_PARTY
|
|
699
|
+
},
|
|
700
|
+
new Error("All retries exhausted")
|
|
701
|
+
);
|
|
702
|
+
}
|
|
703
|
+
async deleteThread({ threadId }) {
|
|
704
|
+
try {
|
|
705
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
706
|
+
await table.delete(`id = '${threadId}'`);
|
|
707
|
+
const messagesTable = await this.client.openTable(TABLE_MESSAGES);
|
|
708
|
+
await messagesTable.delete(`thread_id = '${threadId}'`);
|
|
709
|
+
} catch (error) {
|
|
1104
710
|
throw new MastraError(
|
|
1105
711
|
{
|
|
1106
|
-
id: "
|
|
712
|
+
id: createStorageErrorId("LANCE", "DELETE_THREAD", "FAILED"),
|
|
1107
713
|
domain: ErrorDomain.STORAGE,
|
|
1108
|
-
category: ErrorCategory.
|
|
1109
|
-
text: validationError.message,
|
|
1110
|
-
details: { tableName }
|
|
714
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1111
715
|
},
|
|
1112
|
-
|
|
716
|
+
error
|
|
1113
717
|
);
|
|
1114
718
|
}
|
|
719
|
+
}
|
|
720
|
+
normalizeMessage(message) {
|
|
721
|
+
const { thread_id, ...rest } = message;
|
|
722
|
+
return {
|
|
723
|
+
...rest,
|
|
724
|
+
threadId: thread_id,
|
|
725
|
+
content: typeof message.content === "string" ? (() => {
|
|
726
|
+
try {
|
|
727
|
+
return JSON.parse(message.content);
|
|
728
|
+
} catch {
|
|
729
|
+
return message.content;
|
|
730
|
+
}
|
|
731
|
+
})() : message.content
|
|
732
|
+
};
|
|
733
|
+
}
|
|
734
|
+
async listMessagesById({ messageIds }) {
|
|
735
|
+
if (messageIds.length === 0) return { messages: [] };
|
|
1115
736
|
try {
|
|
1116
|
-
await this.client.
|
|
737
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
738
|
+
const quotedIds = messageIds.map((id) => `'${id}'`).join(", ");
|
|
739
|
+
const allRecords = await table.query().where(`id IN (${quotedIds})`).toArray();
|
|
740
|
+
const messages = processResultWithTypeConversion(
|
|
741
|
+
allRecords,
|
|
742
|
+
await getTableSchema({ tableName: TABLE_MESSAGES, client: this.client })
|
|
743
|
+
);
|
|
744
|
+
const list = new MessageList().add(
|
|
745
|
+
messages.map(this.normalizeMessage),
|
|
746
|
+
"memory"
|
|
747
|
+
);
|
|
748
|
+
return { messages: list.get.all.db() };
|
|
1117
749
|
} catch (error) {
|
|
1118
|
-
if (error.toString().includes("was not found") || error.message?.includes("Table not found")) {
|
|
1119
|
-
this.logger.debug(`Table '${tableName}' does not exist, skipping drop`);
|
|
1120
|
-
return;
|
|
1121
|
-
}
|
|
1122
750
|
throw new MastraError(
|
|
1123
751
|
{
|
|
1124
|
-
id: "
|
|
752
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES_BY_ID", "FAILED"),
|
|
1125
753
|
domain: ErrorDomain.STORAGE,
|
|
1126
754
|
category: ErrorCategory.THIRD_PARTY,
|
|
1127
|
-
details: {
|
|
755
|
+
details: {
|
|
756
|
+
messageIds: JSON.stringify(messageIds)
|
|
757
|
+
}
|
|
1128
758
|
},
|
|
1129
759
|
error
|
|
1130
760
|
);
|
|
1131
761
|
}
|
|
1132
762
|
}
|
|
1133
|
-
async
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
}) {
|
|
1138
|
-
try {
|
|
1139
|
-
if (!this.client) {
|
|
1140
|
-
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1141
|
-
}
|
|
1142
|
-
if (!tableName) {
|
|
1143
|
-
throw new Error("tableName is required for alterTable.");
|
|
1144
|
-
}
|
|
1145
|
-
if (!schema) {
|
|
1146
|
-
throw new Error("schema is required for alterTable.");
|
|
1147
|
-
}
|
|
1148
|
-
if (!ifNotExists || ifNotExists.length === 0) {
|
|
1149
|
-
this.logger.debug("No columns specified to add in alterTable, skipping.");
|
|
1150
|
-
return;
|
|
1151
|
-
}
|
|
1152
|
-
} catch (validationError) {
|
|
763
|
+
async listMessages(args) {
|
|
764
|
+
const { threadId, resourceId, include, filter, perPage: perPageInput, page = 0, orderBy } = args;
|
|
765
|
+
const threadIds = Array.isArray(threadId) ? threadId : [threadId];
|
|
766
|
+
if (threadIds.length === 0 || threadIds.some((id) => !id.trim())) {
|
|
1153
767
|
throw new MastraError(
|
|
1154
768
|
{
|
|
1155
|
-
id: "
|
|
769
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES", "INVALID_THREAD_ID"),
|
|
1156
770
|
domain: ErrorDomain.STORAGE,
|
|
1157
|
-
category: ErrorCategory.
|
|
1158
|
-
|
|
1159
|
-
details: { tableName }
|
|
771
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
772
|
+
details: { threadId: Array.isArray(threadId) ? threadId.join(",") : threadId }
|
|
1160
773
|
},
|
|
1161
|
-
|
|
774
|
+
new Error("threadId must be a non-empty string or array of non-empty strings")
|
|
1162
775
|
);
|
|
1163
776
|
}
|
|
777
|
+
const perPage = normalizePerPage(perPageInput, 40);
|
|
778
|
+
const { offset, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
1164
779
|
try {
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
}
|
|
1176
|
-
const
|
|
1177
|
-
|
|
780
|
+
if (page < 0) {
|
|
781
|
+
throw new MastraError(
|
|
782
|
+
{
|
|
783
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES", "INVALID_PAGE"),
|
|
784
|
+
domain: ErrorDomain.STORAGE,
|
|
785
|
+
category: ErrorCategory.USER,
|
|
786
|
+
details: { page }
|
|
787
|
+
},
|
|
788
|
+
new Error("page must be >= 0")
|
|
789
|
+
);
|
|
790
|
+
}
|
|
791
|
+
const { field, direction } = this.parseOrderBy(orderBy, "ASC");
|
|
792
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
793
|
+
const threadCondition = threadIds.length === 1 ? `thread_id = '${this.escapeSql(threadIds[0])}'` : `thread_id IN (${threadIds.map((t) => `'${this.escapeSql(t)}'`).join(", ")})`;
|
|
794
|
+
const conditions = [threadCondition];
|
|
795
|
+
if (resourceId) {
|
|
796
|
+
conditions.push(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
797
|
+
}
|
|
798
|
+
if (filter?.dateRange?.start) {
|
|
799
|
+
const startTime = filter.dateRange.start instanceof Date ? filter.dateRange.start.getTime() : new Date(filter.dateRange.start).getTime();
|
|
800
|
+
const startOp = filter.dateRange.startExclusive ? ">" : ">=";
|
|
801
|
+
conditions.push(`\`createdAt\` ${startOp} ${startTime}`);
|
|
802
|
+
}
|
|
803
|
+
if (filter?.dateRange?.end) {
|
|
804
|
+
const endTime = filter.dateRange.end instanceof Date ? filter.dateRange.end.getTime() : new Date(filter.dateRange.end).getTime();
|
|
805
|
+
const endOp = filter.dateRange.endExclusive ? "<" : "<=";
|
|
806
|
+
conditions.push(`\`createdAt\` ${endOp} ${endTime}`);
|
|
807
|
+
}
|
|
808
|
+
const whereClause = conditions.join(" AND ");
|
|
809
|
+
const total = await table.countRows(whereClause);
|
|
810
|
+
const query = table.query().where(whereClause);
|
|
811
|
+
let allRecords = await query.toArray();
|
|
812
|
+
allRecords.sort((a, b) => {
|
|
813
|
+
const aValue = field === "createdAt" ? a.createdAt : a[field];
|
|
814
|
+
const bValue = field === "createdAt" ? b.createdAt : b[field];
|
|
815
|
+
if (aValue == null && bValue == null) return 0;
|
|
816
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
817
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
818
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
819
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
820
|
+
}
|
|
821
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
822
|
+
});
|
|
823
|
+
const paginatedRecords = allRecords.slice(offset, offset + perPage);
|
|
824
|
+
const messages = paginatedRecords.map((row) => this.normalizeMessage(row));
|
|
825
|
+
if (total === 0 && messages.length === 0 && (!include || include.length === 0)) {
|
|
1178
826
|
return {
|
|
1179
|
-
|
|
1180
|
-
|
|
827
|
+
messages: [],
|
|
828
|
+
total: 0,
|
|
829
|
+
page,
|
|
830
|
+
perPage: perPageForResponse,
|
|
831
|
+
hasMore: false
|
|
1181
832
|
};
|
|
1182
|
-
});
|
|
1183
|
-
if (columnsToAdd.length > 0) {
|
|
1184
|
-
await table.addColumns(columnsToAdd);
|
|
1185
|
-
this.logger?.info?.(`Added columns [${columnsToAdd.map((c) => c.name).join(", ")}] to table ${tableName}`);
|
|
1186
833
|
}
|
|
834
|
+
const messageIds = new Set(messages.map((m) => m.id));
|
|
835
|
+
if (include && include.length > 0) {
|
|
836
|
+
const threadIds2 = [...new Set(include.map((item) => item.threadId || threadId))];
|
|
837
|
+
const allThreadMessages = [];
|
|
838
|
+
for (const tid of threadIds2) {
|
|
839
|
+
const threadQuery = table.query().where(`thread_id = '${tid}'`);
|
|
840
|
+
let threadRecords = await threadQuery.toArray();
|
|
841
|
+
allThreadMessages.push(...threadRecords);
|
|
842
|
+
}
|
|
843
|
+
allThreadMessages.sort((a, b) => a.createdAt - b.createdAt);
|
|
844
|
+
const contextMessages = this.processMessagesWithContext(allThreadMessages, include);
|
|
845
|
+
const includedMessages = contextMessages.map((row) => this.normalizeMessage(row));
|
|
846
|
+
for (const includeMsg of includedMessages) {
|
|
847
|
+
if (!messageIds.has(includeMsg.id)) {
|
|
848
|
+
messages.push(includeMsg);
|
|
849
|
+
messageIds.add(includeMsg.id);
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
const list = new MessageList().add(messages, "memory");
|
|
854
|
+
let finalMessages = list.get.all.db();
|
|
855
|
+
finalMessages = finalMessages.sort((a, b) => {
|
|
856
|
+
const aValue = field === "createdAt" ? new Date(a.createdAt).getTime() : a[field];
|
|
857
|
+
const bValue = field === "createdAt" ? new Date(b.createdAt).getTime() : b[field];
|
|
858
|
+
if (aValue == null && bValue == null) return 0;
|
|
859
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
860
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
861
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
862
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
863
|
+
}
|
|
864
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
865
|
+
});
|
|
866
|
+
const returnedThreadMessageIds = new Set(finalMessages.filter((m) => m.threadId === threadId).map((m) => m.id));
|
|
867
|
+
const allThreadMessagesReturned = returnedThreadMessageIds.size >= total;
|
|
868
|
+
const fetchedAll = perPageInput === false || allThreadMessagesReturned;
|
|
869
|
+
const hasMore = !fetchedAll && offset + perPage < total;
|
|
870
|
+
return {
|
|
871
|
+
messages: finalMessages,
|
|
872
|
+
total,
|
|
873
|
+
page,
|
|
874
|
+
perPage: perPageForResponse,
|
|
875
|
+
hasMore
|
|
876
|
+
};
|
|
1187
877
|
} catch (error) {
|
|
1188
|
-
|
|
878
|
+
const mastraError = new MastraError(
|
|
1189
879
|
{
|
|
1190
|
-
id: "
|
|
880
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES", "FAILED"),
|
|
1191
881
|
domain: ErrorDomain.STORAGE,
|
|
1192
882
|
category: ErrorCategory.THIRD_PARTY,
|
|
1193
|
-
details: {
|
|
883
|
+
details: {
|
|
884
|
+
threadId: Array.isArray(threadId) ? threadId.join(",") : threadId,
|
|
885
|
+
resourceId: resourceId ?? ""
|
|
886
|
+
}
|
|
1194
887
|
},
|
|
1195
888
|
error
|
|
1196
889
|
);
|
|
890
|
+
this.logger?.error?.(mastraError.toString());
|
|
891
|
+
this.logger?.trackException?.(mastraError);
|
|
892
|
+
return {
|
|
893
|
+
messages: [],
|
|
894
|
+
total: 0,
|
|
895
|
+
page,
|
|
896
|
+
perPage: perPageForResponse,
|
|
897
|
+
hasMore: false
|
|
898
|
+
};
|
|
1197
899
|
}
|
|
1198
900
|
}
|
|
1199
|
-
async
|
|
901
|
+
async saveMessages(args) {
|
|
1200
902
|
try {
|
|
1201
|
-
|
|
1202
|
-
|
|
903
|
+
const { messages } = args;
|
|
904
|
+
if (messages.length === 0) {
|
|
905
|
+
return { messages: [] };
|
|
1203
906
|
}
|
|
1204
|
-
|
|
1205
|
-
|
|
907
|
+
const threadId = messages[0]?.threadId;
|
|
908
|
+
if (!threadId) {
|
|
909
|
+
throw new Error("Thread ID is required");
|
|
1206
910
|
}
|
|
1207
|
-
|
|
911
|
+
for (const message of messages) {
|
|
912
|
+
if (!message.id) {
|
|
913
|
+
throw new Error("Message ID is required");
|
|
914
|
+
}
|
|
915
|
+
if (!message.threadId) {
|
|
916
|
+
throw new Error("Thread ID is required for all messages");
|
|
917
|
+
}
|
|
918
|
+
if (message.resourceId === null || message.resourceId === void 0) {
|
|
919
|
+
throw new Error("Resource ID cannot be null or undefined");
|
|
920
|
+
}
|
|
921
|
+
if (!message.content) {
|
|
922
|
+
throw new Error("Message content is required");
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
const transformedMessages = messages.map((message) => {
|
|
926
|
+
const { threadId: threadId2, type, ...rest } = message;
|
|
927
|
+
return {
|
|
928
|
+
...rest,
|
|
929
|
+
thread_id: threadId2,
|
|
930
|
+
type: type ?? "v2",
|
|
931
|
+
content: JSON.stringify(message.content)
|
|
932
|
+
};
|
|
933
|
+
});
|
|
934
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
935
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(transformedMessages);
|
|
936
|
+
const threadsTable = await this.client.openTable(TABLE_THREADS);
|
|
937
|
+
const currentTime = (/* @__PURE__ */ new Date()).getTime();
|
|
938
|
+
const updateRecord = { id: threadId, updatedAt: currentTime };
|
|
939
|
+
await threadsTable.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([updateRecord]);
|
|
940
|
+
const list = new MessageList().add(messages, "memory");
|
|
941
|
+
return { messages: list.get.all.db() };
|
|
942
|
+
} catch (error) {
|
|
1208
943
|
throw new MastraError(
|
|
1209
944
|
{
|
|
1210
|
-
id: "
|
|
945
|
+
id: createStorageErrorId("LANCE", "SAVE_MESSAGES", "FAILED"),
|
|
1211
946
|
domain: ErrorDomain.STORAGE,
|
|
1212
|
-
category: ErrorCategory.
|
|
1213
|
-
text: validationError.message,
|
|
1214
|
-
details: { tableName }
|
|
947
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1215
948
|
},
|
|
1216
|
-
|
|
949
|
+
error
|
|
1217
950
|
);
|
|
1218
951
|
}
|
|
952
|
+
}
|
|
953
|
+
async listThreads(args) {
|
|
954
|
+
const { page = 0, perPage: perPageInput, orderBy, filter } = args;
|
|
1219
955
|
try {
|
|
1220
|
-
|
|
1221
|
-
await table.delete("1=1");
|
|
956
|
+
this.validatePaginationInput(page, perPageInput ?? 100);
|
|
1222
957
|
} catch (error) {
|
|
1223
958
|
throw new MastraError(
|
|
1224
959
|
{
|
|
1225
|
-
id: "
|
|
960
|
+
id: createStorageErrorId("LANCE", "LIST_THREADS", "INVALID_PAGE"),
|
|
1226
961
|
domain: ErrorDomain.STORAGE,
|
|
1227
|
-
category: ErrorCategory.
|
|
1228
|
-
details: {
|
|
962
|
+
category: ErrorCategory.USER,
|
|
963
|
+
details: { page, ...perPageInput !== void 0 && { perPage: perPageInput } }
|
|
1229
964
|
},
|
|
1230
|
-
error
|
|
965
|
+
error instanceof Error ? error : new Error("Invalid pagination parameters")
|
|
1231
966
|
);
|
|
1232
967
|
}
|
|
1233
|
-
|
|
1234
|
-
async insert({ tableName, record }) {
|
|
968
|
+
const perPage = normalizePerPage(perPageInput, 100);
|
|
1235
969
|
try {
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
}
|
|
1239
|
-
if (!tableName) {
|
|
1240
|
-
throw new Error("tableName is required for insert.");
|
|
1241
|
-
}
|
|
1242
|
-
if (!record || Object.keys(record).length === 0) {
|
|
1243
|
-
throw new Error("record is required and cannot be empty for insert.");
|
|
1244
|
-
}
|
|
1245
|
-
} catch (validationError) {
|
|
970
|
+
this.validateMetadataKeys(filter?.metadata);
|
|
971
|
+
} catch (error) {
|
|
1246
972
|
throw new MastraError(
|
|
1247
973
|
{
|
|
1248
|
-
id: "
|
|
974
|
+
id: createStorageErrorId("LANCE", "LIST_THREADS", "INVALID_METADATA_KEY"),
|
|
1249
975
|
domain: ErrorDomain.STORAGE,
|
|
1250
976
|
category: ErrorCategory.USER,
|
|
1251
|
-
|
|
1252
|
-
details: { tableName }
|
|
977
|
+
details: { metadataKeys: filter?.metadata ? Object.keys(filter.metadata).join(", ") : "" }
|
|
1253
978
|
},
|
|
1254
|
-
|
|
979
|
+
error instanceof Error ? error : new Error("Invalid metadata key")
|
|
1255
980
|
);
|
|
1256
981
|
}
|
|
1257
982
|
try {
|
|
1258
|
-
const
|
|
1259
|
-
const
|
|
1260
|
-
const
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
983
|
+
const { offset, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
984
|
+
const { field, direction } = this.parseOrderBy(orderBy);
|
|
985
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
986
|
+
const whereClauses = [];
|
|
987
|
+
if (filter?.resourceId) {
|
|
988
|
+
whereClauses.push(`\`resourceId\` = '${this.escapeSql(filter.resourceId)}'`);
|
|
989
|
+
}
|
|
990
|
+
const whereClause = whereClauses.length > 0 ? whereClauses.join(" AND ") : "";
|
|
991
|
+
const query = whereClause ? table.query().where(whereClause) : table.query();
|
|
992
|
+
let records = await query.toArray();
|
|
993
|
+
if (filter?.metadata && Object.keys(filter.metadata).length > 0) {
|
|
994
|
+
records = records.filter((record) => {
|
|
995
|
+
if (!record.metadata) return false;
|
|
996
|
+
let recordMeta;
|
|
997
|
+
if (typeof record.metadata === "string") {
|
|
998
|
+
try {
|
|
999
|
+
recordMeta = JSON.parse(record.metadata);
|
|
1000
|
+
} catch {
|
|
1001
|
+
return false;
|
|
1002
|
+
}
|
|
1003
|
+
} else {
|
|
1004
|
+
recordMeta = record.metadata;
|
|
1005
|
+
}
|
|
1006
|
+
return Object.entries(filter.metadata).every(([key, value]) => recordMeta[key] === value);
|
|
1007
|
+
});
|
|
1266
1008
|
}
|
|
1267
|
-
|
|
1268
|
-
|
|
1009
|
+
const total = records.length;
|
|
1010
|
+
records.sort((a, b) => {
|
|
1011
|
+
const aValue = ["createdAt", "updatedAt"].includes(field) ? new Date(a[field]).getTime() : a[field];
|
|
1012
|
+
const bValue = ["createdAt", "updatedAt"].includes(field) ? new Date(b[field]).getTime() : b[field];
|
|
1013
|
+
if (aValue == null && bValue == null) return 0;
|
|
1014
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
1015
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
1016
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
1017
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
1018
|
+
}
|
|
1019
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
1020
|
+
});
|
|
1021
|
+
const paginatedRecords = records.slice(offset, offset + perPage);
|
|
1022
|
+
const schema = await getTableSchema({ tableName: TABLE_THREADS, client: this.client });
|
|
1023
|
+
const threads = paginatedRecords.map(
|
|
1024
|
+
(record) => processResultWithTypeConversion(record, schema)
|
|
1025
|
+
);
|
|
1026
|
+
return {
|
|
1027
|
+
threads,
|
|
1028
|
+
total,
|
|
1029
|
+
page,
|
|
1030
|
+
perPage: perPageForResponse,
|
|
1031
|
+
hasMore: offset + perPage < total
|
|
1032
|
+
};
|
|
1269
1033
|
} catch (error) {
|
|
1270
1034
|
throw new MastraError(
|
|
1271
1035
|
{
|
|
1272
|
-
id: "
|
|
1036
|
+
id: createStorageErrorId("LANCE", "LIST_THREADS", "FAILED"),
|
|
1273
1037
|
domain: ErrorDomain.STORAGE,
|
|
1274
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1275
|
-
details: { tableName }
|
|
1038
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1276
1039
|
},
|
|
1277
1040
|
error
|
|
1278
1041
|
);
|
|
1279
1042
|
}
|
|
1280
1043
|
}
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1044
|
+
/**
|
|
1045
|
+
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
1046
|
+
* @param records - The sorted array of records to process
|
|
1047
|
+
* @param include - The array of include specifications with context parameters
|
|
1048
|
+
* @returns The processed array with context messages included
|
|
1049
|
+
*/
|
|
1050
|
+
processMessagesWithContext(records, include) {
|
|
1051
|
+
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
1052
|
+
if (messagesWithContext.length === 0) {
|
|
1053
|
+
return records;
|
|
1054
|
+
}
|
|
1055
|
+
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
1056
|
+
records.forEach((message, index) => {
|
|
1057
|
+
messageIndexMap.set(message.id, index);
|
|
1058
|
+
});
|
|
1059
|
+
const additionalIndices = /* @__PURE__ */ new Set();
|
|
1060
|
+
for (const item of messagesWithContext) {
|
|
1061
|
+
const messageIndex = messageIndexMap.get(item.id);
|
|
1062
|
+
if (messageIndex !== void 0) {
|
|
1063
|
+
if (item.withPreviousMessages) {
|
|
1064
|
+
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
1065
|
+
for (let i = startIdx; i < messageIndex; i++) {
|
|
1066
|
+
additionalIndices.add(i);
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
if (item.withNextMessages) {
|
|
1070
|
+
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
1071
|
+
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
1072
|
+
additionalIndices.add(i);
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1285
1075
|
}
|
|
1286
|
-
|
|
1287
|
-
|
|
1076
|
+
}
|
|
1077
|
+
if (additionalIndices.size === 0) {
|
|
1078
|
+
return records;
|
|
1079
|
+
}
|
|
1080
|
+
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
1081
|
+
const allIndices = /* @__PURE__ */ new Set();
|
|
1082
|
+
records.forEach((record, index) => {
|
|
1083
|
+
if (originalMatchIds.has(record.id)) {
|
|
1084
|
+
allIndices.add(index);
|
|
1288
1085
|
}
|
|
1289
|
-
|
|
1290
|
-
|
|
1086
|
+
});
|
|
1087
|
+
additionalIndices.forEach((index) => {
|
|
1088
|
+
allIndices.add(index);
|
|
1089
|
+
});
|
|
1090
|
+
return Array.from(allIndices).sort((a, b) => a - b).map((index) => records[index]);
|
|
1091
|
+
}
|
|
1092
|
+
/**
|
|
1093
|
+
* Parse message data from LanceDB record format to MastraDBMessage format
|
|
1094
|
+
*/
|
|
1095
|
+
parseMessageData(data) {
|
|
1096
|
+
const { thread_id, ...rest } = data;
|
|
1097
|
+
return {
|
|
1098
|
+
...rest,
|
|
1099
|
+
threadId: thread_id,
|
|
1100
|
+
content: typeof data.content === "string" ? (() => {
|
|
1101
|
+
try {
|
|
1102
|
+
return JSON.parse(data.content);
|
|
1103
|
+
} catch {
|
|
1104
|
+
return data.content;
|
|
1105
|
+
}
|
|
1106
|
+
})() : data.content,
|
|
1107
|
+
createdAt: new Date(data.createdAt),
|
|
1108
|
+
updatedAt: new Date(data.updatedAt)
|
|
1109
|
+
};
|
|
1110
|
+
}
|
|
1111
|
+
async updateMessages(args) {
|
|
1112
|
+
const { messages } = args;
|
|
1113
|
+
this.logger.debug("Updating messages", { count: messages.length });
|
|
1114
|
+
if (!messages.length) {
|
|
1115
|
+
return [];
|
|
1116
|
+
}
|
|
1117
|
+
const updatedMessages = [];
|
|
1118
|
+
const affectedThreadIds = /* @__PURE__ */ new Set();
|
|
1119
|
+
try {
|
|
1120
|
+
for (const updateData of messages) {
|
|
1121
|
+
const { id, ...updates } = updateData;
|
|
1122
|
+
const existingMessage = await this.#db.load({ tableName: TABLE_MESSAGES, keys: { id } });
|
|
1123
|
+
if (!existingMessage) {
|
|
1124
|
+
this.logger.warn("Message not found for update", { id });
|
|
1125
|
+
continue;
|
|
1126
|
+
}
|
|
1127
|
+
const existingMsg = this.parseMessageData(existingMessage);
|
|
1128
|
+
const originalThreadId = existingMsg.threadId;
|
|
1129
|
+
affectedThreadIds.add(originalThreadId);
|
|
1130
|
+
const updatePayload = {};
|
|
1131
|
+
if ("role" in updates && updates.role !== void 0) updatePayload.role = updates.role;
|
|
1132
|
+
if ("type" in updates && updates.type !== void 0) updatePayload.type = updates.type;
|
|
1133
|
+
if ("resourceId" in updates && updates.resourceId !== void 0) updatePayload.resourceId = updates.resourceId;
|
|
1134
|
+
if ("threadId" in updates && updates.threadId !== void 0 && updates.threadId !== null) {
|
|
1135
|
+
updatePayload.thread_id = updates.threadId;
|
|
1136
|
+
affectedThreadIds.add(updates.threadId);
|
|
1137
|
+
}
|
|
1138
|
+
if (updates.content) {
|
|
1139
|
+
const existingContent = existingMsg.content;
|
|
1140
|
+
let newContent = { ...existingContent };
|
|
1141
|
+
if (updates.content.metadata !== void 0) {
|
|
1142
|
+
newContent.metadata = {
|
|
1143
|
+
...existingContent.metadata || {},
|
|
1144
|
+
...updates.content.metadata || {}
|
|
1145
|
+
};
|
|
1146
|
+
}
|
|
1147
|
+
if (updates.content.content !== void 0) {
|
|
1148
|
+
newContent.content = updates.content.content;
|
|
1149
|
+
}
|
|
1150
|
+
if ("parts" in updates.content && updates.content.parts !== void 0) {
|
|
1151
|
+
newContent.parts = updates.content.parts;
|
|
1152
|
+
}
|
|
1153
|
+
updatePayload.content = JSON.stringify(newContent);
|
|
1154
|
+
}
|
|
1155
|
+
await this.#db.insert({ tableName: TABLE_MESSAGES, record: { id, ...updatePayload } });
|
|
1156
|
+
const updatedMessage = await this.#db.load({ tableName: TABLE_MESSAGES, keys: { id } });
|
|
1157
|
+
if (updatedMessage) {
|
|
1158
|
+
updatedMessages.push(this.parseMessageData(updatedMessage));
|
|
1159
|
+
}
|
|
1291
1160
|
}
|
|
1292
|
-
|
|
1161
|
+
for (const threadId of affectedThreadIds) {
|
|
1162
|
+
await this.#db.insert({
|
|
1163
|
+
tableName: TABLE_THREADS,
|
|
1164
|
+
record: { id: threadId, updatedAt: Date.now() }
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
return updatedMessages;
|
|
1168
|
+
} catch (error) {
|
|
1293
1169
|
throw new MastraError(
|
|
1294
1170
|
{
|
|
1295
|
-
id: "
|
|
1171
|
+
id: createStorageErrorId("LANCE", "UPDATE_MESSAGES", "FAILED"),
|
|
1296
1172
|
domain: ErrorDomain.STORAGE,
|
|
1297
|
-
category: ErrorCategory.
|
|
1298
|
-
|
|
1299
|
-
details: { tableName }
|
|
1173
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1174
|
+
details: { count: messages.length }
|
|
1300
1175
|
},
|
|
1301
|
-
|
|
1176
|
+
error
|
|
1302
1177
|
);
|
|
1303
1178
|
}
|
|
1179
|
+
}
|
|
1180
|
+
async getResourceById({ resourceId }) {
|
|
1304
1181
|
try {
|
|
1305
|
-
const
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1182
|
+
const resource = await this.#db.load({ tableName: TABLE_RESOURCES, keys: { id: resourceId } });
|
|
1183
|
+
if (!resource) {
|
|
1184
|
+
return null;
|
|
1185
|
+
}
|
|
1186
|
+
let createdAt;
|
|
1187
|
+
let updatedAt;
|
|
1188
|
+
try {
|
|
1189
|
+
if (resource.createdAt instanceof Date) {
|
|
1190
|
+
createdAt = resource.createdAt;
|
|
1191
|
+
} else if (typeof resource.createdAt === "string") {
|
|
1192
|
+
createdAt = new Date(resource.createdAt);
|
|
1193
|
+
} else if (typeof resource.createdAt === "number") {
|
|
1194
|
+
createdAt = new Date(resource.createdAt);
|
|
1195
|
+
} else {
|
|
1196
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1314
1197
|
}
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1198
|
+
if (isNaN(createdAt.getTime())) {
|
|
1199
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1200
|
+
}
|
|
1201
|
+
} catch {
|
|
1202
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1203
|
+
}
|
|
1204
|
+
try {
|
|
1205
|
+
if (resource.updatedAt instanceof Date) {
|
|
1206
|
+
updatedAt = resource.updatedAt;
|
|
1207
|
+
} else if (typeof resource.updatedAt === "string") {
|
|
1208
|
+
updatedAt = new Date(resource.updatedAt);
|
|
1209
|
+
} else if (typeof resource.updatedAt === "number") {
|
|
1210
|
+
updatedAt = new Date(resource.updatedAt);
|
|
1211
|
+
} else {
|
|
1212
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1213
|
+
}
|
|
1214
|
+
if (isNaN(updatedAt.getTime())) {
|
|
1215
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1216
|
+
}
|
|
1217
|
+
} catch {
|
|
1218
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1219
|
+
}
|
|
1220
|
+
let workingMemory = resource.workingMemory;
|
|
1221
|
+
if (workingMemory === null || workingMemory === void 0) {
|
|
1222
|
+
workingMemory = void 0;
|
|
1223
|
+
} else if (workingMemory === "") {
|
|
1224
|
+
workingMemory = "";
|
|
1225
|
+
} else if (typeof workingMemory === "object") {
|
|
1226
|
+
workingMemory = JSON.stringify(workingMemory);
|
|
1227
|
+
}
|
|
1228
|
+
let metadata = resource.metadata;
|
|
1229
|
+
if (metadata === "" || metadata === null || metadata === void 0) {
|
|
1230
|
+
metadata = void 0;
|
|
1231
|
+
} else if (typeof metadata === "string") {
|
|
1232
|
+
try {
|
|
1233
|
+
metadata = JSON.parse(metadata);
|
|
1234
|
+
} catch {
|
|
1235
|
+
metadata = metadata;
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
return {
|
|
1239
|
+
...resource,
|
|
1240
|
+
createdAt,
|
|
1241
|
+
updatedAt,
|
|
1242
|
+
workingMemory,
|
|
1243
|
+
metadata
|
|
1244
|
+
};
|
|
1318
1245
|
} catch (error) {
|
|
1319
1246
|
throw new MastraError(
|
|
1320
1247
|
{
|
|
1321
|
-
id: "
|
|
1248
|
+
id: createStorageErrorId("LANCE", "GET_RESOURCE_BY_ID", "FAILED"),
|
|
1322
1249
|
domain: ErrorDomain.STORAGE,
|
|
1323
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1324
|
-
details: { tableName }
|
|
1250
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1325
1251
|
},
|
|
1326
1252
|
error
|
|
1327
1253
|
);
|
|
1328
1254
|
}
|
|
1329
1255
|
}
|
|
1330
|
-
async
|
|
1331
|
-
try {
|
|
1332
|
-
if (!this.client) {
|
|
1333
|
-
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1334
|
-
}
|
|
1335
|
-
if (!tableName) {
|
|
1336
|
-
throw new Error("tableName is required for load.");
|
|
1337
|
-
}
|
|
1338
|
-
if (!keys || Object.keys(keys).length === 0) {
|
|
1339
|
-
throw new Error("keys are required and cannot be empty for load.");
|
|
1340
|
-
}
|
|
1341
|
-
} catch (validationError) {
|
|
1342
|
-
throw new MastraError(
|
|
1343
|
-
{
|
|
1344
|
-
id: "STORAGE_LANCE_STORAGE_LOAD_INVALID_ARGS",
|
|
1345
|
-
domain: ErrorDomain.STORAGE,
|
|
1346
|
-
category: ErrorCategory.USER,
|
|
1347
|
-
text: validationError.message,
|
|
1348
|
-
details: { tableName }
|
|
1349
|
-
},
|
|
1350
|
-
validationError
|
|
1351
|
-
);
|
|
1352
|
-
}
|
|
1256
|
+
async saveResource({ resource }) {
|
|
1353
1257
|
try {
|
|
1354
|
-
const
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
return `${quotedKey} IS NULL`;
|
|
1366
|
-
} else {
|
|
1367
|
-
return `${quotedKey} = ${value}`;
|
|
1368
|
-
}
|
|
1369
|
-
}).join(" AND ");
|
|
1370
|
-
this.logger.debug("where clause generated: " + filterConditions);
|
|
1371
|
-
query.where(filterConditions);
|
|
1372
|
-
}
|
|
1373
|
-
const result = await query.limit(1).toArray();
|
|
1374
|
-
if (result.length === 0) {
|
|
1375
|
-
this.logger.debug("No record found");
|
|
1376
|
-
return null;
|
|
1377
|
-
}
|
|
1378
|
-
return processResultWithTypeConversion(result[0], tableSchema);
|
|
1258
|
+
const record = {
|
|
1259
|
+
...resource,
|
|
1260
|
+
metadata: resource.metadata ? JSON.stringify(resource.metadata) : "",
|
|
1261
|
+
createdAt: resource.createdAt.getTime(),
|
|
1262
|
+
// Store as timestamp (milliseconds)
|
|
1263
|
+
updatedAt: resource.updatedAt.getTime()
|
|
1264
|
+
// Store as timestamp (milliseconds)
|
|
1265
|
+
};
|
|
1266
|
+
const table = await this.client.openTable(TABLE_RESOURCES);
|
|
1267
|
+
await table.add([record], { mode: "append" });
|
|
1268
|
+
return resource;
|
|
1379
1269
|
} catch (error) {
|
|
1380
|
-
if (error instanceof MastraError) throw error;
|
|
1381
1270
|
throw new MastraError(
|
|
1382
1271
|
{
|
|
1383
|
-
id: "
|
|
1272
|
+
id: createStorageErrorId("LANCE", "SAVE_RESOURCE", "FAILED"),
|
|
1384
1273
|
domain: ErrorDomain.STORAGE,
|
|
1385
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1386
|
-
details: { tableName, keyCount: Object.keys(keys).length, firstKey: Object.keys(keys)[0] ?? "" }
|
|
1274
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1387
1275
|
},
|
|
1388
1276
|
error
|
|
1389
1277
|
);
|
|
1390
1278
|
}
|
|
1391
1279
|
}
|
|
1280
|
+
async updateResource({
|
|
1281
|
+
resourceId,
|
|
1282
|
+
workingMemory,
|
|
1283
|
+
metadata
|
|
1284
|
+
}) {
|
|
1285
|
+
const maxRetries = 3;
|
|
1286
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
1287
|
+
try {
|
|
1288
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1289
|
+
if (!existingResource) {
|
|
1290
|
+
const newResource = {
|
|
1291
|
+
id: resourceId,
|
|
1292
|
+
workingMemory,
|
|
1293
|
+
metadata: metadata || {},
|
|
1294
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1295
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1296
|
+
};
|
|
1297
|
+
return this.saveResource({ resource: newResource });
|
|
1298
|
+
}
|
|
1299
|
+
const updatedResource = {
|
|
1300
|
+
...existingResource,
|
|
1301
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1302
|
+
metadata: {
|
|
1303
|
+
...existingResource.metadata,
|
|
1304
|
+
...metadata
|
|
1305
|
+
},
|
|
1306
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1307
|
+
};
|
|
1308
|
+
const record = {
|
|
1309
|
+
id: resourceId,
|
|
1310
|
+
workingMemory: updatedResource.workingMemory || "",
|
|
1311
|
+
metadata: updatedResource.metadata ? JSON.stringify(updatedResource.metadata) : "",
|
|
1312
|
+
updatedAt: updatedResource.updatedAt.getTime()
|
|
1313
|
+
// Store as timestamp (milliseconds)
|
|
1314
|
+
};
|
|
1315
|
+
const table = await this.client.openTable(TABLE_RESOURCES);
|
|
1316
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
1317
|
+
return updatedResource;
|
|
1318
|
+
} catch (error) {
|
|
1319
|
+
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
1320
|
+
const delay = Math.pow(2, attempt) * 10;
|
|
1321
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1322
|
+
continue;
|
|
1323
|
+
}
|
|
1324
|
+
throw new MastraError(
|
|
1325
|
+
{
|
|
1326
|
+
id: createStorageErrorId("LANCE", "UPDATE_RESOURCE", "FAILED"),
|
|
1327
|
+
domain: ErrorDomain.STORAGE,
|
|
1328
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1329
|
+
},
|
|
1330
|
+
error
|
|
1331
|
+
);
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
throw new Error("Unexpected end of retry loop");
|
|
1335
|
+
}
|
|
1392
1336
|
};
|
|
1393
1337
|
var StoreScoresLance = class extends ScoresStorage {
|
|
1394
1338
|
client;
|
|
1395
|
-
|
|
1339
|
+
#db;
|
|
1340
|
+
constructor(config) {
|
|
1396
1341
|
super();
|
|
1342
|
+
const client = resolveLanceConfig(config);
|
|
1397
1343
|
this.client = client;
|
|
1344
|
+
this.#db = new LanceDB({ client });
|
|
1345
|
+
}
|
|
1346
|
+
async init() {
|
|
1347
|
+
await this.#db.createTable({ tableName: TABLE_SCORERS, schema: SCORERS_SCHEMA });
|
|
1348
|
+
await this.#db.alterTable({
|
|
1349
|
+
tableName: TABLE_SCORERS,
|
|
1350
|
+
schema: SCORERS_SCHEMA,
|
|
1351
|
+
ifNotExists: ["spanId", "requestContext"]
|
|
1352
|
+
});
|
|
1353
|
+
}
|
|
1354
|
+
async dangerouslyClearAll() {
|
|
1355
|
+
await this.#db.clearTable({ tableName: TABLE_SCORERS });
|
|
1398
1356
|
}
|
|
1399
1357
|
async saveScore(score) {
|
|
1400
1358
|
let validatedScore;
|
|
@@ -1403,37 +1361,47 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1403
1361
|
} catch (error) {
|
|
1404
1362
|
throw new MastraError(
|
|
1405
1363
|
{
|
|
1406
|
-
id: "
|
|
1364
|
+
id: createStorageErrorId("LANCE", "SAVE_SCORE", "VALIDATION_FAILED"),
|
|
1407
1365
|
text: "Failed to save score in LanceStorage",
|
|
1408
1366
|
domain: ErrorDomain.STORAGE,
|
|
1409
|
-
category: ErrorCategory.
|
|
1367
|
+
category: ErrorCategory.USER,
|
|
1368
|
+
details: {
|
|
1369
|
+
scorer: typeof score.scorer?.id === "string" ? score.scorer.id : String(score.scorer?.id ?? "unknown"),
|
|
1370
|
+
entityId: score.entityId ?? "unknown",
|
|
1371
|
+
entityType: score.entityType ?? "unknown",
|
|
1372
|
+
traceId: score.traceId ?? "",
|
|
1373
|
+
spanId: score.spanId ?? ""
|
|
1374
|
+
}
|
|
1410
1375
|
},
|
|
1411
1376
|
error
|
|
1412
1377
|
);
|
|
1413
1378
|
}
|
|
1379
|
+
const id = crypto.randomUUID();
|
|
1380
|
+
const now = /* @__PURE__ */ new Date();
|
|
1414
1381
|
try {
|
|
1415
|
-
const id = crypto.randomUUID();
|
|
1416
1382
|
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1417
1383
|
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1418
1384
|
const allowedFields = new Set(schema.fields.map((f) => f.name));
|
|
1419
1385
|
const filteredScore = {};
|
|
1420
|
-
Object.keys(validatedScore)
|
|
1386
|
+
for (const key of Object.keys(validatedScore)) {
|
|
1421
1387
|
if (allowedFields.has(key)) {
|
|
1422
|
-
filteredScore[key] =
|
|
1388
|
+
filteredScore[key] = validatedScore[key];
|
|
1423
1389
|
}
|
|
1424
|
-
}
|
|
1390
|
+
}
|
|
1425
1391
|
for (const key in filteredScore) {
|
|
1426
1392
|
if (filteredScore[key] !== null && typeof filteredScore[key] === "object" && !(filteredScore[key] instanceof Date)) {
|
|
1427
1393
|
filteredScore[key] = JSON.stringify(filteredScore[key]);
|
|
1428
1394
|
}
|
|
1429
1395
|
}
|
|
1430
1396
|
filteredScore.id = id;
|
|
1397
|
+
filteredScore.createdAt = now;
|
|
1398
|
+
filteredScore.updatedAt = now;
|
|
1431
1399
|
await table.add([filteredScore], { mode: "append" });
|
|
1432
|
-
return { score };
|
|
1400
|
+
return { score: { ...validatedScore, id, createdAt: now, updatedAt: now } };
|
|
1433
1401
|
} catch (error) {
|
|
1434
1402
|
throw new MastraError(
|
|
1435
1403
|
{
|
|
1436
|
-
id: "
|
|
1404
|
+
id: createStorageErrorId("LANCE", "SAVE_SCORE", "FAILED"),
|
|
1437
1405
|
text: "Failed to save score in LanceStorage",
|
|
1438
1406
|
domain: ErrorDomain.STORAGE,
|
|
1439
1407
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1449,12 +1417,11 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1449
1417
|
const query = table.query().where(`id = '${id}'`).limit(1);
|
|
1450
1418
|
const records = await query.toArray();
|
|
1451
1419
|
if (records.length === 0) return null;
|
|
1452
|
-
|
|
1453
|
-
return processResultWithTypeConversion(records[0], schema);
|
|
1420
|
+
return await this.transformScoreRow(records[0]);
|
|
1454
1421
|
} catch (error) {
|
|
1455
1422
|
throw new MastraError(
|
|
1456
1423
|
{
|
|
1457
|
-
id: "
|
|
1424
|
+
id: createStorageErrorId("LANCE", "GET_SCORE_BY_ID", "FAILED"),
|
|
1458
1425
|
text: "Failed to get score by id in LanceStorage",
|
|
1459
1426
|
domain: ErrorDomain.STORAGE,
|
|
1460
1427
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1464,7 +1431,23 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1464
1431
|
);
|
|
1465
1432
|
}
|
|
1466
1433
|
}
|
|
1467
|
-
|
|
1434
|
+
/**
|
|
1435
|
+
* LanceDB-specific score row transformation.
|
|
1436
|
+
*
|
|
1437
|
+
* Note: This implementation does NOT use coreTransformScoreRow because:
|
|
1438
|
+
* 1. LanceDB stores schema information in the table itself (requires async fetch)
|
|
1439
|
+
* 2. Uses processResultWithTypeConversion utility for LanceDB-specific type handling
|
|
1440
|
+
*/
|
|
1441
|
+
async transformScoreRow(row) {
|
|
1442
|
+
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1443
|
+
const transformed = processResultWithTypeConversion(row, schema);
|
|
1444
|
+
return {
|
|
1445
|
+
...transformed,
|
|
1446
|
+
createdAt: row.createdAt,
|
|
1447
|
+
updatedAt: row.updatedAt
|
|
1448
|
+
};
|
|
1449
|
+
}
|
|
1450
|
+
async listScoresByScorerId({
|
|
1468
1451
|
scorerId,
|
|
1469
1452
|
pagination,
|
|
1470
1453
|
entityId,
|
|
@@ -1472,9 +1455,10 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1472
1455
|
source
|
|
1473
1456
|
}) {
|
|
1474
1457
|
try {
|
|
1458
|
+
const { page, perPage: perPageInput } = pagination;
|
|
1459
|
+
const perPage = normalizePerPage(perPageInput, 100);
|
|
1460
|
+
const { offset: start, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
1475
1461
|
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1476
|
-
const { page = 0, perPage = 10 } = pagination || {};
|
|
1477
|
-
const offset = page * perPage;
|
|
1478
1462
|
let query = table.query().where(`\`scorerId\` = '${scorerId}'`);
|
|
1479
1463
|
if (source) {
|
|
1480
1464
|
query = query.where(`\`source\` = '${source}'`);
|
|
@@ -1485,30 +1469,38 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1485
1469
|
if (entityType) {
|
|
1486
1470
|
query = query.where(`\`entityType\` = '${entityType}'`);
|
|
1487
1471
|
}
|
|
1488
|
-
query = query.limit(perPage);
|
|
1489
|
-
if (offset > 0) query.offset(offset);
|
|
1490
|
-
const records = await query.toArray();
|
|
1491
|
-
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1492
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1493
1472
|
let totalQuery = table.query().where(`\`scorerId\` = '${scorerId}'`);
|
|
1494
1473
|
if (source) {
|
|
1495
1474
|
totalQuery = totalQuery.where(`\`source\` = '${source}'`);
|
|
1496
1475
|
}
|
|
1476
|
+
if (entityId) {
|
|
1477
|
+
totalQuery = totalQuery.where(`\`entityId\` = '${entityId}'`);
|
|
1478
|
+
}
|
|
1479
|
+
if (entityType) {
|
|
1480
|
+
totalQuery = totalQuery.where(`\`entityType\` = '${entityType}'`);
|
|
1481
|
+
}
|
|
1497
1482
|
const allRecords = await totalQuery.toArray();
|
|
1498
1483
|
const total = allRecords.length;
|
|
1484
|
+
const end = perPageInput === false ? total : start + perPage;
|
|
1485
|
+
if (perPageInput !== false) {
|
|
1486
|
+
query = query.limit(perPage);
|
|
1487
|
+
if (start > 0) query = query.offset(start);
|
|
1488
|
+
}
|
|
1489
|
+
const records = await query.toArray();
|
|
1490
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1499
1491
|
return {
|
|
1500
1492
|
pagination: {
|
|
1501
1493
|
page,
|
|
1502
|
-
perPage,
|
|
1494
|
+
perPage: perPageForResponse,
|
|
1503
1495
|
total,
|
|
1504
|
-
hasMore:
|
|
1496
|
+
hasMore: end < total
|
|
1505
1497
|
},
|
|
1506
1498
|
scores
|
|
1507
1499
|
};
|
|
1508
1500
|
} catch (error) {
|
|
1509
1501
|
throw new MastraError(
|
|
1510
1502
|
{
|
|
1511
|
-
id: "
|
|
1503
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_SCORER_ID", "FAILED"),
|
|
1512
1504
|
text: "Failed to get scores by scorerId in LanceStorage",
|
|
1513
1505
|
domain: ErrorDomain.STORAGE,
|
|
1514
1506
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1518,34 +1510,38 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1518
1510
|
);
|
|
1519
1511
|
}
|
|
1520
1512
|
}
|
|
1521
|
-
async
|
|
1513
|
+
async listScoresByRunId({
|
|
1522
1514
|
runId,
|
|
1523
1515
|
pagination
|
|
1524
1516
|
}) {
|
|
1525
1517
|
try {
|
|
1518
|
+
const { page, perPage: perPageInput } = pagination;
|
|
1519
|
+
const perPage = normalizePerPage(perPageInput, 100);
|
|
1520
|
+
const { offset: start, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
1526
1521
|
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1527
|
-
const { page = 0, perPage = 10 } = pagination || {};
|
|
1528
|
-
const offset = page * perPage;
|
|
1529
|
-
const query = table.query().where(`\`runId\` = '${runId}'`).limit(perPage);
|
|
1530
|
-
if (offset > 0) query.offset(offset);
|
|
1531
|
-
const records = await query.toArray();
|
|
1532
|
-
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1533
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1534
1522
|
const allRecords = await table.query().where(`\`runId\` = '${runId}'`).toArray();
|
|
1535
1523
|
const total = allRecords.length;
|
|
1524
|
+
const end = perPageInput === false ? total : start + perPage;
|
|
1525
|
+
let query = table.query().where(`\`runId\` = '${runId}'`);
|
|
1526
|
+
if (perPageInput !== false) {
|
|
1527
|
+
query = query.limit(perPage);
|
|
1528
|
+
if (start > 0) query = query.offset(start);
|
|
1529
|
+
}
|
|
1530
|
+
const records = await query.toArray();
|
|
1531
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1536
1532
|
return {
|
|
1537
1533
|
pagination: {
|
|
1538
1534
|
page,
|
|
1539
|
-
perPage,
|
|
1535
|
+
perPage: perPageForResponse,
|
|
1540
1536
|
total,
|
|
1541
|
-
hasMore:
|
|
1537
|
+
hasMore: end < total
|
|
1542
1538
|
},
|
|
1543
1539
|
scores
|
|
1544
1540
|
};
|
|
1545
1541
|
} catch (error) {
|
|
1546
1542
|
throw new MastraError(
|
|
1547
1543
|
{
|
|
1548
|
-
id: "
|
|
1544
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_RUN_ID", "FAILED"),
|
|
1549
1545
|
text: "Failed to get scores by runId in LanceStorage",
|
|
1550
1546
|
domain: ErrorDomain.STORAGE,
|
|
1551
1547
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1555,35 +1551,39 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1555
1551
|
);
|
|
1556
1552
|
}
|
|
1557
1553
|
}
|
|
1558
|
-
async
|
|
1554
|
+
async listScoresByEntityId({
|
|
1559
1555
|
entityId,
|
|
1560
1556
|
entityType,
|
|
1561
1557
|
pagination
|
|
1562
1558
|
}) {
|
|
1563
1559
|
try {
|
|
1560
|
+
const { page, perPage: perPageInput } = pagination;
|
|
1561
|
+
const perPage = normalizePerPage(perPageInput, 100);
|
|
1562
|
+
const { offset: start, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
1564
1563
|
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1565
|
-
const { page = 0, perPage = 10 } = pagination || {};
|
|
1566
|
-
const offset = page * perPage;
|
|
1567
|
-
const query = table.query().where(`\`entityId\` = '${entityId}' AND \`entityType\` = '${entityType}'`).limit(perPage);
|
|
1568
|
-
if (offset > 0) query.offset(offset);
|
|
1569
|
-
const records = await query.toArray();
|
|
1570
|
-
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1571
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1572
1564
|
const allRecords = await table.query().where(`\`entityId\` = '${entityId}' AND \`entityType\` = '${entityType}'`).toArray();
|
|
1573
1565
|
const total = allRecords.length;
|
|
1566
|
+
const end = perPageInput === false ? total : start + perPage;
|
|
1567
|
+
let query = table.query().where(`\`entityId\` = '${entityId}' AND \`entityType\` = '${entityType}'`);
|
|
1568
|
+
if (perPageInput !== false) {
|
|
1569
|
+
query = query.limit(perPage);
|
|
1570
|
+
if (start > 0) query = query.offset(start);
|
|
1571
|
+
}
|
|
1572
|
+
const records = await query.toArray();
|
|
1573
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1574
1574
|
return {
|
|
1575
1575
|
pagination: {
|
|
1576
1576
|
page,
|
|
1577
|
-
perPage,
|
|
1577
|
+
perPage: perPageForResponse,
|
|
1578
1578
|
total,
|
|
1579
|
-
hasMore:
|
|
1579
|
+
hasMore: end < total
|
|
1580
1580
|
},
|
|
1581
1581
|
scores
|
|
1582
1582
|
};
|
|
1583
1583
|
} catch (error) {
|
|
1584
1584
|
throw new MastraError(
|
|
1585
1585
|
{
|
|
1586
|
-
id: "
|
|
1586
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_ENTITY_ID", "FAILED"),
|
|
1587
1587
|
text: "Failed to get scores by entityId and entityType in LanceStorage",
|
|
1588
1588
|
domain: ErrorDomain.STORAGE,
|
|
1589
1589
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1593,307 +1593,138 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1593
1593
|
);
|
|
1594
1594
|
}
|
|
1595
1595
|
}
|
|
1596
|
-
async
|
|
1596
|
+
async listScoresBySpan({
|
|
1597
1597
|
traceId,
|
|
1598
1598
|
spanId,
|
|
1599
1599
|
pagination
|
|
1600
1600
|
}) {
|
|
1601
1601
|
try {
|
|
1602
|
+
const { page, perPage: perPageInput } = pagination;
|
|
1603
|
+
const perPage = normalizePerPage(perPageInput, 100);
|
|
1604
|
+
const { offset: start, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
1602
1605
|
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1603
|
-
const
|
|
1604
|
-
const
|
|
1605
|
-
const
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
const allRecords = await table.query().where(`\`traceId\` = '${traceId}' AND \`spanId\` = '${spanId}'`).toArray();
|
|
1611
|
-
const total = allRecords.length;
|
|
1612
|
-
return {
|
|
1613
|
-
pagination: {
|
|
1614
|
-
page,
|
|
1615
|
-
perPage,
|
|
1616
|
-
total,
|
|
1617
|
-
hasMore: offset + scores.length < total
|
|
1618
|
-
},
|
|
1619
|
-
scores
|
|
1620
|
-
};
|
|
1621
|
-
} catch (error) {
|
|
1622
|
-
throw new MastraError(
|
|
1623
|
-
{
|
|
1624
|
-
id: "LANCE_STORAGE_GET_SCORES_BY_SPAN_FAILED",
|
|
1625
|
-
text: "Failed to get scores by traceId and spanId in LanceStorage",
|
|
1626
|
-
domain: ErrorDomain.STORAGE,
|
|
1627
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
1628
|
-
details: { error: error?.message }
|
|
1629
|
-
},
|
|
1630
|
-
error
|
|
1631
|
-
);
|
|
1632
|
-
}
|
|
1633
|
-
}
|
|
1634
|
-
};
|
|
1635
|
-
var StoreTracesLance = class extends TracesStorage {
|
|
1636
|
-
client;
|
|
1637
|
-
operations;
|
|
1638
|
-
constructor({ client, operations }) {
|
|
1639
|
-
super();
|
|
1640
|
-
this.client = client;
|
|
1641
|
-
this.operations = operations;
|
|
1642
|
-
}
|
|
1643
|
-
async saveTrace({ trace }) {
|
|
1644
|
-
try {
|
|
1645
|
-
const table = await this.client.openTable(TABLE_TRACES);
|
|
1646
|
-
const record = {
|
|
1647
|
-
...trace,
|
|
1648
|
-
attributes: JSON.stringify(trace.attributes),
|
|
1649
|
-
status: JSON.stringify(trace.status),
|
|
1650
|
-
events: JSON.stringify(trace.events),
|
|
1651
|
-
links: JSON.stringify(trace.links),
|
|
1652
|
-
other: JSON.stringify(trace.other)
|
|
1653
|
-
};
|
|
1654
|
-
await table.add([record], { mode: "append" });
|
|
1655
|
-
return trace;
|
|
1656
|
-
} catch (error) {
|
|
1657
|
-
throw new MastraError(
|
|
1658
|
-
{
|
|
1659
|
-
id: "LANCE_STORE_SAVE_TRACE_FAILED",
|
|
1660
|
-
domain: ErrorDomain.STORAGE,
|
|
1661
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1662
|
-
},
|
|
1663
|
-
error
|
|
1664
|
-
);
|
|
1665
|
-
}
|
|
1666
|
-
}
|
|
1667
|
-
async getTraceById({ traceId }) {
|
|
1668
|
-
try {
|
|
1669
|
-
const table = await this.client.openTable(TABLE_TRACES);
|
|
1670
|
-
const query = table.query().where(`id = '${traceId}'`);
|
|
1671
|
-
const records = await query.toArray();
|
|
1672
|
-
return records[0];
|
|
1673
|
-
} catch (error) {
|
|
1674
|
-
throw new MastraError(
|
|
1675
|
-
{
|
|
1676
|
-
id: "LANCE_STORE_GET_TRACE_BY_ID_FAILED",
|
|
1677
|
-
domain: ErrorDomain.STORAGE,
|
|
1678
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1679
|
-
},
|
|
1680
|
-
error
|
|
1681
|
-
);
|
|
1682
|
-
}
|
|
1683
|
-
}
|
|
1684
|
-
async getTraces({
|
|
1685
|
-
name,
|
|
1686
|
-
scope,
|
|
1687
|
-
page = 1,
|
|
1688
|
-
perPage = 10,
|
|
1689
|
-
attributes
|
|
1690
|
-
}) {
|
|
1691
|
-
try {
|
|
1692
|
-
const table = await this.client.openTable(TABLE_TRACES);
|
|
1693
|
-
const query = table.query();
|
|
1694
|
-
if (name) {
|
|
1695
|
-
query.where(`name = '${name}'`);
|
|
1696
|
-
}
|
|
1697
|
-
if (scope) {
|
|
1698
|
-
query.where(`scope = '${scope}'`);
|
|
1699
|
-
}
|
|
1700
|
-
if (attributes) {
|
|
1701
|
-
query.where(`attributes = '${JSON.stringify(attributes)}'`);
|
|
1702
|
-
}
|
|
1703
|
-
const offset = (page - 1) * perPage;
|
|
1704
|
-
query.limit(perPage);
|
|
1705
|
-
if (offset > 0) {
|
|
1706
|
-
query.offset(offset);
|
|
1707
|
-
}
|
|
1708
|
-
const records = await query.toArray();
|
|
1709
|
-
return records.map((record) => {
|
|
1710
|
-
const processed = {
|
|
1711
|
-
...record,
|
|
1712
|
-
attributes: record.attributes ? JSON.parse(record.attributes) : {},
|
|
1713
|
-
status: record.status ? JSON.parse(record.status) : {},
|
|
1714
|
-
events: record.events ? JSON.parse(record.events) : [],
|
|
1715
|
-
links: record.links ? JSON.parse(record.links) : [],
|
|
1716
|
-
other: record.other ? JSON.parse(record.other) : {},
|
|
1717
|
-
startTime: new Date(record.startTime),
|
|
1718
|
-
endTime: new Date(record.endTime),
|
|
1719
|
-
createdAt: new Date(record.createdAt)
|
|
1720
|
-
};
|
|
1721
|
-
if (processed.parentSpanId === null || processed.parentSpanId === void 0) {
|
|
1722
|
-
processed.parentSpanId = "";
|
|
1723
|
-
} else {
|
|
1724
|
-
processed.parentSpanId = String(processed.parentSpanId);
|
|
1725
|
-
}
|
|
1726
|
-
return processed;
|
|
1727
|
-
});
|
|
1728
|
-
} catch (error) {
|
|
1729
|
-
throw new MastraError(
|
|
1730
|
-
{
|
|
1731
|
-
id: "LANCE_STORE_GET_TRACES_FAILED",
|
|
1732
|
-
domain: ErrorDomain.STORAGE,
|
|
1733
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
1734
|
-
details: { name: name ?? "", scope: scope ?? "" }
|
|
1735
|
-
},
|
|
1736
|
-
error
|
|
1737
|
-
);
|
|
1738
|
-
}
|
|
1739
|
-
}
|
|
1740
|
-
async getTracesPaginated(args) {
|
|
1741
|
-
try {
|
|
1742
|
-
const table = await this.client.openTable(TABLE_TRACES);
|
|
1743
|
-
const query = table.query();
|
|
1744
|
-
const conditions = [];
|
|
1745
|
-
if (args.name) {
|
|
1746
|
-
conditions.push(`name = '${args.name}'`);
|
|
1747
|
-
}
|
|
1748
|
-
if (args.scope) {
|
|
1749
|
-
conditions.push(`scope = '${args.scope}'`);
|
|
1750
|
-
}
|
|
1751
|
-
if (args.attributes) {
|
|
1752
|
-
const attributesStr = JSON.stringify(args.attributes);
|
|
1753
|
-
conditions.push(`attributes LIKE '%${attributesStr.replace(/"/g, '\\"')}%'`);
|
|
1754
|
-
}
|
|
1755
|
-
if (args.dateRange?.start) {
|
|
1756
|
-
conditions.push(`\`createdAt\` >= ${args.dateRange.start.getTime()}`);
|
|
1757
|
-
}
|
|
1758
|
-
if (args.dateRange?.end) {
|
|
1759
|
-
conditions.push(`\`createdAt\` <= ${args.dateRange.end.getTime()}`);
|
|
1760
|
-
}
|
|
1761
|
-
if (conditions.length > 0) {
|
|
1762
|
-
const whereClause = conditions.join(" AND ");
|
|
1763
|
-
query.where(whereClause);
|
|
1764
|
-
}
|
|
1765
|
-
let total = 0;
|
|
1766
|
-
if (conditions.length > 0) {
|
|
1767
|
-
const countQuery = table.query().where(conditions.join(" AND "));
|
|
1768
|
-
const allRecords = await countQuery.toArray();
|
|
1769
|
-
total = allRecords.length;
|
|
1770
|
-
} else {
|
|
1771
|
-
total = await table.countRows();
|
|
1772
|
-
}
|
|
1773
|
-
const page = args.page || 0;
|
|
1774
|
-
const perPage = args.perPage || 10;
|
|
1775
|
-
const offset = page * perPage;
|
|
1776
|
-
query.limit(perPage);
|
|
1777
|
-
if (offset > 0) {
|
|
1778
|
-
query.offset(offset);
|
|
1606
|
+
const allRecords = await table.query().where(`\`traceId\` = '${traceId}' AND \`spanId\` = '${spanId}'`).toArray();
|
|
1607
|
+
const total = allRecords.length;
|
|
1608
|
+
const end = perPageInput === false ? total : start + perPage;
|
|
1609
|
+
let query = table.query().where(`\`traceId\` = '${traceId}' AND \`spanId\` = '${spanId}'`);
|
|
1610
|
+
if (perPageInput !== false) {
|
|
1611
|
+
query = query.limit(perPage);
|
|
1612
|
+
if (start > 0) query = query.offset(start);
|
|
1779
1613
|
}
|
|
1780
1614
|
const records = await query.toArray();
|
|
1781
|
-
const
|
|
1782
|
-
const processed = {
|
|
1783
|
-
...record,
|
|
1784
|
-
attributes: record.attributes ? JSON.parse(record.attributes) : {},
|
|
1785
|
-
status: record.status ? JSON.parse(record.status) : {},
|
|
1786
|
-
events: record.events ? JSON.parse(record.events) : [],
|
|
1787
|
-
links: record.links ? JSON.parse(record.links) : [],
|
|
1788
|
-
other: record.other ? JSON.parse(record.other) : {},
|
|
1789
|
-
startTime: new Date(record.startTime),
|
|
1790
|
-
endTime: new Date(record.endTime),
|
|
1791
|
-
createdAt: new Date(record.createdAt)
|
|
1792
|
-
};
|
|
1793
|
-
if (processed.parentSpanId === null || processed.parentSpanId === void 0) {
|
|
1794
|
-
processed.parentSpanId = "";
|
|
1795
|
-
} else {
|
|
1796
|
-
processed.parentSpanId = String(processed.parentSpanId);
|
|
1797
|
-
}
|
|
1798
|
-
return processed;
|
|
1799
|
-
});
|
|
1615
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1800
1616
|
return {
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1617
|
+
pagination: {
|
|
1618
|
+
page,
|
|
1619
|
+
perPage: perPageForResponse,
|
|
1620
|
+
total,
|
|
1621
|
+
hasMore: end < total
|
|
1622
|
+
},
|
|
1623
|
+
scores
|
|
1806
1624
|
};
|
|
1807
1625
|
} catch (error) {
|
|
1808
1626
|
throw new MastraError(
|
|
1809
1627
|
{
|
|
1810
|
-
id: "
|
|
1628
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_SPAN", "FAILED"),
|
|
1629
|
+
text: "Failed to get scores by traceId and spanId in LanceStorage",
|
|
1811
1630
|
domain: ErrorDomain.STORAGE,
|
|
1812
1631
|
category: ErrorCategory.THIRD_PARTY,
|
|
1813
|
-
details: {
|
|
1632
|
+
details: { error: error?.message }
|
|
1814
1633
|
},
|
|
1815
1634
|
error
|
|
1816
1635
|
);
|
|
1817
1636
|
}
|
|
1818
1637
|
}
|
|
1819
|
-
async batchTraceInsert({ records }) {
|
|
1820
|
-
this.logger.debug("Batch inserting traces", { count: records.length });
|
|
1821
|
-
await this.operations.batchInsert({
|
|
1822
|
-
tableName: TABLE_TRACES,
|
|
1823
|
-
records
|
|
1824
|
-
});
|
|
1825
|
-
}
|
|
1826
1638
|
};
|
|
1827
|
-
function
|
|
1828
|
-
|
|
1829
|
-
if (typeof parsedSnapshot === "string") {
|
|
1830
|
-
try {
|
|
1831
|
-
parsedSnapshot = JSON.parse(row.snapshot);
|
|
1832
|
-
} catch (e) {
|
|
1833
|
-
console.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
|
|
1834
|
-
}
|
|
1835
|
-
}
|
|
1836
|
-
return {
|
|
1837
|
-
workflowName: row.workflow_name,
|
|
1838
|
-
runId: row.run_id,
|
|
1839
|
-
snapshot: parsedSnapshot,
|
|
1840
|
-
createdAt: ensureDate(row.createdAt),
|
|
1841
|
-
updatedAt: ensureDate(row.updatedAt),
|
|
1842
|
-
resourceId: row.resourceId
|
|
1843
|
-
};
|
|
1639
|
+
function escapeSql(str) {
|
|
1640
|
+
return str.replace(/'/g, "''");
|
|
1844
1641
|
}
|
|
1845
1642
|
var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
1846
1643
|
client;
|
|
1847
|
-
|
|
1644
|
+
#db;
|
|
1645
|
+
constructor(config) {
|
|
1848
1646
|
super();
|
|
1647
|
+
const client = resolveLanceConfig(config);
|
|
1849
1648
|
this.client = client;
|
|
1649
|
+
this.#db = new LanceDB({ client });
|
|
1850
1650
|
}
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
// runId,
|
|
1854
|
-
// stepId,
|
|
1855
|
-
// result,
|
|
1856
|
-
// runtimeContext,
|
|
1857
|
-
}) {
|
|
1858
|
-
throw new Error("Method not implemented.");
|
|
1651
|
+
supportsConcurrentUpdates() {
|
|
1652
|
+
return false;
|
|
1859
1653
|
}
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1654
|
+
parseWorkflowRun(row) {
|
|
1655
|
+
let parsedSnapshot = row.snapshot;
|
|
1656
|
+
if (typeof parsedSnapshot === "string") {
|
|
1657
|
+
try {
|
|
1658
|
+
parsedSnapshot = JSON.parse(row.snapshot);
|
|
1659
|
+
} catch (e) {
|
|
1660
|
+
this.logger.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
return {
|
|
1664
|
+
workflowName: row.workflow_name,
|
|
1665
|
+
runId: row.run_id,
|
|
1666
|
+
snapshot: parsedSnapshot,
|
|
1667
|
+
createdAt: ensureDate(row.createdAt),
|
|
1668
|
+
updatedAt: ensureDate(row.updatedAt),
|
|
1669
|
+
resourceId: row.resourceId
|
|
1670
|
+
};
|
|
1671
|
+
}
|
|
1672
|
+
async init() {
|
|
1673
|
+
const schema = TABLE_SCHEMAS[TABLE_WORKFLOW_SNAPSHOT];
|
|
1674
|
+
await this.#db.createTable({ tableName: TABLE_WORKFLOW_SNAPSHOT, schema });
|
|
1675
|
+
await this.#db.alterTable({
|
|
1676
|
+
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
1677
|
+
schema,
|
|
1678
|
+
ifNotExists: ["resourceId"]
|
|
1679
|
+
});
|
|
1680
|
+
}
|
|
1681
|
+
async dangerouslyClearAll() {
|
|
1682
|
+
await this.#db.clearTable({ tableName: TABLE_WORKFLOW_SNAPSHOT });
|
|
1683
|
+
}
|
|
1684
|
+
async updateWorkflowResults(_args) {
|
|
1685
|
+
throw new Error(
|
|
1686
|
+
"updateWorkflowResults is not implemented for LanceDB storage. LanceDB does not support atomic read-modify-write operations needed for concurrent workflow updates."
|
|
1687
|
+
);
|
|
1688
|
+
}
|
|
1689
|
+
async updateWorkflowState(_args) {
|
|
1690
|
+
throw new Error(
|
|
1691
|
+
"updateWorkflowState is not implemented for LanceDB storage. LanceDB does not support atomic read-modify-write operations needed for concurrent workflow updates."
|
|
1692
|
+
);
|
|
1866
1693
|
}
|
|
1867
1694
|
async persistWorkflowSnapshot({
|
|
1868
1695
|
workflowName,
|
|
1869
1696
|
runId,
|
|
1870
1697
|
resourceId,
|
|
1871
|
-
snapshot
|
|
1698
|
+
snapshot,
|
|
1699
|
+
createdAt,
|
|
1700
|
+
updatedAt
|
|
1872
1701
|
}) {
|
|
1873
1702
|
try {
|
|
1874
1703
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1875
|
-
const query = table.query().where(`workflow_name = '${workflowName}' AND run_id = '${runId}'`);
|
|
1704
|
+
const query = table.query().where(`workflow_name = '${escapeSql(workflowName)}' AND run_id = '${escapeSql(runId)}'`);
|
|
1876
1705
|
const records = await query.toArray();
|
|
1877
|
-
let
|
|
1878
|
-
const now = Date.now();
|
|
1706
|
+
let createdAtValue;
|
|
1707
|
+
const now = createdAt?.getTime() ?? Date.now();
|
|
1879
1708
|
if (records.length > 0) {
|
|
1880
|
-
|
|
1709
|
+
createdAtValue = records[0].createdAt ?? now;
|
|
1881
1710
|
} else {
|
|
1882
|
-
|
|
1711
|
+
createdAtValue = now;
|
|
1883
1712
|
}
|
|
1713
|
+
const { status, value, ...rest } = snapshot;
|
|
1884
1714
|
const record = {
|
|
1885
1715
|
workflow_name: workflowName,
|
|
1886
1716
|
run_id: runId,
|
|
1887
1717
|
resourceId,
|
|
1888
|
-
snapshot: JSON.stringify(
|
|
1889
|
-
|
|
1890
|
-
|
|
1718
|
+
snapshot: JSON.stringify({ status, value, ...rest }),
|
|
1719
|
+
// this is to ensure status is always just before value, for when querying the db by status
|
|
1720
|
+
createdAt: createdAtValue,
|
|
1721
|
+
updatedAt: updatedAt ?? now
|
|
1891
1722
|
};
|
|
1892
1723
|
await table.mergeInsert(["workflow_name", "run_id"]).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
1893
1724
|
} catch (error) {
|
|
1894
1725
|
throw new MastraError(
|
|
1895
1726
|
{
|
|
1896
|
-
id: "
|
|
1727
|
+
id: createStorageErrorId("LANCE", "PERSIST_WORKFLOW_SNAPSHOT", "FAILED"),
|
|
1897
1728
|
domain: ErrorDomain.STORAGE,
|
|
1898
1729
|
category: ErrorCategory.THIRD_PARTY,
|
|
1899
1730
|
details: { workflowName, runId }
|
|
@@ -1908,13 +1739,13 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1908
1739
|
}) {
|
|
1909
1740
|
try {
|
|
1910
1741
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1911
|
-
const query = table.query().where(`workflow_name = '${workflowName}' AND run_id = '${runId}'`);
|
|
1742
|
+
const query = table.query().where(`workflow_name = '${escapeSql(workflowName)}' AND run_id = '${escapeSql(runId)}'`);
|
|
1912
1743
|
const records = await query.toArray();
|
|
1913
1744
|
return records.length > 0 ? JSON.parse(records[0].snapshot) : null;
|
|
1914
1745
|
} catch (error) {
|
|
1915
1746
|
throw new MastraError(
|
|
1916
1747
|
{
|
|
1917
|
-
id: "
|
|
1748
|
+
id: createStorageErrorId("LANCE", "LOAD_WORKFLOW_SNAPSHOT", "FAILED"),
|
|
1918
1749
|
domain: ErrorDomain.STORAGE,
|
|
1919
1750
|
category: ErrorCategory.THIRD_PARTY,
|
|
1920
1751
|
details: { workflowName, runId }
|
|
@@ -1926,19 +1757,19 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1926
1757
|
async getWorkflowRunById(args) {
|
|
1927
1758
|
try {
|
|
1928
1759
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1929
|
-
let whereClause = `run_id = '${args.runId}'`;
|
|
1760
|
+
let whereClause = `run_id = '${escapeSql(args.runId)}'`;
|
|
1930
1761
|
if (args.workflowName) {
|
|
1931
|
-
whereClause += ` AND workflow_name = '${args.workflowName}'`;
|
|
1762
|
+
whereClause += ` AND workflow_name = '${escapeSql(args.workflowName)}'`;
|
|
1932
1763
|
}
|
|
1933
1764
|
const query = table.query().where(whereClause);
|
|
1934
1765
|
const records = await query.toArray();
|
|
1935
1766
|
if (records.length === 0) return null;
|
|
1936
1767
|
const record = records[0];
|
|
1937
|
-
return parseWorkflowRun(record);
|
|
1768
|
+
return this.parseWorkflowRun(record);
|
|
1938
1769
|
} catch (error) {
|
|
1939
1770
|
throw new MastraError(
|
|
1940
1771
|
{
|
|
1941
|
-
id: "
|
|
1772
|
+
id: createStorageErrorId("LANCE", "GET_WORKFLOW_RUN_BY_ID", "FAILED"),
|
|
1942
1773
|
domain: ErrorDomain.STORAGE,
|
|
1943
1774
|
category: ErrorCategory.THIRD_PARTY,
|
|
1944
1775
|
details: { runId: args.runId, workflowName: args.workflowName ?? "" }
|
|
@@ -1947,359 +1778,182 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1947
1778
|
);
|
|
1948
1779
|
}
|
|
1949
1780
|
}
|
|
1950
|
-
async
|
|
1781
|
+
async deleteWorkflowRunById({ runId, workflowName }) {
|
|
1951
1782
|
try {
|
|
1952
1783
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
if (args?.workflowName) {
|
|
1956
|
-
conditions.push(`workflow_name = '${args.workflowName.replace(/'/g, "''")}'`);
|
|
1957
|
-
}
|
|
1958
|
-
if (args?.resourceId) {
|
|
1959
|
-
conditions.push(`\`resourceId\` = '${args.resourceId}'`);
|
|
1960
|
-
}
|
|
1961
|
-
if (args?.fromDate instanceof Date) {
|
|
1962
|
-
conditions.push(`\`createdAt\` >= ${args.fromDate.getTime()}`);
|
|
1963
|
-
}
|
|
1964
|
-
if (args?.toDate instanceof Date) {
|
|
1965
|
-
conditions.push(`\`createdAt\` <= ${args.toDate.getTime()}`);
|
|
1966
|
-
}
|
|
1967
|
-
let total = 0;
|
|
1968
|
-
if (conditions.length > 0) {
|
|
1969
|
-
query = query.where(conditions.join(" AND "));
|
|
1970
|
-
total = await table.countRows(conditions.join(" AND "));
|
|
1971
|
-
} else {
|
|
1972
|
-
total = await table.countRows();
|
|
1973
|
-
}
|
|
1974
|
-
if (args?.limit) {
|
|
1975
|
-
query.limit(args.limit);
|
|
1976
|
-
}
|
|
1977
|
-
if (args?.offset) {
|
|
1978
|
-
query.offset(args.offset);
|
|
1979
|
-
}
|
|
1980
|
-
const records = await query.toArray();
|
|
1981
|
-
return {
|
|
1982
|
-
runs: records.map((record) => parseWorkflowRun(record)),
|
|
1983
|
-
total: total || records.length
|
|
1984
|
-
};
|
|
1784
|
+
const whereClause = `run_id = '${escapeSql(runId)}' AND workflow_name = '${escapeSql(workflowName)}'`;
|
|
1785
|
+
await table.delete(whereClause);
|
|
1985
1786
|
} catch (error) {
|
|
1986
1787
|
throw new MastraError(
|
|
1987
1788
|
{
|
|
1988
|
-
id: "
|
|
1789
|
+
id: createStorageErrorId("LANCE", "DELETE_WORKFLOW_RUN_BY_ID", "FAILED"),
|
|
1989
1790
|
domain: ErrorDomain.STORAGE,
|
|
1990
1791
|
category: ErrorCategory.THIRD_PARTY,
|
|
1991
|
-
details: {
|
|
1792
|
+
details: { runId, workflowName }
|
|
1992
1793
|
},
|
|
1993
1794
|
error
|
|
1994
1795
|
);
|
|
1995
1796
|
}
|
|
1996
1797
|
}
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
// src/storage/index.ts
|
|
2000
|
-
var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
2001
|
-
stores;
|
|
2002
|
-
lanceClient;
|
|
2003
|
-
/**
|
|
2004
|
-
* Creates a new instance of LanceStorage
|
|
2005
|
-
* @param uri The URI to connect to LanceDB
|
|
2006
|
-
* @param options connection options
|
|
2007
|
-
*
|
|
2008
|
-
* Usage:
|
|
2009
|
-
*
|
|
2010
|
-
* Connect to a local database
|
|
2011
|
-
* ```ts
|
|
2012
|
-
* const store = await LanceStorage.create('/path/to/db');
|
|
2013
|
-
* ```
|
|
2014
|
-
*
|
|
2015
|
-
* Connect to a LanceDB cloud database
|
|
2016
|
-
* ```ts
|
|
2017
|
-
* const store = await LanceStorage.create('db://host:port');
|
|
2018
|
-
* ```
|
|
2019
|
-
*
|
|
2020
|
-
* Connect to a cloud database
|
|
2021
|
-
* ```ts
|
|
2022
|
-
* const store = await LanceStorage.create('s3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
2023
|
-
* ```
|
|
2024
|
-
*/
|
|
2025
|
-
static async create(name, uri, options) {
|
|
2026
|
-
const instance = new _LanceStorage(name);
|
|
1798
|
+
async listWorkflowRuns(args) {
|
|
2027
1799
|
try {
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
traces: new StoreTracesLance({ client: instance.lanceClient, operations }),
|
|
2034
|
-
scores: new StoreScoresLance({ client: instance.lanceClient }),
|
|
2035
|
-
memory: new StoreMemoryLance({ client: instance.lanceClient, operations }),
|
|
2036
|
-
legacyEvals: new StoreLegacyEvalsLance({ client: instance.lanceClient })
|
|
2037
|
-
};
|
|
2038
|
-
return instance;
|
|
2039
|
-
} catch (e) {
|
|
2040
|
-
throw new MastraError(
|
|
2041
|
-
{
|
|
2042
|
-
id: "STORAGE_LANCE_STORAGE_CONNECT_FAILED",
|
|
2043
|
-
domain: ErrorDomain.STORAGE,
|
|
2044
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
2045
|
-
text: `Failed to connect to LanceDB: ${e.message || e}`,
|
|
2046
|
-
details: { uri, optionsProvided: !!options }
|
|
2047
|
-
},
|
|
2048
|
-
e
|
|
2049
|
-
);
|
|
2050
|
-
}
|
|
2051
|
-
}
|
|
2052
|
-
/**
|
|
2053
|
-
* @internal
|
|
2054
|
-
* Private constructor to enforce using the create factory method
|
|
2055
|
-
*/
|
|
2056
|
-
constructor(name) {
|
|
2057
|
-
super({ name });
|
|
2058
|
-
const operations = new StoreOperationsLance({ client: this.lanceClient });
|
|
2059
|
-
this.stores = {
|
|
2060
|
-
operations: new StoreOperationsLance({ client: this.lanceClient }),
|
|
2061
|
-
workflows: new StoreWorkflowsLance({ client: this.lanceClient }),
|
|
2062
|
-
traces: new StoreTracesLance({ client: this.lanceClient, operations }),
|
|
2063
|
-
scores: new StoreScoresLance({ client: this.lanceClient }),
|
|
2064
|
-
legacyEvals: new StoreLegacyEvalsLance({ client: this.lanceClient }),
|
|
2065
|
-
memory: new StoreMemoryLance({ client: this.lanceClient, operations })
|
|
2066
|
-
};
|
|
2067
|
-
}
|
|
2068
|
-
async createTable({
|
|
2069
|
-
tableName,
|
|
2070
|
-
schema
|
|
2071
|
-
}) {
|
|
2072
|
-
return this.stores.operations.createTable({ tableName, schema });
|
|
2073
|
-
}
|
|
2074
|
-
async dropTable({ tableName }) {
|
|
2075
|
-
return this.stores.operations.dropTable({ tableName });
|
|
2076
|
-
}
|
|
2077
|
-
async alterTable({
|
|
2078
|
-
tableName,
|
|
2079
|
-
schema,
|
|
2080
|
-
ifNotExists
|
|
2081
|
-
}) {
|
|
2082
|
-
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2083
|
-
}
|
|
2084
|
-
async clearTable({ tableName }) {
|
|
2085
|
-
return this.stores.operations.clearTable({ tableName });
|
|
2086
|
-
}
|
|
2087
|
-
async insert({ tableName, record }) {
|
|
2088
|
-
return this.stores.operations.insert({ tableName, record });
|
|
2089
|
-
}
|
|
2090
|
-
async batchInsert({ tableName, records }) {
|
|
2091
|
-
return this.stores.operations.batchInsert({ tableName, records });
|
|
2092
|
-
}
|
|
2093
|
-
async load({ tableName, keys }) {
|
|
2094
|
-
return this.stores.operations.load({ tableName, keys });
|
|
2095
|
-
}
|
|
2096
|
-
async getThreadById({ threadId }) {
|
|
2097
|
-
return this.stores.memory.getThreadById({ threadId });
|
|
2098
|
-
}
|
|
2099
|
-
async getThreadsByResourceId({ resourceId }) {
|
|
2100
|
-
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2101
|
-
}
|
|
2102
|
-
/**
|
|
2103
|
-
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
2104
|
-
* @param thread - The thread to save
|
|
2105
|
-
* @returns The saved thread
|
|
2106
|
-
*/
|
|
2107
|
-
async saveThread({ thread }) {
|
|
2108
|
-
return this.stores.memory.saveThread({ thread });
|
|
2109
|
-
}
|
|
2110
|
-
async updateThread({
|
|
2111
|
-
id,
|
|
2112
|
-
title,
|
|
2113
|
-
metadata
|
|
2114
|
-
}) {
|
|
2115
|
-
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2116
|
-
}
|
|
2117
|
-
async deleteThread({ threadId }) {
|
|
2118
|
-
return this.stores.memory.deleteThread({ threadId });
|
|
2119
|
-
}
|
|
2120
|
-
get supports() {
|
|
2121
|
-
return {
|
|
2122
|
-
selectByIncludeResourceScope: true,
|
|
2123
|
-
resourceWorkingMemory: true,
|
|
2124
|
-
hasColumn: true,
|
|
2125
|
-
createTable: true,
|
|
2126
|
-
deleteMessages: false,
|
|
2127
|
-
getScoresBySpan: true
|
|
2128
|
-
};
|
|
2129
|
-
}
|
|
2130
|
-
async getResourceById({ resourceId }) {
|
|
2131
|
-
return this.stores.memory.getResourceById({ resourceId });
|
|
2132
|
-
}
|
|
2133
|
-
async saveResource({ resource }) {
|
|
2134
|
-
return this.stores.memory.saveResource({ resource });
|
|
2135
|
-
}
|
|
2136
|
-
async updateResource({
|
|
2137
|
-
resourceId,
|
|
2138
|
-
workingMemory,
|
|
2139
|
-
metadata
|
|
2140
|
-
}) {
|
|
2141
|
-
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2142
|
-
}
|
|
2143
|
-
/**
|
|
2144
|
-
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
2145
|
-
* @param records - The sorted array of records to process
|
|
2146
|
-
* @param include - The array of include specifications with context parameters
|
|
2147
|
-
* @returns The processed array with context messages included
|
|
2148
|
-
*/
|
|
2149
|
-
processMessagesWithContext(records, include) {
|
|
2150
|
-
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
2151
|
-
if (messagesWithContext.length === 0) {
|
|
2152
|
-
return records;
|
|
2153
|
-
}
|
|
2154
|
-
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
2155
|
-
records.forEach((message, index) => {
|
|
2156
|
-
messageIndexMap.set(message.id, index);
|
|
2157
|
-
});
|
|
2158
|
-
const additionalIndices = /* @__PURE__ */ new Set();
|
|
2159
|
-
for (const item of messagesWithContext) {
|
|
2160
|
-
const messageIndex = messageIndexMap.get(item.id);
|
|
2161
|
-
if (messageIndex !== void 0) {
|
|
2162
|
-
if (item.withPreviousMessages) {
|
|
2163
|
-
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
2164
|
-
for (let i = startIdx; i < messageIndex; i++) {
|
|
2165
|
-
additionalIndices.add(i);
|
|
2166
|
-
}
|
|
2167
|
-
}
|
|
2168
|
-
if (item.withNextMessages) {
|
|
2169
|
-
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
2170
|
-
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
2171
|
-
additionalIndices.add(i);
|
|
2172
|
-
}
|
|
2173
|
-
}
|
|
1800
|
+
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1801
|
+
let query = table.query();
|
|
1802
|
+
const conditions = [];
|
|
1803
|
+
if (args?.workflowName) {
|
|
1804
|
+
conditions.push(`workflow_name = '${escapeSql(args.workflowName)}'`);
|
|
2174
1805
|
}
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
}
|
|
2179
|
-
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
2180
|
-
const allIndices = /* @__PURE__ */ new Set();
|
|
2181
|
-
records.forEach((record, index) => {
|
|
2182
|
-
if (originalMatchIds.has(record.id)) {
|
|
2183
|
-
allIndices.add(index);
|
|
1806
|
+
if (args?.status) {
|
|
1807
|
+
const escapedStatus = args.status.replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/%/g, "\\%").replace(/_/g, "\\_");
|
|
1808
|
+
conditions.push(`\`snapshot\` LIKE '%"status":"${escapedStatus}","value"%'`);
|
|
2184
1809
|
}
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
return this.stores.workflows.getWorkflowRuns(args);
|
|
2235
|
-
}
|
|
2236
|
-
async getWorkflowRunById(args) {
|
|
2237
|
-
return this.stores.workflows.getWorkflowRunById(args);
|
|
2238
|
-
}
|
|
2239
|
-
async updateWorkflowResults({
|
|
2240
|
-
workflowName,
|
|
2241
|
-
runId,
|
|
2242
|
-
stepId,
|
|
2243
|
-
result,
|
|
2244
|
-
runtimeContext
|
|
2245
|
-
}) {
|
|
2246
|
-
return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, runtimeContext });
|
|
2247
|
-
}
|
|
2248
|
-
async updateWorkflowState({
|
|
2249
|
-
workflowName,
|
|
2250
|
-
runId,
|
|
2251
|
-
opts
|
|
2252
|
-
}) {
|
|
2253
|
-
return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
|
|
2254
|
-
}
|
|
2255
|
-
async persistWorkflowSnapshot({
|
|
2256
|
-
workflowName,
|
|
2257
|
-
runId,
|
|
2258
|
-
resourceId,
|
|
2259
|
-
snapshot
|
|
2260
|
-
}) {
|
|
2261
|
-
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot });
|
|
2262
|
-
}
|
|
2263
|
-
async loadWorkflowSnapshot({
|
|
2264
|
-
workflowName,
|
|
2265
|
-
runId
|
|
2266
|
-
}) {
|
|
2267
|
-
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2268
|
-
}
|
|
2269
|
-
async getScoreById({ id: _id }) {
|
|
2270
|
-
return this.stores.scores.getScoreById({ id: _id });
|
|
2271
|
-
}
|
|
2272
|
-
async getScoresByScorerId({
|
|
2273
|
-
scorerId,
|
|
2274
|
-
source,
|
|
2275
|
-
entityId,
|
|
2276
|
-
entityType,
|
|
2277
|
-
pagination
|
|
2278
|
-
}) {
|
|
2279
|
-
return this.stores.scores.getScoresByScorerId({ scorerId, source, pagination, entityId, entityType });
|
|
2280
|
-
}
|
|
2281
|
-
async saveScore(_score) {
|
|
2282
|
-
return this.stores.scores.saveScore(_score);
|
|
1810
|
+
if (args?.resourceId) {
|
|
1811
|
+
conditions.push(`\`resourceId\` = '${escapeSql(args.resourceId)}'`);
|
|
1812
|
+
}
|
|
1813
|
+
if (args?.fromDate instanceof Date) {
|
|
1814
|
+
conditions.push(`\`createdAt\` >= ${args.fromDate.getTime()}`);
|
|
1815
|
+
}
|
|
1816
|
+
if (args?.toDate instanceof Date) {
|
|
1817
|
+
conditions.push(`\`createdAt\` <= ${args.toDate.getTime()}`);
|
|
1818
|
+
}
|
|
1819
|
+
let total = 0;
|
|
1820
|
+
if (conditions.length > 0) {
|
|
1821
|
+
query = query.where(conditions.join(" AND "));
|
|
1822
|
+
total = await table.countRows(conditions.join(" AND "));
|
|
1823
|
+
} else {
|
|
1824
|
+
total = await table.countRows();
|
|
1825
|
+
}
|
|
1826
|
+
if (args?.perPage !== void 0 && args?.page !== void 0) {
|
|
1827
|
+
const normalizedPerPage = normalizePerPage(args.perPage, Number.MAX_SAFE_INTEGER);
|
|
1828
|
+
if (args.page < 0 || !Number.isInteger(args.page)) {
|
|
1829
|
+
throw new MastraError(
|
|
1830
|
+
{
|
|
1831
|
+
id: createStorageErrorId("LANCE", "LIST_WORKFLOW_RUNS", "INVALID_PAGINATION"),
|
|
1832
|
+
domain: ErrorDomain.STORAGE,
|
|
1833
|
+
category: ErrorCategory.USER,
|
|
1834
|
+
details: { page: args.page, perPage: args.perPage }
|
|
1835
|
+
},
|
|
1836
|
+
new Error(`Invalid pagination parameters: page=${args.page}, perPage=${args.perPage}`)
|
|
1837
|
+
);
|
|
1838
|
+
}
|
|
1839
|
+
const offset = args.page * normalizedPerPage;
|
|
1840
|
+
query.limit(normalizedPerPage);
|
|
1841
|
+
query.offset(offset);
|
|
1842
|
+
}
|
|
1843
|
+
const records = await query.toArray();
|
|
1844
|
+
return {
|
|
1845
|
+
runs: records.map((record) => this.parseWorkflowRun(record)),
|
|
1846
|
+
total: total || records.length
|
|
1847
|
+
};
|
|
1848
|
+
} catch (error) {
|
|
1849
|
+
throw new MastraError(
|
|
1850
|
+
{
|
|
1851
|
+
id: createStorageErrorId("LANCE", "LIST_WORKFLOW_RUNS", "FAILED"),
|
|
1852
|
+
domain: ErrorDomain.STORAGE,
|
|
1853
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1854
|
+
details: { resourceId: args?.resourceId ?? "", workflowName: args?.workflowName ?? "" }
|
|
1855
|
+
},
|
|
1856
|
+
error
|
|
1857
|
+
);
|
|
1858
|
+
}
|
|
2283
1859
|
}
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
1860
|
+
};
|
|
1861
|
+
|
|
1862
|
+
// src/storage/index.ts
|
|
1863
|
+
var LanceStorage = class _LanceStorage extends MastraCompositeStore {
|
|
1864
|
+
stores;
|
|
1865
|
+
lanceClient;
|
|
1866
|
+
/**
|
|
1867
|
+
* Creates a new instance of LanceStorage
|
|
1868
|
+
* @param id The unique identifier for this storage instance
|
|
1869
|
+
* @param name The name for this storage instance
|
|
1870
|
+
* @param uri The URI to connect to LanceDB
|
|
1871
|
+
* @param connectionOptions connection options for LanceDB
|
|
1872
|
+
* @param storageOptions storage options including disableInit
|
|
1873
|
+
*
|
|
1874
|
+
* Usage:
|
|
1875
|
+
*
|
|
1876
|
+
* Connect to a local database
|
|
1877
|
+
* ```ts
|
|
1878
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', '/path/to/db');
|
|
1879
|
+
* ```
|
|
1880
|
+
*
|
|
1881
|
+
* Connect to a LanceDB cloud database
|
|
1882
|
+
* ```ts
|
|
1883
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 'db://host:port');
|
|
1884
|
+
* ```
|
|
1885
|
+
*
|
|
1886
|
+
* Connect to a cloud database
|
|
1887
|
+
* ```ts
|
|
1888
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 's3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
1889
|
+
* ```
|
|
1890
|
+
*
|
|
1891
|
+
* Disable auto-init for runtime (after CI/CD has run migrations)
|
|
1892
|
+
* ```ts
|
|
1893
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', '/path/to/db', undefined, { disableInit: true });
|
|
1894
|
+
* ```
|
|
1895
|
+
*/
|
|
1896
|
+
static async create(id, name, uri, connectionOptions, storageOptions) {
|
|
1897
|
+
const instance = new _LanceStorage(id, name, storageOptions?.disableInit);
|
|
1898
|
+
try {
|
|
1899
|
+
instance.lanceClient = await connect(uri, connectionOptions);
|
|
1900
|
+
instance.stores = {
|
|
1901
|
+
workflows: new StoreWorkflowsLance({ client: instance.lanceClient }),
|
|
1902
|
+
scores: new StoreScoresLance({ client: instance.lanceClient }),
|
|
1903
|
+
memory: new StoreMemoryLance({ client: instance.lanceClient })
|
|
1904
|
+
};
|
|
1905
|
+
return instance;
|
|
1906
|
+
} catch (e) {
|
|
1907
|
+
throw new MastraError(
|
|
1908
|
+
{
|
|
1909
|
+
id: createStorageErrorId("LANCE", "CONNECT", "FAILED"),
|
|
1910
|
+
domain: ErrorDomain.STORAGE,
|
|
1911
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1912
|
+
text: `Failed to connect to LanceDB: ${e.message || e}`,
|
|
1913
|
+
details: { uri, optionsProvided: !!connectionOptions }
|
|
1914
|
+
},
|
|
1915
|
+
e
|
|
1916
|
+
);
|
|
1917
|
+
}
|
|
2289
1918
|
}
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
1919
|
+
/**
|
|
1920
|
+
* Creates a new instance of LanceStorage from a pre-configured LanceDB connection.
|
|
1921
|
+
* Use this when you need to configure the connection before initialization.
|
|
1922
|
+
*
|
|
1923
|
+
* @param id The unique identifier for this storage instance
|
|
1924
|
+
* @param name The name for this storage instance
|
|
1925
|
+
* @param client Pre-configured LanceDB connection
|
|
1926
|
+
* @param options Storage options including disableInit
|
|
1927
|
+
*
|
|
1928
|
+
* @example
|
|
1929
|
+
* ```typescript
|
|
1930
|
+
* import { connect } from '@lancedb/lancedb';
|
|
1931
|
+
*
|
|
1932
|
+
* const client = await connect('/path/to/db', {
|
|
1933
|
+
* // Custom connection options
|
|
1934
|
+
* });
|
|
1935
|
+
*
|
|
1936
|
+
* const store = LanceStorage.fromClient('my-id', 'MyStorage', client);
|
|
1937
|
+
* ```
|
|
1938
|
+
*/
|
|
1939
|
+
static fromClient(id, name, client, options) {
|
|
1940
|
+
const instance = new _LanceStorage(id, name, options?.disableInit);
|
|
1941
|
+
instance.lanceClient = client;
|
|
1942
|
+
instance.stores = {
|
|
1943
|
+
workflows: new StoreWorkflowsLance({ client }),
|
|
1944
|
+
scores: new StoreScoresLance({ client }),
|
|
1945
|
+
memory: new StoreMemoryLance({ client })
|
|
1946
|
+
};
|
|
1947
|
+
return instance;
|
|
2296
1948
|
}
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
1949
|
+
/**
|
|
1950
|
+
* @internal
|
|
1951
|
+
* Private constructor to enforce using the create factory method.
|
|
1952
|
+
* Note: stores is initialized in create() after the lanceClient is connected.
|
|
1953
|
+
*/
|
|
1954
|
+
constructor(id, name, disableInit) {
|
|
1955
|
+
super({ id, name, disableInit });
|
|
1956
|
+
this.stores = {};
|
|
2303
1957
|
}
|
|
2304
1958
|
};
|
|
2305
1959
|
var LanceFilterTranslator = class extends BaseFilterTranslator {
|
|
@@ -2648,14 +2302,14 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2648
2302
|
* ```
|
|
2649
2303
|
*/
|
|
2650
2304
|
static async create(uri, options) {
|
|
2651
|
-
const instance = new _LanceVectorStore();
|
|
2305
|
+
const instance = new _LanceVectorStore(options?.id || crypto.randomUUID());
|
|
2652
2306
|
try {
|
|
2653
2307
|
instance.lanceClient = await connect(uri, options);
|
|
2654
2308
|
return instance;
|
|
2655
2309
|
} catch (e) {
|
|
2656
2310
|
throw new MastraError(
|
|
2657
2311
|
{
|
|
2658
|
-
id: "
|
|
2312
|
+
id: createVectorErrorId("LANCE", "CONNECT", "FAILED"),
|
|
2659
2313
|
domain: ErrorDomain.STORAGE,
|
|
2660
2314
|
category: ErrorCategory.THIRD_PARTY,
|
|
2661
2315
|
details: { uri }
|
|
@@ -2668,8 +2322,8 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2668
2322
|
* @internal
|
|
2669
2323
|
* Private constructor to enforce using the create factory method
|
|
2670
2324
|
*/
|
|
2671
|
-
constructor() {
|
|
2672
|
-
super();
|
|
2325
|
+
constructor(id) {
|
|
2326
|
+
super({ id });
|
|
2673
2327
|
}
|
|
2674
2328
|
close() {
|
|
2675
2329
|
if (this.lanceClient) {
|
|
@@ -2678,6 +2332,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2678
2332
|
}
|
|
2679
2333
|
async query({
|
|
2680
2334
|
tableName,
|
|
2335
|
+
indexName,
|
|
2681
2336
|
queryVector,
|
|
2682
2337
|
filter,
|
|
2683
2338
|
includeVector = false,
|
|
@@ -2685,56 +2340,78 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2685
2340
|
columns = [],
|
|
2686
2341
|
includeAllColumns = false
|
|
2687
2342
|
}) {
|
|
2343
|
+
const resolvedTableName = tableName ?? indexName;
|
|
2344
|
+
if (!queryVector) {
|
|
2345
|
+
throw new MastraError({
|
|
2346
|
+
id: createVectorErrorId("LANCE", "QUERY", "MISSING_VECTOR"),
|
|
2347
|
+
text: "queryVector is required for Lance queries. Metadata-only queries are not supported by this vector store.",
|
|
2348
|
+
domain: ErrorDomain.STORAGE,
|
|
2349
|
+
category: ErrorCategory.USER,
|
|
2350
|
+
details: { indexName: resolvedTableName }
|
|
2351
|
+
});
|
|
2352
|
+
}
|
|
2688
2353
|
try {
|
|
2689
2354
|
if (!this.lanceClient) {
|
|
2690
2355
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2691
2356
|
}
|
|
2692
|
-
if (!
|
|
2693
|
-
throw new Error("tableName is required");
|
|
2694
|
-
}
|
|
2695
|
-
if (!queryVector) {
|
|
2696
|
-
throw new Error("queryVector is required");
|
|
2357
|
+
if (!resolvedTableName) {
|
|
2358
|
+
throw new Error("tableName or indexName is required");
|
|
2697
2359
|
}
|
|
2698
2360
|
} catch (error) {
|
|
2699
2361
|
throw new MastraError(
|
|
2700
2362
|
{
|
|
2701
|
-
id: "
|
|
2363
|
+
id: createVectorErrorId("LANCE", "QUERY", "INVALID_ARGS"),
|
|
2702
2364
|
domain: ErrorDomain.STORAGE,
|
|
2703
2365
|
category: ErrorCategory.USER,
|
|
2704
|
-
text:
|
|
2705
|
-
details: { tableName }
|
|
2366
|
+
text: error instanceof Error ? error.message : "Invalid query arguments",
|
|
2367
|
+
details: { tableName: resolvedTableName }
|
|
2706
2368
|
},
|
|
2707
2369
|
error
|
|
2708
2370
|
);
|
|
2709
2371
|
}
|
|
2710
2372
|
try {
|
|
2711
|
-
const
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2373
|
+
const tables = await this.lanceClient.tableNames();
|
|
2374
|
+
if (!tables.includes(resolvedTableName)) {
|
|
2375
|
+
this.logger.debug(`Table ${resolvedTableName} does not exist. Returning empty results.`);
|
|
2376
|
+
return [];
|
|
2715
2377
|
}
|
|
2378
|
+
const table = await this.lanceClient.openTable(resolvedTableName);
|
|
2716
2379
|
let query = table.search(queryVector);
|
|
2717
2380
|
if (filter && Object.keys(filter).length > 0) {
|
|
2718
2381
|
const whereClause = this.filterTranslator(filter);
|
|
2719
2382
|
this.logger.debug(`Where clause generated: ${whereClause}`);
|
|
2383
|
+
const schema = await table.schema();
|
|
2384
|
+
const schemaColumns = new Set(schema.fields.map((f) => f.name));
|
|
2385
|
+
const filterColumns = this.extractFilterColumns(whereClause);
|
|
2386
|
+
const missingColumns = filterColumns.filter((col) => !schemaColumns.has(col));
|
|
2387
|
+
if (missingColumns.length > 0) {
|
|
2388
|
+
this.logger.debug(
|
|
2389
|
+
`Filter references columns not in schema: ${missingColumns.join(", ")}. Returning empty results.`
|
|
2390
|
+
);
|
|
2391
|
+
return [];
|
|
2392
|
+
}
|
|
2720
2393
|
query = query.where(whereClause);
|
|
2721
2394
|
}
|
|
2722
|
-
if (!includeAllColumns &&
|
|
2395
|
+
if (!includeAllColumns && columns.length > 0) {
|
|
2396
|
+
const selectColumns = [...columns];
|
|
2397
|
+
if (!selectColumns.includes("id")) {
|
|
2398
|
+
selectColumns.push("id");
|
|
2399
|
+
}
|
|
2723
2400
|
query = query.select(selectColumns);
|
|
2724
2401
|
}
|
|
2725
2402
|
query = query.limit(topK);
|
|
2726
2403
|
const results = await query.toArray();
|
|
2727
2404
|
return results.map((result) => {
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
}
|
|
2405
|
+
let metadata = {};
|
|
2406
|
+
if (result._metadata_json) {
|
|
2407
|
+
try {
|
|
2408
|
+
metadata = JSON.parse(result._metadata_json);
|
|
2409
|
+
} catch {
|
|
2410
|
+
metadata = this.extractFlatMetadata(result);
|
|
2735
2411
|
}
|
|
2736
|
-
}
|
|
2737
|
-
|
|
2412
|
+
} else {
|
|
2413
|
+
metadata = this.extractFlatMetadata(result);
|
|
2414
|
+
}
|
|
2738
2415
|
return {
|
|
2739
2416
|
id: String(result.id || ""),
|
|
2740
2417
|
metadata,
|
|
@@ -2746,10 +2423,10 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2746
2423
|
} catch (error) {
|
|
2747
2424
|
throw new MastraError(
|
|
2748
2425
|
{
|
|
2749
|
-
id: "
|
|
2426
|
+
id: createVectorErrorId("LANCE", "QUERY", "FAILED"),
|
|
2750
2427
|
domain: ErrorDomain.STORAGE,
|
|
2751
2428
|
category: ErrorCategory.THIRD_PARTY,
|
|
2752
|
-
details: { tableName, includeVector, columnsCount: columns?.length, includeAllColumns }
|
|
2429
|
+
details: { tableName: resolvedTableName, includeVector, columnsCount: columns?.length, includeAllColumns }
|
|
2753
2430
|
},
|
|
2754
2431
|
error
|
|
2755
2432
|
);
|
|
@@ -2784,13 +2461,14 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2784
2461
|
const translator = new LanceFilterTranslator();
|
|
2785
2462
|
return translator.translate(prefixedFilter);
|
|
2786
2463
|
}
|
|
2787
|
-
async upsert({ tableName, vectors, metadata = [], ids = [] }) {
|
|
2464
|
+
async upsert({ tableName, indexName, vectors, metadata = [], ids = [] }) {
|
|
2465
|
+
const resolvedTableName = tableName ?? indexName;
|
|
2788
2466
|
try {
|
|
2789
2467
|
if (!this.lanceClient) {
|
|
2790
2468
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2791
2469
|
}
|
|
2792
|
-
if (!
|
|
2793
|
-
throw new Error("tableName is required");
|
|
2470
|
+
if (!resolvedTableName) {
|
|
2471
|
+
throw new Error("tableName or indexName is required");
|
|
2794
2472
|
}
|
|
2795
2473
|
if (!vectors || !Array.isArray(vectors) || vectors.length === 0) {
|
|
2796
2474
|
throw new Error("vectors array is required and must not be empty");
|
|
@@ -2798,21 +2476,24 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2798
2476
|
} catch (error) {
|
|
2799
2477
|
throw new MastraError(
|
|
2800
2478
|
{
|
|
2801
|
-
id: "
|
|
2479
|
+
id: createVectorErrorId("LANCE", "UPSERT", "INVALID_ARGS"),
|
|
2802
2480
|
domain: ErrorDomain.STORAGE,
|
|
2803
2481
|
category: ErrorCategory.USER,
|
|
2804
|
-
text:
|
|
2805
|
-
details: { tableName }
|
|
2482
|
+
text: error instanceof Error ? error.message : "Invalid upsert arguments",
|
|
2483
|
+
details: { tableName: resolvedTableName }
|
|
2806
2484
|
},
|
|
2807
2485
|
error
|
|
2808
2486
|
);
|
|
2809
2487
|
}
|
|
2810
2488
|
try {
|
|
2811
2489
|
const tables = await this.lanceClient.tableNames();
|
|
2812
|
-
|
|
2813
|
-
|
|
2490
|
+
const tableExists = tables.includes(resolvedTableName);
|
|
2491
|
+
let table = null;
|
|
2492
|
+
if (!tableExists) {
|
|
2493
|
+
this.logger.debug(`Table ${resolvedTableName} does not exist. Creating it with the first upsert data.`);
|
|
2494
|
+
} else {
|
|
2495
|
+
table = await this.lanceClient.openTable(resolvedTableName);
|
|
2814
2496
|
}
|
|
2815
|
-
const table = await this.lanceClient.openTable(tableName);
|
|
2816
2497
|
const vectorIds = ids.length === vectors.length ? ids : vectors.map((_, i) => ids[i] || crypto.randomUUID());
|
|
2817
2498
|
const data = vectors.map((vector, i) => {
|
|
2818
2499
|
const id = String(vectorIds[i]);
|
|
@@ -2826,18 +2507,61 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2826
2507
|
Object.entries(flattenedMetadata).forEach(([key, value]) => {
|
|
2827
2508
|
rowData[key] = value;
|
|
2828
2509
|
});
|
|
2510
|
+
rowData["_metadata_json"] = JSON.stringify(metadataItem);
|
|
2511
|
+
} else {
|
|
2512
|
+
rowData["_metadata_json"] = "";
|
|
2829
2513
|
}
|
|
2830
2514
|
return rowData;
|
|
2831
2515
|
});
|
|
2832
|
-
|
|
2516
|
+
if (table !== null) {
|
|
2517
|
+
const rowCount = await table.countRows();
|
|
2518
|
+
const schema = await table.schema();
|
|
2519
|
+
const existingColumns = new Set(schema.fields.map((f) => f.name));
|
|
2520
|
+
const dataColumns = new Set(Object.keys(data[0] || {}));
|
|
2521
|
+
const extraColumns = [...dataColumns].filter((col) => !existingColumns.has(col));
|
|
2522
|
+
const missingSchemaColumns = [...existingColumns].filter((col) => !dataColumns.has(col));
|
|
2523
|
+
const hasSchemaMismatch = extraColumns.length > 0 || missingSchemaColumns.length > 0;
|
|
2524
|
+
if (rowCount === 0 && extraColumns.length > 0) {
|
|
2525
|
+
this.logger.warn(
|
|
2526
|
+
`Table ${resolvedTableName} is empty and data has extra columns ${extraColumns.join(", ")}. Recreating with new schema.`
|
|
2527
|
+
);
|
|
2528
|
+
await this.lanceClient.dropTable(resolvedTableName);
|
|
2529
|
+
await this.lanceClient.createTable(resolvedTableName, data);
|
|
2530
|
+
} else if (hasSchemaMismatch) {
|
|
2531
|
+
if (extraColumns.length > 0) {
|
|
2532
|
+
this.logger.warn(
|
|
2533
|
+
`Table ${resolvedTableName} has ${rowCount} rows. Columns ${extraColumns.join(", ")} will be dropped from upsert.`
|
|
2534
|
+
);
|
|
2535
|
+
}
|
|
2536
|
+
const schemaFieldNames = schema.fields.map((f) => f.name);
|
|
2537
|
+
const normalizedData = data.map((row) => {
|
|
2538
|
+
const normalized = {};
|
|
2539
|
+
for (const col of schemaFieldNames) {
|
|
2540
|
+
normalized[col] = col in row ? row[col] : null;
|
|
2541
|
+
}
|
|
2542
|
+
return normalized;
|
|
2543
|
+
});
|
|
2544
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(normalizedData);
|
|
2545
|
+
} else {
|
|
2546
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(data);
|
|
2547
|
+
}
|
|
2548
|
+
} else {
|
|
2549
|
+
this.logger.debug(`Creating table ${resolvedTableName} with initial data`);
|
|
2550
|
+
await this.lanceClient.createTable(resolvedTableName, data);
|
|
2551
|
+
}
|
|
2833
2552
|
return vectorIds;
|
|
2834
2553
|
} catch (error) {
|
|
2835
2554
|
throw new MastraError(
|
|
2836
2555
|
{
|
|
2837
|
-
id: "
|
|
2556
|
+
id: createVectorErrorId("LANCE", "UPSERT", "FAILED"),
|
|
2838
2557
|
domain: ErrorDomain.STORAGE,
|
|
2839
2558
|
category: ErrorCategory.THIRD_PARTY,
|
|
2840
|
-
details: {
|
|
2559
|
+
details: {
|
|
2560
|
+
tableName: resolvedTableName,
|
|
2561
|
+
vectorCount: vectors.length,
|
|
2562
|
+
metadataCount: metadata.length,
|
|
2563
|
+
idsCount: ids.length
|
|
2564
|
+
}
|
|
2841
2565
|
},
|
|
2842
2566
|
error
|
|
2843
2567
|
);
|
|
@@ -2861,7 +2585,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2861
2585
|
async createTable(tableName, data, options) {
|
|
2862
2586
|
if (!this.lanceClient) {
|
|
2863
2587
|
throw new MastraError({
|
|
2864
|
-
id: "
|
|
2588
|
+
id: createVectorErrorId("LANCE", "CREATE_TABLE", "INVALID_ARGS"),
|
|
2865
2589
|
domain: ErrorDomain.STORAGE,
|
|
2866
2590
|
category: ErrorCategory.USER,
|
|
2867
2591
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2876,7 +2600,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2876
2600
|
} catch (error) {
|
|
2877
2601
|
throw new MastraError(
|
|
2878
2602
|
{
|
|
2879
|
-
id: "
|
|
2603
|
+
id: createVectorErrorId("LANCE", "CREATE_TABLE", "FAILED"),
|
|
2880
2604
|
domain: ErrorDomain.STORAGE,
|
|
2881
2605
|
category: ErrorCategory.THIRD_PARTY,
|
|
2882
2606
|
details: { tableName }
|
|
@@ -2888,7 +2612,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2888
2612
|
async listTables() {
|
|
2889
2613
|
if (!this.lanceClient) {
|
|
2890
2614
|
throw new MastraError({
|
|
2891
|
-
id: "
|
|
2615
|
+
id: createVectorErrorId("LANCE", "LIST_TABLES", "INVALID_ARGS"),
|
|
2892
2616
|
domain: ErrorDomain.STORAGE,
|
|
2893
2617
|
category: ErrorCategory.USER,
|
|
2894
2618
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2900,7 +2624,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2900
2624
|
} catch (error) {
|
|
2901
2625
|
throw new MastraError(
|
|
2902
2626
|
{
|
|
2903
|
-
id: "
|
|
2627
|
+
id: createVectorErrorId("LANCE", "LIST_TABLES", "FAILED"),
|
|
2904
2628
|
domain: ErrorDomain.STORAGE,
|
|
2905
2629
|
category: ErrorCategory.THIRD_PARTY
|
|
2906
2630
|
},
|
|
@@ -2911,7 +2635,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2911
2635
|
async getTableSchema(tableName) {
|
|
2912
2636
|
if (!this.lanceClient) {
|
|
2913
2637
|
throw new MastraError({
|
|
2914
|
-
id: "
|
|
2638
|
+
id: createVectorErrorId("LANCE", "GET_TABLE_SCHEMA", "INVALID_ARGS"),
|
|
2915
2639
|
domain: ErrorDomain.STORAGE,
|
|
2916
2640
|
category: ErrorCategory.USER,
|
|
2917
2641
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2924,7 +2648,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2924
2648
|
} catch (error) {
|
|
2925
2649
|
throw new MastraError(
|
|
2926
2650
|
{
|
|
2927
|
-
id: "
|
|
2651
|
+
id: createVectorErrorId("LANCE", "GET_TABLE_SCHEMA", "FAILED"),
|
|
2928
2652
|
domain: ErrorDomain.STORAGE,
|
|
2929
2653
|
category: ErrorCategory.THIRD_PARTY,
|
|
2930
2654
|
details: { tableName }
|
|
@@ -2934,7 +2658,17 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2934
2658
|
}
|
|
2935
2659
|
}
|
|
2936
2660
|
/**
|
|
2937
|
-
*
|
|
2661
|
+
* Creates a vector index on a table.
|
|
2662
|
+
*
|
|
2663
|
+
* The behavior of `indexName` depends on whether `tableName` is provided:
|
|
2664
|
+
* - With `tableName`: `indexName` is the column to index (advanced use case)
|
|
2665
|
+
* - Without `tableName`: `indexName` becomes the table name, and 'vector' is used as the column (Memory compatibility)
|
|
2666
|
+
*
|
|
2667
|
+
* @param tableName - Optional table name. If not provided, defaults to indexName.
|
|
2668
|
+
* @param indexName - The index/column name, or table name if tableName is not provided.
|
|
2669
|
+
* @param dimension - Vector dimension size.
|
|
2670
|
+
* @param metric - Distance metric: 'cosine', 'euclidean', or 'dotproduct'.
|
|
2671
|
+
* @param indexConfig - Optional index configuration.
|
|
2938
2672
|
*/
|
|
2939
2673
|
async createIndex({
|
|
2940
2674
|
tableName,
|
|
@@ -2943,13 +2677,12 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2943
2677
|
metric = "cosine",
|
|
2944
2678
|
indexConfig = {}
|
|
2945
2679
|
}) {
|
|
2680
|
+
const resolvedTableName = tableName ?? indexName;
|
|
2681
|
+
const columnToIndex = tableName ? indexName : "vector";
|
|
2946
2682
|
try {
|
|
2947
2683
|
if (!this.lanceClient) {
|
|
2948
2684
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2949
2685
|
}
|
|
2950
|
-
if (!tableName) {
|
|
2951
|
-
throw new Error("tableName is required");
|
|
2952
|
-
}
|
|
2953
2686
|
if (!indexName) {
|
|
2954
2687
|
throw new Error("indexName is required");
|
|
2955
2688
|
}
|
|
@@ -2959,22 +2692,38 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2959
2692
|
} catch (err) {
|
|
2960
2693
|
throw new MastraError(
|
|
2961
2694
|
{
|
|
2962
|
-
id: "
|
|
2695
|
+
id: createVectorErrorId("LANCE", "CREATE_INDEX", "INVALID_ARGS"),
|
|
2963
2696
|
domain: ErrorDomain.STORAGE,
|
|
2964
2697
|
category: ErrorCategory.USER,
|
|
2965
|
-
details: { tableName:
|
|
2698
|
+
details: { tableName: resolvedTableName, indexName, dimension, metric }
|
|
2966
2699
|
},
|
|
2967
2700
|
err
|
|
2968
2701
|
);
|
|
2969
2702
|
}
|
|
2970
2703
|
try {
|
|
2971
2704
|
const tables = await this.lanceClient.tableNames();
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2705
|
+
let table;
|
|
2706
|
+
if (!tables.includes(resolvedTableName)) {
|
|
2707
|
+
this.logger.debug(
|
|
2708
|
+
`Table ${resolvedTableName} does not exist. Creating empty table with dimension ${dimension}.`
|
|
2975
2709
|
);
|
|
2710
|
+
const initVector = new Array(dimension).fill(0);
|
|
2711
|
+
table = await this.lanceClient.createTable(resolvedTableName, [
|
|
2712
|
+
{ id: "__init__", vector: initVector, _metadata_json: "" }
|
|
2713
|
+
]);
|
|
2714
|
+
try {
|
|
2715
|
+
await table.delete("id = '__init__'");
|
|
2716
|
+
} catch (deleteError) {
|
|
2717
|
+
this.logger.warn(
|
|
2718
|
+
`Failed to delete initialization row from ${resolvedTableName}. Subsequent queries may include '__init__' row.`,
|
|
2719
|
+
deleteError
|
|
2720
|
+
);
|
|
2721
|
+
}
|
|
2722
|
+
this.logger.debug(`Table ${resolvedTableName} created. Index creation deferred until data is available.`);
|
|
2723
|
+
return;
|
|
2724
|
+
} else {
|
|
2725
|
+
table = await this.lanceClient.openTable(resolvedTableName);
|
|
2976
2726
|
}
|
|
2977
|
-
const table = await this.lanceClient.openTable(tableName);
|
|
2978
2727
|
let metricType;
|
|
2979
2728
|
if (metric === "euclidean") {
|
|
2980
2729
|
metricType = "l2";
|
|
@@ -2983,8 +2732,15 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2983
2732
|
} else if (metric === "cosine") {
|
|
2984
2733
|
metricType = "cosine";
|
|
2985
2734
|
}
|
|
2735
|
+
const rowCount = await table.countRows();
|
|
2736
|
+
if (rowCount < 256) {
|
|
2737
|
+
this.logger.warn(
|
|
2738
|
+
`Table ${resolvedTableName} has ${rowCount} rows, which is below the 256 row minimum for index creation. Skipping index creation.`
|
|
2739
|
+
);
|
|
2740
|
+
return;
|
|
2741
|
+
}
|
|
2986
2742
|
if (indexConfig.type === "ivfflat") {
|
|
2987
|
-
await table.createIndex(
|
|
2743
|
+
await table.createIndex(columnToIndex, {
|
|
2988
2744
|
config: Index.ivfPq({
|
|
2989
2745
|
numPartitions: indexConfig.numPartitions || 128,
|
|
2990
2746
|
numSubVectors: indexConfig.numSubVectors || 16,
|
|
@@ -2993,7 +2749,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2993
2749
|
});
|
|
2994
2750
|
} else {
|
|
2995
2751
|
this.logger.debug("Creating HNSW PQ index with config:", indexConfig);
|
|
2996
|
-
await table.createIndex(
|
|
2752
|
+
await table.createIndex(columnToIndex, {
|
|
2997
2753
|
config: Index.hnswPq({
|
|
2998
2754
|
m: indexConfig?.hnsw?.m || 16,
|
|
2999
2755
|
efConstruction: indexConfig?.hnsw?.efConstruction || 100,
|
|
@@ -3004,10 +2760,10 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3004
2760
|
} catch (error) {
|
|
3005
2761
|
throw new MastraError(
|
|
3006
2762
|
{
|
|
3007
|
-
id: "
|
|
2763
|
+
id: createVectorErrorId("LANCE", "CREATE_INDEX", "FAILED"),
|
|
3008
2764
|
domain: ErrorDomain.STORAGE,
|
|
3009
2765
|
category: ErrorCategory.THIRD_PARTY,
|
|
3010
|
-
details: { tableName:
|
|
2766
|
+
details: { tableName: resolvedTableName, indexName, dimension }
|
|
3011
2767
|
},
|
|
3012
2768
|
error
|
|
3013
2769
|
);
|
|
@@ -3016,7 +2772,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3016
2772
|
async listIndexes() {
|
|
3017
2773
|
if (!this.lanceClient) {
|
|
3018
2774
|
throw new MastraError({
|
|
3019
|
-
id: "
|
|
2775
|
+
id: createVectorErrorId("LANCE", "LIST_INDEXES", "INVALID_ARGS"),
|
|
3020
2776
|
domain: ErrorDomain.STORAGE,
|
|
3021
2777
|
category: ErrorCategory.USER,
|
|
3022
2778
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -3035,7 +2791,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3035
2791
|
} catch (error) {
|
|
3036
2792
|
throw new MastraError(
|
|
3037
2793
|
{
|
|
3038
|
-
id: "
|
|
2794
|
+
id: createVectorErrorId("LANCE", "LIST_INDEXES", "FAILED"),
|
|
3039
2795
|
domain: ErrorDomain.STORAGE,
|
|
3040
2796
|
category: ErrorCategory.THIRD_PARTY
|
|
3041
2797
|
},
|
|
@@ -3054,7 +2810,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3054
2810
|
} catch (err) {
|
|
3055
2811
|
throw new MastraError(
|
|
3056
2812
|
{
|
|
3057
|
-
id: "
|
|
2813
|
+
id: createVectorErrorId("LANCE", "DESCRIBE_INDEX", "INVALID_ARGS"),
|
|
3058
2814
|
domain: ErrorDomain.STORAGE,
|
|
3059
2815
|
category: ErrorCategory.USER,
|
|
3060
2816
|
details: { indexName }
|
|
@@ -3089,7 +2845,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3089
2845
|
} catch (error) {
|
|
3090
2846
|
throw new MastraError(
|
|
3091
2847
|
{
|
|
3092
|
-
id: "
|
|
2848
|
+
id: createVectorErrorId("LANCE", "DESCRIBE_INDEX", "FAILED"),
|
|
3093
2849
|
domain: ErrorDomain.STORAGE,
|
|
3094
2850
|
category: ErrorCategory.THIRD_PARTY,
|
|
3095
2851
|
details: { indexName }
|
|
@@ -3109,7 +2865,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3109
2865
|
} catch (err) {
|
|
3110
2866
|
throw new MastraError(
|
|
3111
2867
|
{
|
|
3112
|
-
id: "
|
|
2868
|
+
id: createVectorErrorId("LANCE", "DELETE_INDEX", "INVALID_ARGS"),
|
|
3113
2869
|
domain: ErrorDomain.STORAGE,
|
|
3114
2870
|
category: ErrorCategory.USER,
|
|
3115
2871
|
details: { indexName }
|
|
@@ -3132,7 +2888,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3132
2888
|
} catch (error) {
|
|
3133
2889
|
throw new MastraError(
|
|
3134
2890
|
{
|
|
3135
|
-
id: "
|
|
2891
|
+
id: createVectorErrorId("LANCE", "DELETE_INDEX", "FAILED"),
|
|
3136
2892
|
domain: ErrorDomain.STORAGE,
|
|
3137
2893
|
category: ErrorCategory.THIRD_PARTY,
|
|
3138
2894
|
details: { indexName }
|
|
@@ -3147,7 +2903,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3147
2903
|
async deleteAllTables() {
|
|
3148
2904
|
if (!this.lanceClient) {
|
|
3149
2905
|
throw new MastraError({
|
|
3150
|
-
id: "
|
|
2906
|
+
id: createVectorErrorId("LANCE", "DELETE_ALL_TABLES", "INVALID_ARGS"),
|
|
3151
2907
|
domain: ErrorDomain.STORAGE,
|
|
3152
2908
|
category: ErrorCategory.USER,
|
|
3153
2909
|
details: { methodName: "deleteAllTables" },
|
|
@@ -3159,7 +2915,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3159
2915
|
} catch (error) {
|
|
3160
2916
|
throw new MastraError(
|
|
3161
2917
|
{
|
|
3162
|
-
id: "
|
|
2918
|
+
id: createVectorErrorId("LANCE", "DELETE_ALL_TABLES", "FAILED"),
|
|
3163
2919
|
domain: ErrorDomain.STORAGE,
|
|
3164
2920
|
category: ErrorCategory.THIRD_PARTY,
|
|
3165
2921
|
details: { methodName: "deleteAllTables" }
|
|
@@ -3171,7 +2927,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3171
2927
|
async deleteTable(tableName) {
|
|
3172
2928
|
if (!this.lanceClient) {
|
|
3173
2929
|
throw new MastraError({
|
|
3174
|
-
id: "
|
|
2930
|
+
id: createVectorErrorId("LANCE", "DELETE_TABLE", "INVALID_ARGS"),
|
|
3175
2931
|
domain: ErrorDomain.STORAGE,
|
|
3176
2932
|
category: ErrorCategory.USER,
|
|
3177
2933
|
details: { tableName },
|
|
@@ -3183,7 +2939,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3183
2939
|
} catch (error) {
|
|
3184
2940
|
throw new MastraError(
|
|
3185
2941
|
{
|
|
3186
|
-
id: "
|
|
2942
|
+
id: createVectorErrorId("LANCE", "DELETE_TABLE", "FAILED"),
|
|
3187
2943
|
domain: ErrorDomain.STORAGE,
|
|
3188
2944
|
category: ErrorCategory.THIRD_PARTY,
|
|
3189
2945
|
details: { tableName }
|
|
@@ -3192,7 +2948,44 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3192
2948
|
);
|
|
3193
2949
|
}
|
|
3194
2950
|
}
|
|
3195
|
-
async updateVector(
|
|
2951
|
+
async updateVector(params) {
|
|
2952
|
+
const { indexName, update } = params;
|
|
2953
|
+
if ("id" in params && "filter" in params && params.id && params.filter) {
|
|
2954
|
+
throw new MastraError({
|
|
2955
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "MUTUALLY_EXCLUSIVE"),
|
|
2956
|
+
domain: ErrorDomain.STORAGE,
|
|
2957
|
+
category: ErrorCategory.USER,
|
|
2958
|
+
text: "id and filter are mutually exclusive",
|
|
2959
|
+
details: { indexName }
|
|
2960
|
+
});
|
|
2961
|
+
}
|
|
2962
|
+
if (!("id" in params || "filter" in params) || !params.id && !params.filter) {
|
|
2963
|
+
throw new MastraError({
|
|
2964
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "NO_TARGET"),
|
|
2965
|
+
domain: ErrorDomain.STORAGE,
|
|
2966
|
+
category: ErrorCategory.USER,
|
|
2967
|
+
text: "Either id or filter must be provided",
|
|
2968
|
+
details: { indexName }
|
|
2969
|
+
});
|
|
2970
|
+
}
|
|
2971
|
+
if ("filter" in params && params.filter && Object.keys(params.filter).length === 0) {
|
|
2972
|
+
throw new MastraError({
|
|
2973
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "EMPTY_FILTER"),
|
|
2974
|
+
domain: ErrorDomain.STORAGE,
|
|
2975
|
+
category: ErrorCategory.USER,
|
|
2976
|
+
text: "Cannot update with empty filter",
|
|
2977
|
+
details: { indexName }
|
|
2978
|
+
});
|
|
2979
|
+
}
|
|
2980
|
+
if (!update.vector && !update.metadata) {
|
|
2981
|
+
throw new MastraError({
|
|
2982
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "NO_PAYLOAD"),
|
|
2983
|
+
domain: ErrorDomain.STORAGE,
|
|
2984
|
+
category: ErrorCategory.USER,
|
|
2985
|
+
text: "No updates provided",
|
|
2986
|
+
details: { indexName }
|
|
2987
|
+
});
|
|
2988
|
+
}
|
|
3196
2989
|
try {
|
|
3197
2990
|
if (!this.lanceClient) {
|
|
3198
2991
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
@@ -3200,21 +2993,6 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3200
2993
|
if (!indexName) {
|
|
3201
2994
|
throw new Error("indexName is required");
|
|
3202
2995
|
}
|
|
3203
|
-
if (!id) {
|
|
3204
|
-
throw new Error("id is required");
|
|
3205
|
-
}
|
|
3206
|
-
} catch (err) {
|
|
3207
|
-
throw new MastraError(
|
|
3208
|
-
{
|
|
3209
|
-
id: "STORAGE_LANCE_VECTOR_UPDATE_VECTOR_FAILED_INVALID_ARGS",
|
|
3210
|
-
domain: ErrorDomain.STORAGE,
|
|
3211
|
-
category: ErrorCategory.USER,
|
|
3212
|
-
details: { indexName, id }
|
|
3213
|
-
},
|
|
3214
|
-
err
|
|
3215
|
-
);
|
|
3216
|
-
}
|
|
3217
|
-
try {
|
|
3218
2996
|
const tables = await this.lanceClient.tableNames();
|
|
3219
2997
|
for (const tableName of tables) {
|
|
3220
2998
|
this.logger.debug("Checking table:" + tableName);
|
|
@@ -3224,39 +3002,77 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3224
3002
|
const hasColumn = schema.fields.some((field) => field.name === indexName);
|
|
3225
3003
|
if (hasColumn) {
|
|
3226
3004
|
this.logger.debug(`Found column ${indexName} in table ${tableName}`);
|
|
3227
|
-
|
|
3228
|
-
if (
|
|
3229
|
-
|
|
3005
|
+
let whereClause;
|
|
3006
|
+
if ("id" in params && params.id) {
|
|
3007
|
+
whereClause = `id = '${params.id}'`;
|
|
3008
|
+
} else if ("filter" in params && params.filter) {
|
|
3009
|
+
const translator = new LanceFilterTranslator();
|
|
3010
|
+
const processFilterKeys = (filter) => {
|
|
3011
|
+
const processedFilter = {};
|
|
3012
|
+
Object.entries(filter).forEach(([key, value]) => {
|
|
3013
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
3014
|
+
Object.entries(value).forEach(([nestedKey, nestedValue]) => {
|
|
3015
|
+
processedFilter[`metadata_${key}_${nestedKey}`] = nestedValue;
|
|
3016
|
+
});
|
|
3017
|
+
} else {
|
|
3018
|
+
processedFilter[`metadata_${key}`] = value;
|
|
3019
|
+
}
|
|
3020
|
+
});
|
|
3021
|
+
return processedFilter;
|
|
3022
|
+
};
|
|
3023
|
+
const prefixedFilter = processFilterKeys(params.filter);
|
|
3024
|
+
whereClause = translator.translate(prefixedFilter) || "";
|
|
3025
|
+
if (!whereClause) {
|
|
3026
|
+
throw new Error("Failed to translate filter to SQL");
|
|
3027
|
+
}
|
|
3028
|
+
} else {
|
|
3029
|
+
throw new Error("Either id or filter must be provided");
|
|
3230
3030
|
}
|
|
3231
|
-
const
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3031
|
+
const existingRecords = await table.query().where(whereClause).select(schema.fields.map((field) => field.name)).toArray();
|
|
3032
|
+
if (existingRecords.length === 0) {
|
|
3033
|
+
this.logger.info(`No records found matching criteria in table ${tableName}`);
|
|
3034
|
+
return;
|
|
3035
|
+
}
|
|
3036
|
+
const updatedRecords = existingRecords.map((record) => {
|
|
3037
|
+
const rowData = {};
|
|
3038
|
+
Object.entries(record).forEach(([key, value]) => {
|
|
3039
|
+
if (key !== "_distance") {
|
|
3040
|
+
if (key === indexName) {
|
|
3041
|
+
if (update.vector) {
|
|
3042
|
+
rowData[key] = update.vector;
|
|
3242
3043
|
} else {
|
|
3243
|
-
|
|
3044
|
+
if (Array.isArray(value)) {
|
|
3045
|
+
rowData[key] = [...value];
|
|
3046
|
+
} else if (typeof value === "object" && value !== null) {
|
|
3047
|
+
rowData[key] = Array.from(value);
|
|
3048
|
+
} else {
|
|
3049
|
+
rowData[key] = value;
|
|
3050
|
+
}
|
|
3051
|
+
}
|
|
3052
|
+
} else {
|
|
3053
|
+
rowData[key] = value;
|
|
3054
|
+
}
|
|
3055
|
+
}
|
|
3056
|
+
});
|
|
3057
|
+
if (update.metadata) {
|
|
3058
|
+
Object.entries(update.metadata).forEach(([key, value]) => {
|
|
3059
|
+
rowData[`metadata_${key}`] = value;
|
|
3060
|
+
});
|
|
3061
|
+
const hasMetadataJson = schema.fields.some((f) => f.name === "_metadata_json");
|
|
3062
|
+
if (hasMetadataJson) {
|
|
3063
|
+
let existingMetadata = {};
|
|
3064
|
+
if (record._metadata_json) {
|
|
3065
|
+
try {
|
|
3066
|
+
existingMetadata = JSON.parse(record._metadata_json);
|
|
3067
|
+
} catch {
|
|
3244
3068
|
}
|
|
3245
3069
|
}
|
|
3246
|
-
|
|
3247
|
-
rowData[key] = value;
|
|
3070
|
+
rowData["_metadata_json"] = JSON.stringify({ ...existingMetadata, ...update.metadata });
|
|
3248
3071
|
}
|
|
3249
3072
|
}
|
|
3073
|
+
return rowData;
|
|
3250
3074
|
});
|
|
3251
|
-
|
|
3252
|
-
rowData[indexName] = update.vector;
|
|
3253
|
-
}
|
|
3254
|
-
if (update.metadata) {
|
|
3255
|
-
Object.entries(update.metadata).forEach(([key, value]) => {
|
|
3256
|
-
rowData[`metadata_${key}`] = value;
|
|
3257
|
-
});
|
|
3258
|
-
}
|
|
3259
|
-
await table.add([rowData], { mode: "overwrite" });
|
|
3075
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(updatedRecords);
|
|
3260
3076
|
return;
|
|
3261
3077
|
}
|
|
3262
3078
|
} catch (err) {
|
|
@@ -3266,12 +3082,19 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3266
3082
|
}
|
|
3267
3083
|
throw new Error(`No table found with column/index '${indexName}'`);
|
|
3268
3084
|
} catch (error) {
|
|
3085
|
+
if (error instanceof MastraError) throw error;
|
|
3269
3086
|
throw new MastraError(
|
|
3270
3087
|
{
|
|
3271
|
-
id: "
|
|
3088
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "FAILED"),
|
|
3272
3089
|
domain: ErrorDomain.STORAGE,
|
|
3273
3090
|
category: ErrorCategory.THIRD_PARTY,
|
|
3274
|
-
details: {
|
|
3091
|
+
details: {
|
|
3092
|
+
indexName,
|
|
3093
|
+
..."id" in params && params.id && { id: params.id },
|
|
3094
|
+
..."filter" in params && params.filter && { filter: JSON.stringify(params.filter) },
|
|
3095
|
+
hasVector: !!update.vector,
|
|
3096
|
+
hasMetadata: !!update.metadata
|
|
3097
|
+
}
|
|
3275
3098
|
},
|
|
3276
3099
|
error
|
|
3277
3100
|
);
|
|
@@ -3291,10 +3114,13 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3291
3114
|
} catch (err) {
|
|
3292
3115
|
throw new MastraError(
|
|
3293
3116
|
{
|
|
3294
|
-
id: "
|
|
3117
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTOR", "INVALID_ARGS"),
|
|
3295
3118
|
domain: ErrorDomain.STORAGE,
|
|
3296
3119
|
category: ErrorCategory.USER,
|
|
3297
|
-
details: {
|
|
3120
|
+
details: {
|
|
3121
|
+
indexName,
|
|
3122
|
+
...id && { id }
|
|
3123
|
+
}
|
|
3298
3124
|
},
|
|
3299
3125
|
err
|
|
3300
3126
|
);
|
|
@@ -3321,42 +3147,146 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3321
3147
|
} catch (error) {
|
|
3322
3148
|
throw new MastraError(
|
|
3323
3149
|
{
|
|
3324
|
-
id: "
|
|
3150
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTOR", "FAILED"),
|
|
3325
3151
|
domain: ErrorDomain.STORAGE,
|
|
3326
3152
|
category: ErrorCategory.THIRD_PARTY,
|
|
3327
|
-
details: {
|
|
3153
|
+
details: {
|
|
3154
|
+
indexName,
|
|
3155
|
+
...id && { id }
|
|
3156
|
+
}
|
|
3328
3157
|
},
|
|
3329
3158
|
error
|
|
3330
3159
|
);
|
|
3331
3160
|
}
|
|
3332
3161
|
}
|
|
3333
3162
|
/**
|
|
3334
|
-
*
|
|
3335
|
-
*
|
|
3163
|
+
* Extracts column names referenced in a SQL WHERE clause.
|
|
3164
|
+
* Identifies metadata_* prefixed identifiers used in filter conditions.
|
|
3336
3165
|
*/
|
|
3337
|
-
|
|
3338
|
-
const
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
|
|
3166
|
+
extractFilterColumns(whereClause) {
|
|
3167
|
+
const matches = whereClause.match(/metadata_\w+/g);
|
|
3168
|
+
return matches ? [...new Set(matches)] : [];
|
|
3169
|
+
}
|
|
3170
|
+
/**
|
|
3171
|
+
* Extracts metadata from flattened column names (legacy data without _metadata_json).
|
|
3172
|
+
* Returns keys as-is after stripping the 'metadata_' prefix, without any unflattening.
|
|
3173
|
+
*/
|
|
3174
|
+
extractFlatMetadata(result) {
|
|
3175
|
+
const metadata = {};
|
|
3176
|
+
Object.keys(result).forEach((key) => {
|
|
3177
|
+
if (key !== "id" && key !== "score" && key !== "vector" && key !== "_distance" && key !== "_metadata_json") {
|
|
3178
|
+
if (key.startsWith("metadata_")) {
|
|
3179
|
+
metadata[key.substring("metadata_".length)] = result[key];
|
|
3348
3180
|
}
|
|
3349
|
-
current = current[part];
|
|
3350
|
-
}
|
|
3351
|
-
const lastPart = parts[parts.length - 1];
|
|
3352
|
-
if (lastPart) {
|
|
3353
|
-
current[lastPart] = value;
|
|
3354
3181
|
}
|
|
3355
3182
|
});
|
|
3356
|
-
return
|
|
3183
|
+
return metadata;
|
|
3184
|
+
}
|
|
3185
|
+
async deleteVectors({ indexName, filter, ids }) {
|
|
3186
|
+
if (ids && filter) {
|
|
3187
|
+
throw new MastraError({
|
|
3188
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "MUTUALLY_EXCLUSIVE"),
|
|
3189
|
+
domain: ErrorDomain.STORAGE,
|
|
3190
|
+
category: ErrorCategory.USER,
|
|
3191
|
+
text: "ids and filter are mutually exclusive",
|
|
3192
|
+
details: { indexName }
|
|
3193
|
+
});
|
|
3194
|
+
}
|
|
3195
|
+
if (!ids && !filter) {
|
|
3196
|
+
throw new MastraError({
|
|
3197
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "NO_TARGET"),
|
|
3198
|
+
domain: ErrorDomain.STORAGE,
|
|
3199
|
+
category: ErrorCategory.USER,
|
|
3200
|
+
text: "Either filter or ids must be provided",
|
|
3201
|
+
details: { indexName }
|
|
3202
|
+
});
|
|
3203
|
+
}
|
|
3204
|
+
if (ids && ids.length === 0) {
|
|
3205
|
+
throw new MastraError({
|
|
3206
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "EMPTY_IDS"),
|
|
3207
|
+
domain: ErrorDomain.STORAGE,
|
|
3208
|
+
category: ErrorCategory.USER,
|
|
3209
|
+
text: "Cannot delete with empty ids array",
|
|
3210
|
+
details: { indexName }
|
|
3211
|
+
});
|
|
3212
|
+
}
|
|
3213
|
+
if (filter && Object.keys(filter).length === 0) {
|
|
3214
|
+
throw new MastraError({
|
|
3215
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "EMPTY_FILTER"),
|
|
3216
|
+
domain: ErrorDomain.STORAGE,
|
|
3217
|
+
category: ErrorCategory.USER,
|
|
3218
|
+
text: "Cannot delete with empty filter",
|
|
3219
|
+
details: { indexName }
|
|
3220
|
+
});
|
|
3221
|
+
}
|
|
3222
|
+
try {
|
|
3223
|
+
if (!this.lanceClient) {
|
|
3224
|
+
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
3225
|
+
}
|
|
3226
|
+
if (!indexName) {
|
|
3227
|
+
throw new Error("indexName is required");
|
|
3228
|
+
}
|
|
3229
|
+
const tables = await this.lanceClient.tableNames();
|
|
3230
|
+
for (const tableName of tables) {
|
|
3231
|
+
this.logger.debug("Checking table:" + tableName);
|
|
3232
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
3233
|
+
try {
|
|
3234
|
+
const schema = await table.schema();
|
|
3235
|
+
const hasColumn = schema.fields.some((field) => field.name === indexName);
|
|
3236
|
+
if (hasColumn) {
|
|
3237
|
+
this.logger.debug(`Found column ${indexName} in table ${tableName}`);
|
|
3238
|
+
if (ids) {
|
|
3239
|
+
const idsConditions = ids.map((id) => `id = '${id}'`).join(" OR ");
|
|
3240
|
+
await table.delete(idsConditions);
|
|
3241
|
+
} else if (filter) {
|
|
3242
|
+
const translator = new LanceFilterTranslator();
|
|
3243
|
+
const processFilterKeys = (filter2) => {
|
|
3244
|
+
const processedFilter = {};
|
|
3245
|
+
Object.entries(filter2).forEach(([key, value]) => {
|
|
3246
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
3247
|
+
Object.entries(value).forEach(([nestedKey, nestedValue]) => {
|
|
3248
|
+
processedFilter[`metadata_${key}_${nestedKey}`] = nestedValue;
|
|
3249
|
+
});
|
|
3250
|
+
} else {
|
|
3251
|
+
processedFilter[`metadata_${key}`] = value;
|
|
3252
|
+
}
|
|
3253
|
+
});
|
|
3254
|
+
return processedFilter;
|
|
3255
|
+
};
|
|
3256
|
+
const prefixedFilter = processFilterKeys(filter);
|
|
3257
|
+
const whereClause = translator.translate(prefixedFilter);
|
|
3258
|
+
if (!whereClause) {
|
|
3259
|
+
throw new Error("Failed to translate filter to SQL");
|
|
3260
|
+
}
|
|
3261
|
+
await table.delete(whereClause);
|
|
3262
|
+
}
|
|
3263
|
+
return;
|
|
3264
|
+
}
|
|
3265
|
+
} catch (err) {
|
|
3266
|
+
this.logger.error(`Error checking schema for table ${tableName}:` + err);
|
|
3267
|
+
continue;
|
|
3268
|
+
}
|
|
3269
|
+
}
|
|
3270
|
+
throw new Error(`No table found with column/index '${indexName}'`);
|
|
3271
|
+
} catch (error) {
|
|
3272
|
+
if (error instanceof MastraError) throw error;
|
|
3273
|
+
throw new MastraError(
|
|
3274
|
+
{
|
|
3275
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "FAILED"),
|
|
3276
|
+
domain: ErrorDomain.STORAGE,
|
|
3277
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
3278
|
+
details: {
|
|
3279
|
+
indexName,
|
|
3280
|
+
...filter && { filter: JSON.stringify(filter) },
|
|
3281
|
+
...ids && { idsCount: ids.length }
|
|
3282
|
+
}
|
|
3283
|
+
},
|
|
3284
|
+
error
|
|
3285
|
+
);
|
|
3286
|
+
}
|
|
3357
3287
|
}
|
|
3358
3288
|
};
|
|
3359
3289
|
|
|
3360
|
-
export { LanceStorage, LanceVectorStore };
|
|
3290
|
+
export { LanceStorage, LanceVectorStore, StoreMemoryLance, StoreScoresLance, StoreWorkflowsLance };
|
|
3361
3291
|
//# sourceMappingURL=index.js.map
|
|
3362
3292
|
//# sourceMappingURL=index.js.map
|