@mastra/lance 0.0.0-remove-unused-model-providers-api-20251030210744 → 0.0.0-remove-ai-peer-dep-from-evals-20260105220639
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1114 -3
- package/README.md +61 -4
- package/dist/docs/README.md +33 -0
- package/dist/docs/SKILL.md +34 -0
- package/dist/docs/SOURCE_MAP.json +6 -0
- package/dist/docs/rag/01-vector-databases.md +638 -0
- package/dist/docs/storage/01-reference.md +113 -0
- package/dist/docs/vectors/01-reference.md +149 -0
- package/dist/index.cjs +1438 -1415
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1436 -1416
- package/dist/index.js.map +1 -1
- package/dist/storage/{domains/operations → db}/index.d.ts +21 -2
- package/dist/storage/db/index.d.ts.map +1 -0
- package/dist/storage/db/utils.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +19 -45
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/dist/storage/domains/scores/index.d.ts +24 -27
- package/dist/storage/domains/scores/index.d.ts.map +1 -1
- package/dist/storage/domains/workflows/index.d.ts +17 -15
- package/dist/storage/domains/workflows/index.d.ts.map +1 -1
- package/dist/storage/index.d.ts +101 -200
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/vector/filter.d.ts +5 -5
- package/dist/vector/index.d.ts +6 -3
- package/dist/vector/index.d.ts.map +1 -1
- package/package.json +17 -12
- package/dist/storage/domains/operations/index.d.ts.map +0 -1
- package/dist/storage/domains/utils.d.ts.map +0 -1
- /package/dist/storage/{domains → db}/utils.d.ts +0 -0
package/dist/index.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { connect, Index } from '@lancedb/lancedb';
|
|
2
2
|
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
|
-
import {
|
|
3
|
+
import { MemoryStorage, TABLE_SCHEMAS, TABLE_THREADS, TABLE_MESSAGES, TABLE_RESOURCES, createStorageErrorId, normalizePerPage, calculatePagination, ScoresStorage, SCORERS_SCHEMA, TABLE_SCORERS, WorkflowsStorage, TABLE_WORKFLOW_SNAPSHOT, MastraStorage, createVectorErrorId, getDefaultValue, ensureDate } from '@mastra/core/storage';
|
|
4
4
|
import { MessageList } from '@mastra/core/agent';
|
|
5
|
+
import { MastraBase } from '@mastra/core/base';
|
|
5
6
|
import { Utf8, Float64, Binary, Float32, Int32, Field, Schema } from 'apache-arrow';
|
|
6
|
-
import { saveScorePayloadSchema } from '@mastra/core/
|
|
7
|
+
import { saveScorePayloadSchema } from '@mastra/core/evals';
|
|
7
8
|
import { MastraVector } from '@mastra/core/vector';
|
|
8
9
|
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
9
10
|
|
|
@@ -89,7 +90,7 @@ async function getTableSchema({
|
|
|
89
90
|
} catch (validationError) {
|
|
90
91
|
throw new MastraError(
|
|
91
92
|
{
|
|
92
|
-
id: "
|
|
93
|
+
id: createStorageErrorId("LANCE", "GET_TABLE_SCHEMA", "INVALID_ARGS"),
|
|
93
94
|
domain: ErrorDomain.STORAGE,
|
|
94
95
|
category: ErrorCategory.USER,
|
|
95
96
|
text: validationError.message,
|
|
@@ -112,7 +113,7 @@ async function getTableSchema({
|
|
|
112
113
|
} catch (error) {
|
|
113
114
|
throw new MastraError(
|
|
114
115
|
{
|
|
115
|
-
id: "
|
|
116
|
+
id: createStorageErrorId("LANCE", "GET_TABLE_SCHEMA", "FAILED"),
|
|
116
117
|
domain: ErrorDomain.STORAGE,
|
|
117
118
|
category: ErrorCategory.THIRD_PARTY,
|
|
118
119
|
details: { tableName }
|
|
@@ -122,793 +123,503 @@ async function getTableSchema({
|
|
|
122
123
|
}
|
|
123
124
|
}
|
|
124
125
|
|
|
125
|
-
// src/storage/
|
|
126
|
-
|
|
126
|
+
// src/storage/db/index.ts
|
|
127
|
+
function resolveLanceConfig(config) {
|
|
128
|
+
return config.client;
|
|
129
|
+
}
|
|
130
|
+
var LanceDB = class extends MastraBase {
|
|
127
131
|
client;
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
super();
|
|
132
|
+
constructor({ client }) {
|
|
133
|
+
super({ name: "lance-db" });
|
|
131
134
|
this.client = client;
|
|
132
|
-
this.operations = operations;
|
|
133
135
|
}
|
|
134
|
-
|
|
136
|
+
getDefaultValue(type) {
|
|
137
|
+
switch (type) {
|
|
138
|
+
case "text":
|
|
139
|
+
return "''";
|
|
140
|
+
case "timestamp":
|
|
141
|
+
return "CURRENT_TIMESTAMP";
|
|
142
|
+
case "integer":
|
|
143
|
+
case "bigint":
|
|
144
|
+
return "0";
|
|
145
|
+
case "jsonb":
|
|
146
|
+
return "'{}'";
|
|
147
|
+
case "uuid":
|
|
148
|
+
return "''";
|
|
149
|
+
default:
|
|
150
|
+
return getDefaultValue(type);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
async hasColumn(tableName, columnName) {
|
|
154
|
+
const table = await this.client.openTable(tableName);
|
|
155
|
+
const schema = await table.schema();
|
|
156
|
+
return schema.fields.some((field) => field.name === columnName);
|
|
157
|
+
}
|
|
158
|
+
translateSchema(schema) {
|
|
159
|
+
const fields = Object.entries(schema).map(([name, column]) => {
|
|
160
|
+
let arrowType;
|
|
161
|
+
switch (column.type.toLowerCase()) {
|
|
162
|
+
case "text":
|
|
163
|
+
case "uuid":
|
|
164
|
+
arrowType = new Utf8();
|
|
165
|
+
break;
|
|
166
|
+
case "int":
|
|
167
|
+
case "integer":
|
|
168
|
+
arrowType = new Int32();
|
|
169
|
+
break;
|
|
170
|
+
case "bigint":
|
|
171
|
+
arrowType = new Float64();
|
|
172
|
+
break;
|
|
173
|
+
case "float":
|
|
174
|
+
arrowType = new Float32();
|
|
175
|
+
break;
|
|
176
|
+
case "jsonb":
|
|
177
|
+
case "json":
|
|
178
|
+
arrowType = new Utf8();
|
|
179
|
+
break;
|
|
180
|
+
case "binary":
|
|
181
|
+
arrowType = new Binary();
|
|
182
|
+
break;
|
|
183
|
+
case "timestamp":
|
|
184
|
+
arrowType = new Float64();
|
|
185
|
+
break;
|
|
186
|
+
default:
|
|
187
|
+
arrowType = new Utf8();
|
|
188
|
+
}
|
|
189
|
+
return new Field(name, arrowType, column.nullable ?? true);
|
|
190
|
+
});
|
|
191
|
+
return new Schema(fields);
|
|
192
|
+
}
|
|
193
|
+
async createTable({
|
|
194
|
+
tableName,
|
|
195
|
+
schema
|
|
196
|
+
}) {
|
|
135
197
|
try {
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
198
|
+
if (!this.client) {
|
|
199
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
200
|
+
}
|
|
201
|
+
if (!tableName) {
|
|
202
|
+
throw new Error("tableName is required for createTable.");
|
|
203
|
+
}
|
|
204
|
+
if (!schema) {
|
|
205
|
+
throw new Error("schema is required for createTable.");
|
|
139
206
|
}
|
|
140
|
-
return {
|
|
141
|
-
...thread,
|
|
142
|
-
createdAt: new Date(thread.createdAt),
|
|
143
|
-
updatedAt: new Date(thread.updatedAt)
|
|
144
|
-
};
|
|
145
207
|
} catch (error) {
|
|
146
208
|
throw new MastraError(
|
|
147
209
|
{
|
|
148
|
-
id: "
|
|
210
|
+
id: createStorageErrorId("LANCE", "CREATE_TABLE", "INVALID_ARGS"),
|
|
149
211
|
domain: ErrorDomain.STORAGE,
|
|
150
|
-
category: ErrorCategory.
|
|
212
|
+
category: ErrorCategory.USER,
|
|
213
|
+
details: { tableName }
|
|
151
214
|
},
|
|
152
215
|
error
|
|
153
216
|
);
|
|
154
217
|
}
|
|
155
|
-
}
|
|
156
|
-
async getThreadsByResourceId({ resourceId }) {
|
|
157
218
|
try {
|
|
158
|
-
const
|
|
159
|
-
|
|
160
|
-
const records = await query.toArray();
|
|
161
|
-
return processResultWithTypeConversion(
|
|
162
|
-
records,
|
|
163
|
-
await getTableSchema({ tableName: TABLE_THREADS, client: this.client })
|
|
164
|
-
);
|
|
219
|
+
const arrowSchema = this.translateSchema(schema);
|
|
220
|
+
await this.client.createEmptyTable(tableName, arrowSchema);
|
|
165
221
|
} catch (error) {
|
|
222
|
+
if (error.message?.includes("already exists")) {
|
|
223
|
+
this.logger.debug(`Table '${tableName}' already exists, skipping create`);
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
166
226
|
throw new MastraError(
|
|
167
227
|
{
|
|
168
|
-
id: "
|
|
228
|
+
id: createStorageErrorId("LANCE", "CREATE_TABLE", "FAILED"),
|
|
169
229
|
domain: ErrorDomain.STORAGE,
|
|
170
|
-
category: ErrorCategory.THIRD_PARTY
|
|
230
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
231
|
+
details: { tableName }
|
|
171
232
|
},
|
|
172
233
|
error
|
|
173
234
|
);
|
|
174
235
|
}
|
|
175
236
|
}
|
|
176
|
-
|
|
177
|
-
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
178
|
-
* @param thread - The thread to save
|
|
179
|
-
* @returns The saved thread
|
|
180
|
-
*/
|
|
181
|
-
async saveThread({ thread }) {
|
|
237
|
+
async dropTable({ tableName }) {
|
|
182
238
|
try {
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
239
|
+
if (!this.client) {
|
|
240
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
241
|
+
}
|
|
242
|
+
if (!tableName) {
|
|
243
|
+
throw new Error("tableName is required for dropTable.");
|
|
244
|
+
}
|
|
245
|
+
} catch (validationError) {
|
|
188
246
|
throw new MastraError(
|
|
189
247
|
{
|
|
190
|
-
id: "
|
|
248
|
+
id: createStorageErrorId("LANCE", "DROP_TABLE", "INVALID_ARGS"),
|
|
191
249
|
domain: ErrorDomain.STORAGE,
|
|
192
|
-
category: ErrorCategory.
|
|
250
|
+
category: ErrorCategory.USER,
|
|
251
|
+
text: validationError.message,
|
|
252
|
+
details: { tableName }
|
|
193
253
|
},
|
|
194
|
-
|
|
254
|
+
validationError
|
|
195
255
|
);
|
|
196
256
|
}
|
|
197
|
-
}
|
|
198
|
-
async updateThread({
|
|
199
|
-
id,
|
|
200
|
-
title,
|
|
201
|
-
metadata
|
|
202
|
-
}) {
|
|
203
|
-
const maxRetries = 5;
|
|
204
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
205
|
-
try {
|
|
206
|
-
const current = await this.getThreadById({ threadId: id });
|
|
207
|
-
if (!current) {
|
|
208
|
-
throw new Error(`Thread with id ${id} not found`);
|
|
209
|
-
}
|
|
210
|
-
const mergedMetadata = { ...current.metadata, ...metadata };
|
|
211
|
-
const record = {
|
|
212
|
-
id,
|
|
213
|
-
title,
|
|
214
|
-
metadata: JSON.stringify(mergedMetadata),
|
|
215
|
-
updatedAt: (/* @__PURE__ */ new Date()).getTime()
|
|
216
|
-
};
|
|
217
|
-
const table = await this.client.openTable(TABLE_THREADS);
|
|
218
|
-
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
219
|
-
const updatedThread = await this.getThreadById({ threadId: id });
|
|
220
|
-
if (!updatedThread) {
|
|
221
|
-
throw new Error(`Failed to retrieve updated thread ${id}`);
|
|
222
|
-
}
|
|
223
|
-
return updatedThread;
|
|
224
|
-
} catch (error) {
|
|
225
|
-
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
226
|
-
const delay = Math.pow(2, attempt) * 10;
|
|
227
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
228
|
-
continue;
|
|
229
|
-
}
|
|
230
|
-
throw new MastraError(
|
|
231
|
-
{
|
|
232
|
-
id: "LANCE_STORE_UPDATE_THREAD_FAILED",
|
|
233
|
-
domain: ErrorDomain.STORAGE,
|
|
234
|
-
category: ErrorCategory.THIRD_PARTY
|
|
235
|
-
},
|
|
236
|
-
error
|
|
237
|
-
);
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
throw new MastraError(
|
|
241
|
-
{
|
|
242
|
-
id: "LANCE_STORE_UPDATE_THREAD_FAILED",
|
|
243
|
-
domain: ErrorDomain.STORAGE,
|
|
244
|
-
category: ErrorCategory.THIRD_PARTY
|
|
245
|
-
},
|
|
246
|
-
new Error("All retries exhausted")
|
|
247
|
-
);
|
|
248
|
-
}
|
|
249
|
-
async deleteThread({ threadId }) {
|
|
250
257
|
try {
|
|
251
|
-
|
|
252
|
-
await table.delete(`id = '${threadId}'`);
|
|
253
|
-
const messagesTable = await this.client.openTable(TABLE_MESSAGES);
|
|
254
|
-
await messagesTable.delete(`thread_id = '${threadId}'`);
|
|
258
|
+
await this.client.dropTable(tableName);
|
|
255
259
|
} catch (error) {
|
|
260
|
+
if (error.toString().includes("was not found") || error.message?.includes("Table not found")) {
|
|
261
|
+
this.logger.debug(`Table '${tableName}' does not exist, skipping drop`);
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
256
264
|
throw new MastraError(
|
|
257
265
|
{
|
|
258
|
-
id: "
|
|
266
|
+
id: createStorageErrorId("LANCE", "DROP_TABLE", "FAILED"),
|
|
259
267
|
domain: ErrorDomain.STORAGE,
|
|
260
|
-
category: ErrorCategory.THIRD_PARTY
|
|
268
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
269
|
+
details: { tableName }
|
|
261
270
|
},
|
|
262
271
|
error
|
|
263
272
|
);
|
|
264
273
|
}
|
|
265
274
|
}
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
threadId: thread_id,
|
|
271
|
-
content: typeof message.content === "string" ? (() => {
|
|
272
|
-
try {
|
|
273
|
-
return JSON.parse(message.content);
|
|
274
|
-
} catch {
|
|
275
|
-
return message.content;
|
|
276
|
-
}
|
|
277
|
-
})() : message.content
|
|
278
|
-
};
|
|
279
|
-
}
|
|
280
|
-
async getMessages({
|
|
281
|
-
threadId,
|
|
282
|
-
resourceId,
|
|
283
|
-
selectBy,
|
|
284
|
-
format,
|
|
285
|
-
threadConfig
|
|
275
|
+
async alterTable({
|
|
276
|
+
tableName,
|
|
277
|
+
schema,
|
|
278
|
+
ifNotExists
|
|
286
279
|
}) {
|
|
287
280
|
try {
|
|
288
|
-
if (!
|
|
289
|
-
|
|
290
|
-
throw new Error("ThreadConfig is not supported by LanceDB storage");
|
|
281
|
+
if (!this.client) {
|
|
282
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
291
283
|
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
let allRecords = [];
|
|
295
|
-
if (selectBy?.include && selectBy.include.length > 0) {
|
|
296
|
-
const threadIds = [...new Set(selectBy.include.map((item) => item.threadId))];
|
|
297
|
-
for (const threadId2 of threadIds) {
|
|
298
|
-
const threadQuery = table.query().where(`thread_id = '${threadId2}'`);
|
|
299
|
-
let threadRecords = await threadQuery.toArray();
|
|
300
|
-
allRecords.push(...threadRecords);
|
|
301
|
-
}
|
|
302
|
-
} else {
|
|
303
|
-
let query = table.query().where(`\`thread_id\` = '${threadId}'`);
|
|
304
|
-
allRecords = await query.toArray();
|
|
284
|
+
if (!tableName) {
|
|
285
|
+
throw new Error("tableName is required for alterTable.");
|
|
305
286
|
}
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
const dateB = new Date(b.createdAt).getTime();
|
|
309
|
-
return dateA - dateB;
|
|
310
|
-
});
|
|
311
|
-
if (selectBy?.include && selectBy.include.length > 0) {
|
|
312
|
-
allRecords = this.processMessagesWithContext(allRecords, selectBy.include);
|
|
287
|
+
if (!schema) {
|
|
288
|
+
throw new Error("schema is required for alterTable.");
|
|
313
289
|
}
|
|
314
|
-
if (
|
|
315
|
-
|
|
290
|
+
if (!ifNotExists || ifNotExists.length === 0) {
|
|
291
|
+
this.logger.debug("No columns specified to add in alterTable, skipping.");
|
|
292
|
+
return;
|
|
316
293
|
}
|
|
317
|
-
|
|
318
|
-
allRecords,
|
|
319
|
-
await getTableSchema({ tableName: TABLE_MESSAGES, client: this.client })
|
|
320
|
-
);
|
|
321
|
-
const list = new MessageList({ threadId, resourceId }).add(messages.map(this.normalizeMessage), "memory");
|
|
322
|
-
if (format === "v2") return list.get.all.v2();
|
|
323
|
-
return list.get.all.v1();
|
|
324
|
-
} catch (error) {
|
|
294
|
+
} catch (validationError) {
|
|
325
295
|
throw new MastraError(
|
|
326
296
|
{
|
|
327
|
-
id: "
|
|
297
|
+
id: createStorageErrorId("LANCE", "ALTER_TABLE", "INVALID_ARGS"),
|
|
328
298
|
domain: ErrorDomain.STORAGE,
|
|
329
|
-
category: ErrorCategory.
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
resourceId: resourceId ?? ""
|
|
333
|
-
}
|
|
299
|
+
category: ErrorCategory.USER,
|
|
300
|
+
text: validationError.message,
|
|
301
|
+
details: { tableName }
|
|
334
302
|
},
|
|
335
|
-
|
|
303
|
+
validationError
|
|
336
304
|
);
|
|
337
305
|
}
|
|
338
|
-
}
|
|
339
|
-
async listMessagesById({ messageIds }) {
|
|
340
|
-
if (messageIds.length === 0) return [];
|
|
341
306
|
try {
|
|
342
|
-
const table = await this.client.openTable(
|
|
343
|
-
const
|
|
344
|
-
const
|
|
345
|
-
const
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
async listMessages(args) {
|
|
366
|
-
const { threadId, resourceId, include, filter, limit, offset = 0, orderBy } = args;
|
|
367
|
-
if (!threadId.trim()) {
|
|
307
|
+
const table = await this.client.openTable(tableName);
|
|
308
|
+
const currentSchema = await table.schema();
|
|
309
|
+
const existingFields = new Set(currentSchema.fields.map((f) => f.name));
|
|
310
|
+
const typeMap = {
|
|
311
|
+
text: "string",
|
|
312
|
+
integer: "int",
|
|
313
|
+
bigint: "bigint",
|
|
314
|
+
timestamp: "timestamp",
|
|
315
|
+
jsonb: "string",
|
|
316
|
+
uuid: "string"
|
|
317
|
+
};
|
|
318
|
+
const columnsToAdd = ifNotExists.filter((col) => schema[col] && !existingFields.has(col)).map((col) => {
|
|
319
|
+
const colDef = schema[col];
|
|
320
|
+
return {
|
|
321
|
+
name: col,
|
|
322
|
+
valueSql: colDef?.nullable ? `cast(NULL as ${typeMap[colDef.type ?? "text"]})` : `cast(${this.getDefaultValue(colDef?.type ?? "text")} as ${typeMap[colDef?.type ?? "text"]})`
|
|
323
|
+
};
|
|
324
|
+
});
|
|
325
|
+
if (columnsToAdd.length > 0) {
|
|
326
|
+
await table.addColumns(columnsToAdd);
|
|
327
|
+
this.logger?.info?.(`Added columns [${columnsToAdd.map((c) => c.name).join(", ")}] to table ${tableName}`);
|
|
328
|
+
}
|
|
329
|
+
} catch (error) {
|
|
368
330
|
throw new MastraError(
|
|
369
331
|
{
|
|
370
|
-
id: "
|
|
332
|
+
id: createStorageErrorId("LANCE", "ALTER_TABLE", "FAILED"),
|
|
371
333
|
domain: ErrorDomain.STORAGE,
|
|
372
334
|
category: ErrorCategory.THIRD_PARTY,
|
|
373
|
-
details: {
|
|
335
|
+
details: { tableName }
|
|
374
336
|
},
|
|
375
|
-
|
|
337
|
+
error
|
|
376
338
|
);
|
|
377
339
|
}
|
|
340
|
+
}
|
|
341
|
+
async clearTable({ tableName }) {
|
|
378
342
|
try {
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
if (limit === false) {
|
|
382
|
-
perPage = Number.MAX_SAFE_INTEGER;
|
|
383
|
-
} else if (limit === 0) {
|
|
384
|
-
perPage = 0;
|
|
385
|
-
} else if (typeof limit === "number" && limit > 0) {
|
|
386
|
-
perPage = limit;
|
|
387
|
-
}
|
|
343
|
+
if (!this.client) {
|
|
344
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
388
345
|
}
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
const sortDirection = orderBy?.direction || "DESC";
|
|
392
|
-
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
393
|
-
const escapeSql = (str) => str.replace(/'/g, "''");
|
|
394
|
-
const conditions = [`thread_id = '${escapeSql(threadId)}'`];
|
|
395
|
-
if (resourceId) {
|
|
396
|
-
conditions.push(`\`resourceId\` = '${escapeSql(resourceId)}'`);
|
|
346
|
+
if (!tableName) {
|
|
347
|
+
throw new Error("tableName is required for clearTable.");
|
|
397
348
|
}
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
349
|
+
} catch (validationError) {
|
|
350
|
+
throw new MastraError(
|
|
351
|
+
{
|
|
352
|
+
id: createStorageErrorId("LANCE", "CLEAR_TABLE", "INVALID_ARGS"),
|
|
353
|
+
domain: ErrorDomain.STORAGE,
|
|
354
|
+
category: ErrorCategory.USER,
|
|
355
|
+
text: validationError.message,
|
|
356
|
+
details: { tableName }
|
|
357
|
+
},
|
|
358
|
+
validationError
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
try {
|
|
362
|
+
const table = await this.client.openTable(tableName);
|
|
363
|
+
await table.delete("1=1");
|
|
364
|
+
} catch (error) {
|
|
365
|
+
throw new MastraError(
|
|
366
|
+
{
|
|
367
|
+
id: createStorageErrorId("LANCE", "CLEAR_TABLE", "FAILED"),
|
|
368
|
+
domain: ErrorDomain.STORAGE,
|
|
369
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
370
|
+
details: { tableName }
|
|
371
|
+
},
|
|
372
|
+
error
|
|
373
|
+
);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
async insert({ tableName, record }) {
|
|
377
|
+
try {
|
|
378
|
+
if (!this.client) {
|
|
379
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
401
380
|
}
|
|
402
|
-
if (
|
|
403
|
-
|
|
404
|
-
conditions.push(`\`createdAt\` <= ${endTime}`);
|
|
381
|
+
if (!tableName) {
|
|
382
|
+
throw new Error("tableName is required for insert.");
|
|
405
383
|
}
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
const query = table.query().where(whereClause);
|
|
409
|
-
let allRecords = await query.toArray();
|
|
410
|
-
allRecords.sort((a, b) => {
|
|
411
|
-
const aValue = sortField === "createdAt" ? a.createdAt : a[sortField];
|
|
412
|
-
const bValue = sortField === "createdAt" ? b.createdAt : b[sortField];
|
|
413
|
-
return sortDirection === "ASC" ? aValue - bValue : bValue - aValue;
|
|
414
|
-
});
|
|
415
|
-
const paginatedRecords = allRecords.slice(offset, offset + perPage);
|
|
416
|
-
const messages = paginatedRecords.map((row) => this.normalizeMessage(row));
|
|
417
|
-
if (total === 0 && messages.length === 0) {
|
|
418
|
-
return {
|
|
419
|
-
messages: [],
|
|
420
|
-
total: 0,
|
|
421
|
-
page,
|
|
422
|
-
perPage,
|
|
423
|
-
hasMore: false
|
|
424
|
-
};
|
|
384
|
+
if (!record || Object.keys(record).length === 0) {
|
|
385
|
+
throw new Error("record is required and cannot be empty for insert.");
|
|
425
386
|
}
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
}
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
387
|
+
} catch (validationError) {
|
|
388
|
+
throw new MastraError(
|
|
389
|
+
{
|
|
390
|
+
id: createStorageErrorId("LANCE", "INSERT", "INVALID_ARGS"),
|
|
391
|
+
domain: ErrorDomain.STORAGE,
|
|
392
|
+
category: ErrorCategory.USER,
|
|
393
|
+
text: validationError.message,
|
|
394
|
+
details: { tableName }
|
|
395
|
+
},
|
|
396
|
+
validationError
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
try {
|
|
400
|
+
const table = await this.client.openTable(tableName);
|
|
401
|
+
const primaryId = getPrimaryKeys(tableName);
|
|
402
|
+
const processedRecord = { ...record };
|
|
403
|
+
for (const key in processedRecord) {
|
|
404
|
+
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
405
|
+
this.logger.debug("Converting object to JSON string: ", processedRecord[key]);
|
|
406
|
+
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
443
407
|
}
|
|
444
408
|
}
|
|
445
|
-
|
|
446
|
-
let finalMessages = list.get.all.v2();
|
|
447
|
-
finalMessages = finalMessages.sort((a, b) => {
|
|
448
|
-
const aValue = sortField === "createdAt" ? new Date(a.createdAt).getTime() : a[sortField];
|
|
449
|
-
const bValue = sortField === "createdAt" ? new Date(b.createdAt).getTime() : b[sortField];
|
|
450
|
-
return sortDirection === "ASC" ? aValue - bValue : bValue - aValue;
|
|
451
|
-
});
|
|
452
|
-
const returnedThreadMessageIds = new Set(finalMessages.filter((m) => m.threadId === threadId).map((m) => m.id));
|
|
453
|
-
const allThreadMessagesReturned = returnedThreadMessageIds.size >= total;
|
|
454
|
-
const hasMore = limit === false ? false : allThreadMessagesReturned ? false : offset + paginatedRecords.length < total;
|
|
455
|
-
return {
|
|
456
|
-
messages: finalMessages,
|
|
457
|
-
total,
|
|
458
|
-
page,
|
|
459
|
-
perPage,
|
|
460
|
-
hasMore
|
|
461
|
-
};
|
|
409
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([processedRecord]);
|
|
462
410
|
} catch (error) {
|
|
463
|
-
|
|
464
|
-
const mastraError = new MastraError(
|
|
411
|
+
throw new MastraError(
|
|
465
412
|
{
|
|
466
|
-
id: "
|
|
413
|
+
id: createStorageErrorId("LANCE", "INSERT", "FAILED"),
|
|
467
414
|
domain: ErrorDomain.STORAGE,
|
|
468
415
|
category: ErrorCategory.THIRD_PARTY,
|
|
469
|
-
details: {
|
|
470
|
-
threadId,
|
|
471
|
-
resourceId: resourceId ?? ""
|
|
472
|
-
}
|
|
416
|
+
details: { tableName }
|
|
473
417
|
},
|
|
474
418
|
error
|
|
475
419
|
);
|
|
476
|
-
this.logger?.error?.(mastraError.toString());
|
|
477
|
-
this.logger?.trackException?.(mastraError);
|
|
478
|
-
return {
|
|
479
|
-
messages: [],
|
|
480
|
-
total: 0,
|
|
481
|
-
page: errorPerPage === 0 ? 0 : Math.floor(offset / errorPerPage),
|
|
482
|
-
perPage: errorPerPage,
|
|
483
|
-
hasMore: false
|
|
484
|
-
};
|
|
485
420
|
}
|
|
486
421
|
}
|
|
487
|
-
|
|
488
|
-
* @todo When migrating from getThreadsByResourceIdPaginated to this method,
|
|
489
|
-
* implement orderBy and sortDirection support for full sorting capabilities
|
|
490
|
-
*/
|
|
491
|
-
async listThreadsByResourceId(args) {
|
|
492
|
-
const { resourceId, limit, offset } = args;
|
|
493
|
-
const page = Math.floor(offset / limit);
|
|
494
|
-
const perPage = limit;
|
|
495
|
-
return this.getThreadsByResourceIdPaginated({ resourceId, page, perPage });
|
|
496
|
-
}
|
|
497
|
-
async saveMessages(args) {
|
|
422
|
+
async batchInsert({ tableName, records }) {
|
|
498
423
|
try {
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
return [];
|
|
424
|
+
if (!this.client) {
|
|
425
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
502
426
|
}
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
throw new Error("Thread ID is required");
|
|
427
|
+
if (!tableName) {
|
|
428
|
+
throw new Error("tableName is required for batchInsert.");
|
|
506
429
|
}
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
throw new Error("Message ID is required");
|
|
510
|
-
}
|
|
511
|
-
if (!message.threadId) {
|
|
512
|
-
throw new Error("Thread ID is required for all messages");
|
|
513
|
-
}
|
|
514
|
-
if (message.resourceId === null || message.resourceId === void 0) {
|
|
515
|
-
throw new Error("Resource ID cannot be null or undefined");
|
|
516
|
-
}
|
|
517
|
-
if (!message.content) {
|
|
518
|
-
throw new Error("Message content is required");
|
|
519
|
-
}
|
|
430
|
+
if (!records || records.length === 0) {
|
|
431
|
+
throw new Error("records array is required and cannot be empty for batchInsert.");
|
|
520
432
|
}
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
433
|
+
} catch (validationError) {
|
|
434
|
+
throw new MastraError(
|
|
435
|
+
{
|
|
436
|
+
id: createStorageErrorId("LANCE", "BATCH_INSERT", "INVALID_ARGS"),
|
|
437
|
+
domain: ErrorDomain.STORAGE,
|
|
438
|
+
category: ErrorCategory.USER,
|
|
439
|
+
text: validationError.message,
|
|
440
|
+
details: { tableName }
|
|
441
|
+
},
|
|
442
|
+
validationError
|
|
443
|
+
);
|
|
444
|
+
}
|
|
445
|
+
try {
|
|
446
|
+
const table = await this.client.openTable(tableName);
|
|
447
|
+
const primaryId = getPrimaryKeys(tableName);
|
|
448
|
+
const processedRecords = records.map((record) => {
|
|
449
|
+
const processedRecord = { ...record };
|
|
450
|
+
for (const key in processedRecord) {
|
|
451
|
+
if (processedRecord[key] == null) continue;
|
|
452
|
+
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
453
|
+
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
return processedRecord;
|
|
529
457
|
});
|
|
530
|
-
|
|
531
|
-
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(transformedMessages);
|
|
532
|
-
const threadsTable = await this.client.openTable(TABLE_THREADS);
|
|
533
|
-
const currentTime = (/* @__PURE__ */ new Date()).getTime();
|
|
534
|
-
const updateRecord = { id: threadId, updatedAt: currentTime };
|
|
535
|
-
await threadsTable.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([updateRecord]);
|
|
536
|
-
const list = new MessageList().add(messages, "memory");
|
|
537
|
-
if (format === `v2`) return list.get.all.v2();
|
|
538
|
-
return list.get.all.v1();
|
|
458
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(processedRecords);
|
|
539
459
|
} catch (error) {
|
|
540
460
|
throw new MastraError(
|
|
541
461
|
{
|
|
542
|
-
id: "
|
|
462
|
+
id: createStorageErrorId("LANCE", "BATCH_INSERT", "FAILED"),
|
|
543
463
|
domain: ErrorDomain.STORAGE,
|
|
544
|
-
category: ErrorCategory.THIRD_PARTY
|
|
464
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
465
|
+
details: { tableName }
|
|
545
466
|
},
|
|
546
467
|
error
|
|
547
468
|
);
|
|
548
469
|
}
|
|
549
470
|
}
|
|
550
|
-
async
|
|
471
|
+
async load({ tableName, keys }) {
|
|
551
472
|
try {
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
const total = await table.countRows(`\`resourceId\` = '${resourceId}'`);
|
|
555
|
-
const query = table.query().where(`\`resourceId\` = '${resourceId}'`);
|
|
556
|
-
const offset = page * perPage;
|
|
557
|
-
query.limit(perPage);
|
|
558
|
-
if (offset > 0) {
|
|
559
|
-
query.offset(offset);
|
|
473
|
+
if (!this.client) {
|
|
474
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
560
475
|
}
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
page,
|
|
569
|
-
perPage,
|
|
570
|
-
hasMore: total > (page + 1) * perPage
|
|
571
|
-
};
|
|
572
|
-
} catch (error) {
|
|
476
|
+
if (!tableName) {
|
|
477
|
+
throw new Error("tableName is required for load.");
|
|
478
|
+
}
|
|
479
|
+
if (!keys || Object.keys(keys).length === 0) {
|
|
480
|
+
throw new Error("keys are required and cannot be empty for load.");
|
|
481
|
+
}
|
|
482
|
+
} catch (validationError) {
|
|
573
483
|
throw new MastraError(
|
|
574
484
|
{
|
|
575
|
-
id: "
|
|
485
|
+
id: createStorageErrorId("LANCE", "LOAD", "INVALID_ARGS"),
|
|
576
486
|
domain: ErrorDomain.STORAGE,
|
|
577
|
-
category: ErrorCategory.
|
|
487
|
+
category: ErrorCategory.USER,
|
|
488
|
+
text: validationError.message,
|
|
489
|
+
details: { tableName }
|
|
578
490
|
},
|
|
579
|
-
|
|
491
|
+
validationError
|
|
580
492
|
);
|
|
581
493
|
}
|
|
582
|
-
}
|
|
583
|
-
/**
|
|
584
|
-
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
585
|
-
* @param records - The sorted array of records to process
|
|
586
|
-
* @param include - The array of include specifications with context parameters
|
|
587
|
-
* @returns The processed array with context messages included
|
|
588
|
-
*/
|
|
589
|
-
processMessagesWithContext(records, include) {
|
|
590
|
-
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
591
|
-
if (messagesWithContext.length === 0) {
|
|
592
|
-
return records;
|
|
593
|
-
}
|
|
594
|
-
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
595
|
-
records.forEach((message, index) => {
|
|
596
|
-
messageIndexMap.set(message.id, index);
|
|
597
|
-
});
|
|
598
|
-
const additionalIndices = /* @__PURE__ */ new Set();
|
|
599
|
-
for (const item of messagesWithContext) {
|
|
600
|
-
const messageIndex = messageIndexMap.get(item.id);
|
|
601
|
-
if (messageIndex !== void 0) {
|
|
602
|
-
if (item.withPreviousMessages) {
|
|
603
|
-
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
604
|
-
for (let i = startIdx; i < messageIndex; i++) {
|
|
605
|
-
additionalIndices.add(i);
|
|
606
|
-
}
|
|
607
|
-
}
|
|
608
|
-
if (item.withNextMessages) {
|
|
609
|
-
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
610
|
-
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
611
|
-
additionalIndices.add(i);
|
|
612
|
-
}
|
|
613
|
-
}
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
if (additionalIndices.size === 0) {
|
|
617
|
-
return records;
|
|
618
|
-
}
|
|
619
|
-
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
620
|
-
const allIndices = /* @__PURE__ */ new Set();
|
|
621
|
-
records.forEach((record, index) => {
|
|
622
|
-
if (originalMatchIds.has(record.id)) {
|
|
623
|
-
allIndices.add(index);
|
|
624
|
-
}
|
|
625
|
-
});
|
|
626
|
-
additionalIndices.forEach((index) => {
|
|
627
|
-
allIndices.add(index);
|
|
628
|
-
});
|
|
629
|
-
return Array.from(allIndices).sort((a, b) => a - b).map((index) => records[index]);
|
|
630
|
-
}
|
|
631
|
-
async getMessagesPaginated(args) {
|
|
632
|
-
const { threadId, resourceId, selectBy, format = "v1" } = args;
|
|
633
|
-
const page = selectBy?.pagination?.page ?? 0;
|
|
634
|
-
const perPage = selectBy?.pagination?.perPage ?? 10;
|
|
635
494
|
try {
|
|
636
|
-
|
|
637
|
-
const
|
|
638
|
-
const
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
messages.push(...contextMessages);
|
|
655
|
-
}
|
|
656
|
-
const conditions = [`thread_id = '${threadId}'`];
|
|
657
|
-
if (resourceId) {
|
|
658
|
-
conditions.push(`\`resourceId\` = '${resourceId}'`);
|
|
659
|
-
}
|
|
660
|
-
if (fromDate) {
|
|
661
|
-
conditions.push(`\`createdAt\` >= ${fromDate.getTime()}`);
|
|
662
|
-
}
|
|
663
|
-
if (toDate) {
|
|
664
|
-
conditions.push(`\`createdAt\` <= ${toDate.getTime()}`);
|
|
665
|
-
}
|
|
666
|
-
let total = 0;
|
|
667
|
-
if (conditions.length > 0) {
|
|
668
|
-
total = await table.countRows(conditions.join(" AND "));
|
|
669
|
-
} else {
|
|
670
|
-
total = await table.countRows();
|
|
495
|
+
const table = await this.client.openTable(tableName);
|
|
496
|
+
const tableSchema = await getTableSchema({ tableName, client: this.client });
|
|
497
|
+
const query = table.query();
|
|
498
|
+
if (Object.keys(keys).length > 0) {
|
|
499
|
+
validateKeyTypes(keys, tableSchema);
|
|
500
|
+
const filterConditions = Object.entries(keys).map(([key, value]) => {
|
|
501
|
+
const isCamelCase = /^[a-z][a-zA-Z]*$/.test(key) && /[A-Z]/.test(key);
|
|
502
|
+
const quotedKey = isCamelCase ? `\`${key}\`` : key;
|
|
503
|
+
if (typeof value === "string") {
|
|
504
|
+
return `${quotedKey} = '${value}'`;
|
|
505
|
+
} else if (value === null) {
|
|
506
|
+
return `${quotedKey} IS NULL`;
|
|
507
|
+
} else {
|
|
508
|
+
return `${quotedKey} = ${value}`;
|
|
509
|
+
}
|
|
510
|
+
}).join(" AND ");
|
|
511
|
+
this.logger.debug("where clause generated: " + filterConditions);
|
|
512
|
+
query.where(filterConditions);
|
|
671
513
|
}
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
page,
|
|
677
|
-
perPage,
|
|
678
|
-
hasMore: false
|
|
679
|
-
};
|
|
514
|
+
const result = await query.limit(1).toArray();
|
|
515
|
+
if (result.length === 0) {
|
|
516
|
+
this.logger.debug("No record found");
|
|
517
|
+
return null;
|
|
680
518
|
}
|
|
681
|
-
|
|
682
|
-
let selectedMessages = [];
|
|
683
|
-
if (selectBy?.last && selectBy.last > 0) {
|
|
684
|
-
const query = table.query();
|
|
685
|
-
if (conditions.length > 0) {
|
|
686
|
-
query.where(conditions.join(" AND "));
|
|
687
|
-
}
|
|
688
|
-
let records = await query.toArray();
|
|
689
|
-
records = records.sort((a, b) => a.createdAt - b.createdAt);
|
|
690
|
-
if (excludeIds.length > 0) {
|
|
691
|
-
records = records.filter((m) => !excludeIds.includes(m.id));
|
|
692
|
-
}
|
|
693
|
-
selectedMessages = records.slice(-selectBy.last);
|
|
694
|
-
} else {
|
|
695
|
-
const query = table.query();
|
|
696
|
-
if (conditions.length > 0) {
|
|
697
|
-
query.where(conditions.join(" AND "));
|
|
698
|
-
}
|
|
699
|
-
let records = await query.toArray();
|
|
700
|
-
records = records.sort((a, b) => a.createdAt - b.createdAt);
|
|
701
|
-
if (excludeIds.length > 0) {
|
|
702
|
-
records = records.filter((m) => !excludeIds.includes(m.id));
|
|
703
|
-
}
|
|
704
|
-
selectedMessages = records.slice(page * perPage, (page + 1) * perPage);
|
|
705
|
-
}
|
|
706
|
-
const allMessages = [...messages, ...selectedMessages];
|
|
707
|
-
const seen = /* @__PURE__ */ new Set();
|
|
708
|
-
const dedupedMessages = allMessages.filter((m) => {
|
|
709
|
-
const key = `${m.id}:${m.thread_id}`;
|
|
710
|
-
if (seen.has(key)) return false;
|
|
711
|
-
seen.add(key);
|
|
712
|
-
return true;
|
|
713
|
-
});
|
|
714
|
-
const formattedMessages = dedupedMessages.map((msg) => {
|
|
715
|
-
const { thread_id, ...rest } = msg;
|
|
716
|
-
return {
|
|
717
|
-
...rest,
|
|
718
|
-
threadId: thread_id,
|
|
719
|
-
content: typeof msg.content === "string" ? (() => {
|
|
720
|
-
try {
|
|
721
|
-
return JSON.parse(msg.content);
|
|
722
|
-
} catch {
|
|
723
|
-
return msg.content;
|
|
724
|
-
}
|
|
725
|
-
})() : msg.content
|
|
726
|
-
};
|
|
727
|
-
});
|
|
728
|
-
const list = new MessageList().add(formattedMessages, "memory");
|
|
729
|
-
return {
|
|
730
|
-
messages: format === "v2" ? list.get.all.v2() : list.get.all.v1(),
|
|
731
|
-
total,
|
|
732
|
-
// Total should be the count of messages matching the filters
|
|
733
|
-
page,
|
|
734
|
-
perPage,
|
|
735
|
-
hasMore: total > (page + 1) * perPage
|
|
736
|
-
};
|
|
519
|
+
return processResultWithTypeConversion(result[0], tableSchema);
|
|
737
520
|
} catch (error) {
|
|
738
|
-
|
|
521
|
+
if (error instanceof MastraError) throw error;
|
|
522
|
+
throw new MastraError(
|
|
739
523
|
{
|
|
740
|
-
id: "
|
|
524
|
+
id: createStorageErrorId("LANCE", "LOAD", "FAILED"),
|
|
741
525
|
domain: ErrorDomain.STORAGE,
|
|
742
526
|
category: ErrorCategory.THIRD_PARTY,
|
|
743
|
-
details: {
|
|
744
|
-
threadId,
|
|
745
|
-
resourceId: resourceId ?? ""
|
|
746
|
-
}
|
|
527
|
+
details: { tableName, keyCount: Object.keys(keys).length, firstKey: Object.keys(keys)[0] ?? "" }
|
|
747
528
|
},
|
|
748
529
|
error
|
|
749
530
|
);
|
|
750
|
-
this.logger?.trackException?.(mastraError);
|
|
751
|
-
this.logger?.error?.(mastraError.toString());
|
|
752
|
-
return { messages: [], total: 0, page, perPage, hasMore: false };
|
|
753
531
|
}
|
|
754
532
|
}
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
533
|
+
};
|
|
534
|
+
|
|
535
|
+
// src/storage/domains/memory/index.ts
|
|
536
|
+
var StoreMemoryLance = class extends MemoryStorage {
|
|
537
|
+
client;
|
|
538
|
+
#db;
|
|
539
|
+
constructor(config) {
|
|
540
|
+
super();
|
|
541
|
+
const client = resolveLanceConfig(config);
|
|
542
|
+
this.client = client;
|
|
543
|
+
this.#db = new LanceDB({ client });
|
|
544
|
+
}
|
|
545
|
+
async init() {
|
|
546
|
+
await this.#db.createTable({ tableName: TABLE_THREADS, schema: TABLE_SCHEMAS[TABLE_THREADS] });
|
|
547
|
+
await this.#db.createTable({ tableName: TABLE_MESSAGES, schema: TABLE_SCHEMAS[TABLE_MESSAGES] });
|
|
548
|
+
await this.#db.createTable({ tableName: TABLE_RESOURCES, schema: TABLE_SCHEMAS[TABLE_RESOURCES] });
|
|
549
|
+
await this.#db.alterTable({
|
|
550
|
+
tableName: TABLE_MESSAGES,
|
|
551
|
+
schema: TABLE_SCHEMAS[TABLE_MESSAGES],
|
|
552
|
+
ifNotExists: ["resourceId"]
|
|
553
|
+
});
|
|
773
554
|
}
|
|
774
|
-
async
|
|
775
|
-
|
|
776
|
-
this.
|
|
777
|
-
|
|
778
|
-
|
|
555
|
+
async dangerouslyClearAll() {
|
|
556
|
+
await this.#db.clearTable({ tableName: TABLE_THREADS });
|
|
557
|
+
await this.#db.clearTable({ tableName: TABLE_MESSAGES });
|
|
558
|
+
await this.#db.clearTable({ tableName: TABLE_RESOURCES });
|
|
559
|
+
}
|
|
560
|
+
async deleteMessages(messageIds) {
|
|
561
|
+
if (!messageIds || messageIds.length === 0) {
|
|
562
|
+
return;
|
|
779
563
|
}
|
|
780
|
-
|
|
781
|
-
const affectedThreadIds = /* @__PURE__ */ new Set();
|
|
564
|
+
this.logger.debug("Deleting messages", { count: messageIds.length });
|
|
782
565
|
try {
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
const
|
|
786
|
-
if (
|
|
787
|
-
|
|
788
|
-
continue;
|
|
789
|
-
}
|
|
790
|
-
const existingMsg = this.parseMessageData(existingMessage);
|
|
791
|
-
const originalThreadId = existingMsg.threadId;
|
|
792
|
-
affectedThreadIds.add(originalThreadId);
|
|
793
|
-
const updatePayload = {};
|
|
794
|
-
if ("role" in updates && updates.role !== void 0) updatePayload.role = updates.role;
|
|
795
|
-
if ("type" in updates && updates.type !== void 0) updatePayload.type = updates.type;
|
|
796
|
-
if ("resourceId" in updates && updates.resourceId !== void 0) updatePayload.resourceId = updates.resourceId;
|
|
797
|
-
if ("threadId" in updates && updates.threadId !== void 0 && updates.threadId !== null) {
|
|
798
|
-
updatePayload.thread_id = updates.threadId;
|
|
799
|
-
affectedThreadIds.add(updates.threadId);
|
|
800
|
-
}
|
|
801
|
-
if (updates.content) {
|
|
802
|
-
const existingContent = existingMsg.content;
|
|
803
|
-
let newContent = { ...existingContent };
|
|
804
|
-
if (updates.content.metadata !== void 0) {
|
|
805
|
-
newContent.metadata = {
|
|
806
|
-
...existingContent.metadata || {},
|
|
807
|
-
...updates.content.metadata || {}
|
|
808
|
-
};
|
|
809
|
-
}
|
|
810
|
-
if (updates.content.content !== void 0) {
|
|
811
|
-
newContent.content = updates.content.content;
|
|
812
|
-
}
|
|
813
|
-
if ("parts" in updates.content && updates.content.parts !== void 0) {
|
|
814
|
-
newContent.parts = updates.content.parts;
|
|
815
|
-
}
|
|
816
|
-
updatePayload.content = JSON.stringify(newContent);
|
|
817
|
-
}
|
|
818
|
-
await this.operations.insert({ tableName: TABLE_MESSAGES, record: { id, ...updatePayload } });
|
|
819
|
-
const updatedMessage = await this.operations.load({ tableName: TABLE_MESSAGES, keys: { id } });
|
|
820
|
-
if (updatedMessage) {
|
|
821
|
-
updatedMessages.push(this.parseMessageData(updatedMessage));
|
|
566
|
+
const threadIds = /* @__PURE__ */ new Set();
|
|
567
|
+
for (const messageId of messageIds) {
|
|
568
|
+
const message = await this.#db.load({ tableName: TABLE_MESSAGES, keys: { id: messageId } });
|
|
569
|
+
if (message?.thread_id) {
|
|
570
|
+
threadIds.add(message.thread_id);
|
|
822
571
|
}
|
|
823
572
|
}
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
573
|
+
const messagesTable = await this.client.openTable(TABLE_MESSAGES);
|
|
574
|
+
const idConditions = messageIds.map((id) => `id = '${this.escapeSql(id)}'`).join(" OR ");
|
|
575
|
+
await messagesTable.delete(idConditions);
|
|
576
|
+
const now = (/* @__PURE__ */ new Date()).getTime();
|
|
577
|
+
const threadsTable = await this.client.openTable(TABLE_THREADS);
|
|
578
|
+
for (const threadId of threadIds) {
|
|
579
|
+
const thread = await this.getThreadById({ threadId });
|
|
580
|
+
if (thread) {
|
|
581
|
+
const record = {
|
|
582
|
+
id: threadId,
|
|
583
|
+
resourceId: thread.resourceId,
|
|
584
|
+
title: thread.title,
|
|
585
|
+
metadata: JSON.stringify(thread.metadata),
|
|
586
|
+
createdAt: new Date(thread.createdAt).getTime(),
|
|
587
|
+
updatedAt: now
|
|
588
|
+
};
|
|
589
|
+
await threadsTable.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
590
|
+
}
|
|
829
591
|
}
|
|
830
|
-
return updatedMessages;
|
|
831
592
|
} catch (error) {
|
|
832
593
|
throw new MastraError(
|
|
833
594
|
{
|
|
834
|
-
id: "
|
|
595
|
+
id: createStorageErrorId("LANCE", "DELETE_MESSAGES", "FAILED"),
|
|
835
596
|
domain: ErrorDomain.STORAGE,
|
|
836
597
|
category: ErrorCategory.THIRD_PARTY,
|
|
837
|
-
details: { count:
|
|
598
|
+
details: { count: messageIds.length }
|
|
838
599
|
},
|
|
839
600
|
error
|
|
840
601
|
);
|
|
841
602
|
}
|
|
842
603
|
}
|
|
843
|
-
|
|
604
|
+
// Utility to escape single quotes in SQL strings
|
|
605
|
+
escapeSql(str) {
|
|
606
|
+
return str.replace(/'/g, "''");
|
|
607
|
+
}
|
|
608
|
+
async getThreadById({ threadId }) {
|
|
844
609
|
try {
|
|
845
|
-
const
|
|
846
|
-
if (!
|
|
610
|
+
const thread = await this.#db.load({ tableName: TABLE_THREADS, keys: { id: threadId } });
|
|
611
|
+
if (!thread) {
|
|
847
612
|
return null;
|
|
848
613
|
}
|
|
849
|
-
let createdAt;
|
|
850
|
-
let updatedAt;
|
|
851
|
-
try {
|
|
852
|
-
if (resource.createdAt instanceof Date) {
|
|
853
|
-
createdAt = resource.createdAt;
|
|
854
|
-
} else if (typeof resource.createdAt === "string") {
|
|
855
|
-
createdAt = new Date(resource.createdAt);
|
|
856
|
-
} else if (typeof resource.createdAt === "number") {
|
|
857
|
-
createdAt = new Date(resource.createdAt);
|
|
858
|
-
} else {
|
|
859
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
860
|
-
}
|
|
861
|
-
if (isNaN(createdAt.getTime())) {
|
|
862
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
863
|
-
}
|
|
864
|
-
} catch {
|
|
865
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
866
|
-
}
|
|
867
|
-
try {
|
|
868
|
-
if (resource.updatedAt instanceof Date) {
|
|
869
|
-
updatedAt = resource.updatedAt;
|
|
870
|
-
} else if (typeof resource.updatedAt === "string") {
|
|
871
|
-
updatedAt = new Date(resource.updatedAt);
|
|
872
|
-
} else if (typeof resource.updatedAt === "number") {
|
|
873
|
-
updatedAt = new Date(resource.updatedAt);
|
|
874
|
-
} else {
|
|
875
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
876
|
-
}
|
|
877
|
-
if (isNaN(updatedAt.getTime())) {
|
|
878
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
879
|
-
}
|
|
880
|
-
} catch {
|
|
881
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
882
|
-
}
|
|
883
|
-
let workingMemory = resource.workingMemory;
|
|
884
|
-
if (workingMemory === null || workingMemory === void 0) {
|
|
885
|
-
workingMemory = void 0;
|
|
886
|
-
} else if (workingMemory === "") {
|
|
887
|
-
workingMemory = "";
|
|
888
|
-
} else if (typeof workingMemory === "object") {
|
|
889
|
-
workingMemory = JSON.stringify(workingMemory);
|
|
890
|
-
}
|
|
891
|
-
let metadata = resource.metadata;
|
|
892
|
-
if (metadata === "" || metadata === null || metadata === void 0) {
|
|
893
|
-
metadata = void 0;
|
|
894
|
-
} else if (typeof metadata === "string") {
|
|
895
|
-
try {
|
|
896
|
-
metadata = JSON.parse(metadata);
|
|
897
|
-
} catch {
|
|
898
|
-
metadata = metadata;
|
|
899
|
-
}
|
|
900
|
-
}
|
|
901
614
|
return {
|
|
902
|
-
...
|
|
903
|
-
createdAt,
|
|
904
|
-
updatedAt
|
|
905
|
-
workingMemory,
|
|
906
|
-
metadata
|
|
615
|
+
...thread,
|
|
616
|
+
createdAt: new Date(thread.createdAt),
|
|
617
|
+
updatedAt: new Date(thread.updatedAt)
|
|
907
618
|
};
|
|
908
619
|
} catch (error) {
|
|
909
620
|
throw new MastraError(
|
|
910
621
|
{
|
|
911
|
-
id: "
|
|
622
|
+
id: createStorageErrorId("LANCE", "GET_THREAD_BY_ID", "FAILED"),
|
|
912
623
|
domain: ErrorDomain.STORAGE,
|
|
913
624
|
category: ErrorCategory.THIRD_PARTY
|
|
914
625
|
},
|
|
@@ -916,23 +627,21 @@ var StoreMemoryLance = class extends MemoryStorage {
|
|
|
916
627
|
);
|
|
917
628
|
}
|
|
918
629
|
}
|
|
919
|
-
|
|
630
|
+
/**
|
|
631
|
+
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
632
|
+
* @param thread - The thread to save
|
|
633
|
+
* @returns The saved thread
|
|
634
|
+
*/
|
|
635
|
+
async saveThread({ thread }) {
|
|
920
636
|
try {
|
|
921
|
-
const record = {
|
|
922
|
-
|
|
923
|
-
metadata: resource.metadata ? JSON.stringify(resource.metadata) : "",
|
|
924
|
-
createdAt: resource.createdAt.getTime(),
|
|
925
|
-
// Store as timestamp (milliseconds)
|
|
926
|
-
updatedAt: resource.updatedAt.getTime()
|
|
927
|
-
// Store as timestamp (milliseconds)
|
|
928
|
-
};
|
|
929
|
-
const table = await this.client.openTable(TABLE_RESOURCES);
|
|
637
|
+
const record = { ...thread, metadata: JSON.stringify(thread.metadata) };
|
|
638
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
930
639
|
await table.add([record], { mode: "append" });
|
|
931
|
-
return
|
|
640
|
+
return thread;
|
|
932
641
|
} catch (error) {
|
|
933
642
|
throw new MastraError(
|
|
934
643
|
{
|
|
935
|
-
id: "
|
|
644
|
+
id: createStorageErrorId("LANCE", "SAVE_THREAD", "FAILED"),
|
|
936
645
|
domain: ErrorDomain.STORAGE,
|
|
937
646
|
category: ErrorCategory.THIRD_PARTY
|
|
938
647
|
},
|
|
@@ -940,44 +649,32 @@ var StoreMemoryLance = class extends MemoryStorage {
|
|
|
940
649
|
);
|
|
941
650
|
}
|
|
942
651
|
}
|
|
943
|
-
async
|
|
944
|
-
|
|
945
|
-
|
|
652
|
+
async updateThread({
|
|
653
|
+
id,
|
|
654
|
+
title,
|
|
946
655
|
metadata
|
|
947
656
|
}) {
|
|
948
|
-
const maxRetries =
|
|
657
|
+
const maxRetries = 5;
|
|
949
658
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
950
659
|
try {
|
|
951
|
-
const
|
|
952
|
-
if (!
|
|
953
|
-
|
|
954
|
-
id: resourceId,
|
|
955
|
-
workingMemory,
|
|
956
|
-
metadata: metadata || {},
|
|
957
|
-
createdAt: /* @__PURE__ */ new Date(),
|
|
958
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
959
|
-
};
|
|
960
|
-
return this.saveResource({ resource: newResource });
|
|
660
|
+
const current = await this.getThreadById({ threadId: id });
|
|
661
|
+
if (!current) {
|
|
662
|
+
throw new Error(`Thread with id ${id} not found`);
|
|
961
663
|
}
|
|
962
|
-
const
|
|
963
|
-
...existingResource,
|
|
964
|
-
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
965
|
-
metadata: {
|
|
966
|
-
...existingResource.metadata,
|
|
967
|
-
...metadata
|
|
968
|
-
},
|
|
969
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
970
|
-
};
|
|
664
|
+
const mergedMetadata = { ...current.metadata, ...metadata };
|
|
971
665
|
const record = {
|
|
972
|
-
id
|
|
973
|
-
|
|
974
|
-
metadata:
|
|
975
|
-
updatedAt:
|
|
976
|
-
// Store as timestamp (milliseconds)
|
|
666
|
+
id,
|
|
667
|
+
title,
|
|
668
|
+
metadata: JSON.stringify(mergedMetadata),
|
|
669
|
+
updatedAt: (/* @__PURE__ */ new Date()).getTime()
|
|
977
670
|
};
|
|
978
|
-
const table = await this.client.openTable(
|
|
671
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
979
672
|
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
980
|
-
|
|
673
|
+
const updatedThread = await this.getThreadById({ threadId: id });
|
|
674
|
+
if (!updatedThread) {
|
|
675
|
+
throw new Error(`Failed to retrieve updated thread ${id}`);
|
|
676
|
+
}
|
|
677
|
+
return updatedThread;
|
|
981
678
|
} catch (error) {
|
|
982
679
|
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
983
680
|
const delay = Math.pow(2, attempt) * 10;
|
|
@@ -986,7 +683,7 @@ var StoreMemoryLance = class extends MemoryStorage {
|
|
|
986
683
|
}
|
|
987
684
|
throw new MastraError(
|
|
988
685
|
{
|
|
989
|
-
id: "
|
|
686
|
+
id: createStorageErrorId("LANCE", "UPDATE_THREAD", "FAILED"),
|
|
990
687
|
domain: ErrorDomain.STORAGE,
|
|
991
688
|
category: ErrorCategory.THIRD_PARTY
|
|
992
689
|
},
|
|
@@ -994,419 +691,632 @@ var StoreMemoryLance = class extends MemoryStorage {
|
|
|
994
691
|
);
|
|
995
692
|
}
|
|
996
693
|
}
|
|
997
|
-
throw new
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
}
|
|
1006
|
-
getDefaultValue(type) {
|
|
1007
|
-
switch (type) {
|
|
1008
|
-
case "text":
|
|
1009
|
-
return "''";
|
|
1010
|
-
case "timestamp":
|
|
1011
|
-
return "CURRENT_TIMESTAMP";
|
|
1012
|
-
case "integer":
|
|
1013
|
-
case "bigint":
|
|
1014
|
-
return "0";
|
|
1015
|
-
case "jsonb":
|
|
1016
|
-
return "'{}'";
|
|
1017
|
-
case "uuid":
|
|
1018
|
-
return "''";
|
|
1019
|
-
default:
|
|
1020
|
-
return super.getDefaultValue(type);
|
|
1021
|
-
}
|
|
1022
|
-
}
|
|
1023
|
-
async hasColumn(tableName, columnName) {
|
|
1024
|
-
const table = await this.client.openTable(tableName);
|
|
1025
|
-
const schema = await table.schema();
|
|
1026
|
-
return schema.fields.some((field) => field.name === columnName);
|
|
1027
|
-
}
|
|
1028
|
-
translateSchema(schema) {
|
|
1029
|
-
const fields = Object.entries(schema).map(([name, column]) => {
|
|
1030
|
-
let arrowType;
|
|
1031
|
-
switch (column.type.toLowerCase()) {
|
|
1032
|
-
case "text":
|
|
1033
|
-
case "uuid":
|
|
1034
|
-
arrowType = new Utf8();
|
|
1035
|
-
break;
|
|
1036
|
-
case "int":
|
|
1037
|
-
case "integer":
|
|
1038
|
-
arrowType = new Int32();
|
|
1039
|
-
break;
|
|
1040
|
-
case "bigint":
|
|
1041
|
-
arrowType = new Float64();
|
|
1042
|
-
break;
|
|
1043
|
-
case "float":
|
|
1044
|
-
arrowType = new Float32();
|
|
1045
|
-
break;
|
|
1046
|
-
case "jsonb":
|
|
1047
|
-
case "json":
|
|
1048
|
-
arrowType = new Utf8();
|
|
1049
|
-
break;
|
|
1050
|
-
case "binary":
|
|
1051
|
-
arrowType = new Binary();
|
|
1052
|
-
break;
|
|
1053
|
-
case "timestamp":
|
|
1054
|
-
arrowType = new Float64();
|
|
1055
|
-
break;
|
|
1056
|
-
default:
|
|
1057
|
-
arrowType = new Utf8();
|
|
1058
|
-
}
|
|
1059
|
-
return new Field(name, arrowType, column.nullable ?? true);
|
|
1060
|
-
});
|
|
1061
|
-
return new Schema(fields);
|
|
694
|
+
throw new MastraError(
|
|
695
|
+
{
|
|
696
|
+
id: createStorageErrorId("LANCE", "UPDATE_THREAD", "FAILED"),
|
|
697
|
+
domain: ErrorDomain.STORAGE,
|
|
698
|
+
category: ErrorCategory.THIRD_PARTY
|
|
699
|
+
},
|
|
700
|
+
new Error("All retries exhausted")
|
|
701
|
+
);
|
|
1062
702
|
}
|
|
1063
|
-
async
|
|
1064
|
-
tableName,
|
|
1065
|
-
schema
|
|
1066
|
-
}) {
|
|
703
|
+
async deleteThread({ threadId }) {
|
|
1067
704
|
try {
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
throw new Error("tableName is required for createTable.");
|
|
1073
|
-
}
|
|
1074
|
-
if (!schema) {
|
|
1075
|
-
throw new Error("schema is required for createTable.");
|
|
1076
|
-
}
|
|
705
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
706
|
+
await table.delete(`id = '${threadId}'`);
|
|
707
|
+
const messagesTable = await this.client.openTable(TABLE_MESSAGES);
|
|
708
|
+
await messagesTable.delete(`thread_id = '${threadId}'`);
|
|
1077
709
|
} catch (error) {
|
|
1078
710
|
throw new MastraError(
|
|
1079
711
|
{
|
|
1080
|
-
id: "
|
|
712
|
+
id: createStorageErrorId("LANCE", "DELETE_THREAD", "FAILED"),
|
|
1081
713
|
domain: ErrorDomain.STORAGE,
|
|
1082
|
-
category: ErrorCategory.
|
|
1083
|
-
details: { tableName }
|
|
714
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1084
715
|
},
|
|
1085
716
|
error
|
|
1086
717
|
);
|
|
1087
718
|
}
|
|
719
|
+
}
|
|
720
|
+
normalizeMessage(message) {
|
|
721
|
+
const { thread_id, ...rest } = message;
|
|
722
|
+
return {
|
|
723
|
+
...rest,
|
|
724
|
+
threadId: thread_id,
|
|
725
|
+
content: typeof message.content === "string" ? (() => {
|
|
726
|
+
try {
|
|
727
|
+
return JSON.parse(message.content);
|
|
728
|
+
} catch {
|
|
729
|
+
return message.content;
|
|
730
|
+
}
|
|
731
|
+
})() : message.content
|
|
732
|
+
};
|
|
733
|
+
}
|
|
734
|
+
async listMessagesById({ messageIds }) {
|
|
735
|
+
if (messageIds.length === 0) return { messages: [] };
|
|
1088
736
|
try {
|
|
1089
|
-
const
|
|
1090
|
-
|
|
737
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
738
|
+
const quotedIds = messageIds.map((id) => `'${id}'`).join(", ");
|
|
739
|
+
const allRecords = await table.query().where(`id IN (${quotedIds})`).toArray();
|
|
740
|
+
const messages = processResultWithTypeConversion(
|
|
741
|
+
allRecords,
|
|
742
|
+
await getTableSchema({ tableName: TABLE_MESSAGES, client: this.client })
|
|
743
|
+
);
|
|
744
|
+
const list = new MessageList().add(
|
|
745
|
+
messages.map(this.normalizeMessage),
|
|
746
|
+
"memory"
|
|
747
|
+
);
|
|
748
|
+
return { messages: list.get.all.db() };
|
|
1091
749
|
} catch (error) {
|
|
1092
|
-
if (error.message?.includes("already exists")) {
|
|
1093
|
-
this.logger.debug(`Table '${tableName}' already exists, skipping create`);
|
|
1094
|
-
return;
|
|
1095
|
-
}
|
|
1096
750
|
throw new MastraError(
|
|
1097
751
|
{
|
|
1098
|
-
id: "
|
|
752
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES_BY_ID", "FAILED"),
|
|
1099
753
|
domain: ErrorDomain.STORAGE,
|
|
1100
754
|
category: ErrorCategory.THIRD_PARTY,
|
|
1101
|
-
details: {
|
|
755
|
+
details: {
|
|
756
|
+
messageIds: JSON.stringify(messageIds)
|
|
757
|
+
}
|
|
1102
758
|
},
|
|
1103
759
|
error
|
|
1104
760
|
);
|
|
1105
761
|
}
|
|
1106
762
|
}
|
|
1107
|
-
async
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
}
|
|
1112
|
-
if (!tableName) {
|
|
1113
|
-
throw new Error("tableName is required for dropTable.");
|
|
1114
|
-
}
|
|
1115
|
-
} catch (validationError) {
|
|
1116
|
-
throw new MastraError(
|
|
1117
|
-
{
|
|
1118
|
-
id: "STORAGE_LANCE_STORAGE_DROP_TABLE_INVALID_ARGS",
|
|
1119
|
-
domain: ErrorDomain.STORAGE,
|
|
1120
|
-
category: ErrorCategory.USER,
|
|
1121
|
-
text: validationError.message,
|
|
1122
|
-
details: { tableName }
|
|
1123
|
-
},
|
|
1124
|
-
validationError
|
|
1125
|
-
);
|
|
1126
|
-
}
|
|
1127
|
-
try {
|
|
1128
|
-
await this.client.dropTable(tableName);
|
|
1129
|
-
} catch (error) {
|
|
1130
|
-
if (error.toString().includes("was not found") || error.message?.includes("Table not found")) {
|
|
1131
|
-
this.logger.debug(`Table '${tableName}' does not exist, skipping drop`);
|
|
1132
|
-
return;
|
|
1133
|
-
}
|
|
763
|
+
async listMessages(args) {
|
|
764
|
+
const { threadId, resourceId, include, filter, perPage: perPageInput, page = 0, orderBy } = args;
|
|
765
|
+
const threadIds = Array.isArray(threadId) ? threadId : [threadId];
|
|
766
|
+
if (threadIds.length === 0 || threadIds.some((id) => !id.trim())) {
|
|
1134
767
|
throw new MastraError(
|
|
1135
768
|
{
|
|
1136
|
-
id: "
|
|
769
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES", "INVALID_THREAD_ID"),
|
|
1137
770
|
domain: ErrorDomain.STORAGE,
|
|
1138
771
|
category: ErrorCategory.THIRD_PARTY,
|
|
1139
|
-
details: {
|
|
772
|
+
details: { threadId: Array.isArray(threadId) ? threadId.join(",") : threadId }
|
|
1140
773
|
},
|
|
1141
|
-
|
|
774
|
+
new Error("threadId must be a non-empty string or array of non-empty strings")
|
|
1142
775
|
);
|
|
1143
776
|
}
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
tableName,
|
|
1147
|
-
schema,
|
|
1148
|
-
ifNotExists
|
|
1149
|
-
}) {
|
|
777
|
+
const perPage = normalizePerPage(perPageInput, 40);
|
|
778
|
+
const { offset, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
1150
779
|
try {
|
|
1151
|
-
if (
|
|
1152
|
-
throw new
|
|
780
|
+
if (page < 0) {
|
|
781
|
+
throw new MastraError(
|
|
782
|
+
{
|
|
783
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES", "INVALID_PAGE"),
|
|
784
|
+
domain: ErrorDomain.STORAGE,
|
|
785
|
+
category: ErrorCategory.USER,
|
|
786
|
+
details: { page }
|
|
787
|
+
},
|
|
788
|
+
new Error("page must be >= 0")
|
|
789
|
+
);
|
|
1153
790
|
}
|
|
1154
|
-
|
|
1155
|
-
|
|
791
|
+
const { field, direction } = this.parseOrderBy(orderBy, "ASC");
|
|
792
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
793
|
+
const threadCondition = threadIds.length === 1 ? `thread_id = '${this.escapeSql(threadIds[0])}'` : `thread_id IN (${threadIds.map((t) => `'${this.escapeSql(t)}'`).join(", ")})`;
|
|
794
|
+
const conditions = [threadCondition];
|
|
795
|
+
if (resourceId) {
|
|
796
|
+
conditions.push(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
1156
797
|
}
|
|
1157
|
-
if (
|
|
1158
|
-
|
|
798
|
+
if (filter?.dateRange?.start) {
|
|
799
|
+
const startTime = filter.dateRange.start instanceof Date ? filter.dateRange.start.getTime() : new Date(filter.dateRange.start).getTime();
|
|
800
|
+
const startOp = filter.dateRange.startExclusive ? ">" : ">=";
|
|
801
|
+
conditions.push(`\`createdAt\` ${startOp} ${startTime}`);
|
|
1159
802
|
}
|
|
1160
|
-
if (
|
|
1161
|
-
|
|
1162
|
-
|
|
803
|
+
if (filter?.dateRange?.end) {
|
|
804
|
+
const endTime = filter.dateRange.end instanceof Date ? filter.dateRange.end.getTime() : new Date(filter.dateRange.end).getTime();
|
|
805
|
+
const endOp = filter.dateRange.endExclusive ? "<" : "<=";
|
|
806
|
+
conditions.push(`\`createdAt\` ${endOp} ${endTime}`);
|
|
1163
807
|
}
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
const
|
|
1180
|
-
const
|
|
1181
|
-
|
|
1182
|
-
integer: "int",
|
|
1183
|
-
bigint: "bigint",
|
|
1184
|
-
timestamp: "timestamp",
|
|
1185
|
-
jsonb: "string",
|
|
1186
|
-
uuid: "string"
|
|
1187
|
-
};
|
|
1188
|
-
const columnsToAdd = ifNotExists.filter((col) => schema[col] && !existingFields.has(col)).map((col) => {
|
|
1189
|
-
const colDef = schema[col];
|
|
808
|
+
const whereClause = conditions.join(" AND ");
|
|
809
|
+
const total = await table.countRows(whereClause);
|
|
810
|
+
const query = table.query().where(whereClause);
|
|
811
|
+
let allRecords = await query.toArray();
|
|
812
|
+
allRecords.sort((a, b) => {
|
|
813
|
+
const aValue = field === "createdAt" ? a.createdAt : a[field];
|
|
814
|
+
const bValue = field === "createdAt" ? b.createdAt : b[field];
|
|
815
|
+
if (aValue == null && bValue == null) return 0;
|
|
816
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
817
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
818
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
819
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
820
|
+
}
|
|
821
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
822
|
+
});
|
|
823
|
+
const paginatedRecords = allRecords.slice(offset, offset + perPage);
|
|
824
|
+
const messages = paginatedRecords.map((row) => this.normalizeMessage(row));
|
|
825
|
+
if (total === 0 && messages.length === 0 && (!include || include.length === 0)) {
|
|
1190
826
|
return {
|
|
1191
|
-
|
|
1192
|
-
|
|
827
|
+
messages: [],
|
|
828
|
+
total: 0,
|
|
829
|
+
page,
|
|
830
|
+
perPage: perPageForResponse,
|
|
831
|
+
hasMore: false
|
|
1193
832
|
};
|
|
1194
|
-
});
|
|
1195
|
-
if (columnsToAdd.length > 0) {
|
|
1196
|
-
await table.addColumns(columnsToAdd);
|
|
1197
|
-
this.logger?.info?.(`Added columns [${columnsToAdd.map((c) => c.name).join(", ")}] to table ${tableName}`);
|
|
1198
833
|
}
|
|
834
|
+
const messageIds = new Set(messages.map((m) => m.id));
|
|
835
|
+
if (include && include.length > 0) {
|
|
836
|
+
const threadIds2 = [...new Set(include.map((item) => item.threadId || threadId))];
|
|
837
|
+
const allThreadMessages = [];
|
|
838
|
+
for (const tid of threadIds2) {
|
|
839
|
+
const threadQuery = table.query().where(`thread_id = '${tid}'`);
|
|
840
|
+
let threadRecords = await threadQuery.toArray();
|
|
841
|
+
allThreadMessages.push(...threadRecords);
|
|
842
|
+
}
|
|
843
|
+
allThreadMessages.sort((a, b) => a.createdAt - b.createdAt);
|
|
844
|
+
const contextMessages = this.processMessagesWithContext(allThreadMessages, include);
|
|
845
|
+
const includedMessages = contextMessages.map((row) => this.normalizeMessage(row));
|
|
846
|
+
for (const includeMsg of includedMessages) {
|
|
847
|
+
if (!messageIds.has(includeMsg.id)) {
|
|
848
|
+
messages.push(includeMsg);
|
|
849
|
+
messageIds.add(includeMsg.id);
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
const list = new MessageList().add(messages, "memory");
|
|
854
|
+
let finalMessages = list.get.all.db();
|
|
855
|
+
finalMessages = finalMessages.sort((a, b) => {
|
|
856
|
+
const aValue = field === "createdAt" ? new Date(a.createdAt).getTime() : a[field];
|
|
857
|
+
const bValue = field === "createdAt" ? new Date(b.createdAt).getTime() : b[field];
|
|
858
|
+
if (aValue == null && bValue == null) return 0;
|
|
859
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
860
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
861
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
862
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
863
|
+
}
|
|
864
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
865
|
+
});
|
|
866
|
+
const returnedThreadMessageIds = new Set(finalMessages.filter((m) => m.threadId === threadId).map((m) => m.id));
|
|
867
|
+
const allThreadMessagesReturned = returnedThreadMessageIds.size >= total;
|
|
868
|
+
const fetchedAll = perPageInput === false || allThreadMessagesReturned;
|
|
869
|
+
const hasMore = !fetchedAll && offset + perPage < total;
|
|
870
|
+
return {
|
|
871
|
+
messages: finalMessages,
|
|
872
|
+
total,
|
|
873
|
+
page,
|
|
874
|
+
perPage: perPageForResponse,
|
|
875
|
+
hasMore
|
|
876
|
+
};
|
|
1199
877
|
} catch (error) {
|
|
1200
|
-
|
|
878
|
+
const mastraError = new MastraError(
|
|
1201
879
|
{
|
|
1202
|
-
id: "
|
|
880
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES", "FAILED"),
|
|
1203
881
|
domain: ErrorDomain.STORAGE,
|
|
1204
882
|
category: ErrorCategory.THIRD_PARTY,
|
|
1205
|
-
details: {
|
|
883
|
+
details: {
|
|
884
|
+
threadId: Array.isArray(threadId) ? threadId.join(",") : threadId,
|
|
885
|
+
resourceId: resourceId ?? ""
|
|
886
|
+
}
|
|
1206
887
|
},
|
|
1207
888
|
error
|
|
1208
889
|
);
|
|
890
|
+
this.logger?.error?.(mastraError.toString());
|
|
891
|
+
this.logger?.trackException?.(mastraError);
|
|
892
|
+
return {
|
|
893
|
+
messages: [],
|
|
894
|
+
total: 0,
|
|
895
|
+
page,
|
|
896
|
+
perPage: perPageForResponse,
|
|
897
|
+
hasMore: false
|
|
898
|
+
};
|
|
1209
899
|
}
|
|
1210
900
|
}
|
|
1211
|
-
async
|
|
901
|
+
async saveMessages(args) {
|
|
1212
902
|
try {
|
|
1213
|
-
|
|
1214
|
-
|
|
903
|
+
const { messages } = args;
|
|
904
|
+
if (messages.length === 0) {
|
|
905
|
+
return { messages: [] };
|
|
1215
906
|
}
|
|
1216
|
-
|
|
1217
|
-
|
|
907
|
+
const threadId = messages[0]?.threadId;
|
|
908
|
+
if (!threadId) {
|
|
909
|
+
throw new Error("Thread ID is required");
|
|
1218
910
|
}
|
|
1219
|
-
|
|
911
|
+
for (const message of messages) {
|
|
912
|
+
if (!message.id) {
|
|
913
|
+
throw new Error("Message ID is required");
|
|
914
|
+
}
|
|
915
|
+
if (!message.threadId) {
|
|
916
|
+
throw new Error("Thread ID is required for all messages");
|
|
917
|
+
}
|
|
918
|
+
if (message.resourceId === null || message.resourceId === void 0) {
|
|
919
|
+
throw new Error("Resource ID cannot be null or undefined");
|
|
920
|
+
}
|
|
921
|
+
if (!message.content) {
|
|
922
|
+
throw new Error("Message content is required");
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
const transformedMessages = messages.map((message) => {
|
|
926
|
+
const { threadId: threadId2, type, ...rest } = message;
|
|
927
|
+
return {
|
|
928
|
+
...rest,
|
|
929
|
+
thread_id: threadId2,
|
|
930
|
+
type: type ?? "v2",
|
|
931
|
+
content: JSON.stringify(message.content)
|
|
932
|
+
};
|
|
933
|
+
});
|
|
934
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
935
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(transformedMessages);
|
|
936
|
+
const threadsTable = await this.client.openTable(TABLE_THREADS);
|
|
937
|
+
const currentTime = (/* @__PURE__ */ new Date()).getTime();
|
|
938
|
+
const updateRecord = { id: threadId, updatedAt: currentTime };
|
|
939
|
+
await threadsTable.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([updateRecord]);
|
|
940
|
+
const list = new MessageList().add(messages, "memory");
|
|
941
|
+
return { messages: list.get.all.db() };
|
|
942
|
+
} catch (error) {
|
|
1220
943
|
throw new MastraError(
|
|
1221
944
|
{
|
|
1222
|
-
id: "
|
|
945
|
+
id: createStorageErrorId("LANCE", "SAVE_MESSAGES", "FAILED"),
|
|
1223
946
|
domain: ErrorDomain.STORAGE,
|
|
1224
|
-
category: ErrorCategory.
|
|
1225
|
-
text: validationError.message,
|
|
1226
|
-
details: { tableName }
|
|
947
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1227
948
|
},
|
|
1228
|
-
|
|
949
|
+
error
|
|
1229
950
|
);
|
|
1230
951
|
}
|
|
952
|
+
}
|
|
953
|
+
async listThreadsByResourceId(args) {
|
|
1231
954
|
try {
|
|
1232
|
-
const
|
|
1233
|
-
|
|
955
|
+
const { resourceId, page = 0, perPage: perPageInput, orderBy } = args;
|
|
956
|
+
const perPage = normalizePerPage(perPageInput, 100);
|
|
957
|
+
if (page < 0) {
|
|
958
|
+
throw new MastraError(
|
|
959
|
+
{
|
|
960
|
+
id: createStorageErrorId("LANCE", "LIST_THREADS_BY_RESOURCE_ID", "INVALID_PAGE"),
|
|
961
|
+
domain: ErrorDomain.STORAGE,
|
|
962
|
+
category: ErrorCategory.USER,
|
|
963
|
+
details: { page }
|
|
964
|
+
},
|
|
965
|
+
new Error("page must be >= 0")
|
|
966
|
+
);
|
|
967
|
+
}
|
|
968
|
+
const { offset, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
969
|
+
const { field, direction } = this.parseOrderBy(orderBy);
|
|
970
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
971
|
+
const total = await table.countRows(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
972
|
+
const query = table.query().where(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
973
|
+
const records = await query.toArray();
|
|
974
|
+
records.sort((a, b) => {
|
|
975
|
+
const aValue = ["createdAt", "updatedAt"].includes(field) ? new Date(a[field]).getTime() : a[field];
|
|
976
|
+
const bValue = ["createdAt", "updatedAt"].includes(field) ? new Date(b[field]).getTime() : b[field];
|
|
977
|
+
if (aValue == null && bValue == null) return 0;
|
|
978
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
979
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
980
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
981
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
982
|
+
}
|
|
983
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
984
|
+
});
|
|
985
|
+
const paginatedRecords = records.slice(offset, offset + perPage);
|
|
986
|
+
const schema = await getTableSchema({ tableName: TABLE_THREADS, client: this.client });
|
|
987
|
+
const threads = paginatedRecords.map(
|
|
988
|
+
(record) => processResultWithTypeConversion(record, schema)
|
|
989
|
+
);
|
|
990
|
+
return {
|
|
991
|
+
threads,
|
|
992
|
+
total,
|
|
993
|
+
page,
|
|
994
|
+
perPage: perPageForResponse,
|
|
995
|
+
hasMore: offset + perPage < total
|
|
996
|
+
};
|
|
1234
997
|
} catch (error) {
|
|
1235
998
|
throw new MastraError(
|
|
1236
999
|
{
|
|
1237
|
-
id: "
|
|
1000
|
+
id: createStorageErrorId("LANCE", "LIST_THREADS_BY_RESOURCE_ID", "FAILED"),
|
|
1238
1001
|
domain: ErrorDomain.STORAGE,
|
|
1239
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1240
|
-
details: { tableName }
|
|
1002
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1241
1003
|
},
|
|
1242
1004
|
error
|
|
1243
1005
|
);
|
|
1244
1006
|
}
|
|
1245
1007
|
}
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1008
|
+
/**
|
|
1009
|
+
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
1010
|
+
* @param records - The sorted array of records to process
|
|
1011
|
+
* @param include - The array of include specifications with context parameters
|
|
1012
|
+
* @returns The processed array with context messages included
|
|
1013
|
+
*/
|
|
1014
|
+
processMessagesWithContext(records, include) {
|
|
1015
|
+
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
1016
|
+
if (messagesWithContext.length === 0) {
|
|
1017
|
+
return records;
|
|
1018
|
+
}
|
|
1019
|
+
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
1020
|
+
records.forEach((message, index) => {
|
|
1021
|
+
messageIndexMap.set(message.id, index);
|
|
1022
|
+
});
|
|
1023
|
+
const additionalIndices = /* @__PURE__ */ new Set();
|
|
1024
|
+
for (const item of messagesWithContext) {
|
|
1025
|
+
const messageIndex = messageIndexMap.get(item.id);
|
|
1026
|
+
if (messageIndex !== void 0) {
|
|
1027
|
+
if (item.withPreviousMessages) {
|
|
1028
|
+
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
1029
|
+
for (let i = startIdx; i < messageIndex; i++) {
|
|
1030
|
+
additionalIndices.add(i);
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
if (item.withNextMessages) {
|
|
1034
|
+
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
1035
|
+
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
1036
|
+
additionalIndices.add(i);
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1253
1039
|
}
|
|
1254
|
-
|
|
1255
|
-
|
|
1040
|
+
}
|
|
1041
|
+
if (additionalIndices.size === 0) {
|
|
1042
|
+
return records;
|
|
1043
|
+
}
|
|
1044
|
+
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
1045
|
+
const allIndices = /* @__PURE__ */ new Set();
|
|
1046
|
+
records.forEach((record, index) => {
|
|
1047
|
+
if (originalMatchIds.has(record.id)) {
|
|
1048
|
+
allIndices.add(index);
|
|
1256
1049
|
}
|
|
1257
|
-
}
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1050
|
+
});
|
|
1051
|
+
additionalIndices.forEach((index) => {
|
|
1052
|
+
allIndices.add(index);
|
|
1053
|
+
});
|
|
1054
|
+
return Array.from(allIndices).sort((a, b) => a - b).map((index) => records[index]);
|
|
1055
|
+
}
|
|
1056
|
+
/**
|
|
1057
|
+
* Parse message data from LanceDB record format to MastraDBMessage format
|
|
1058
|
+
*/
|
|
1059
|
+
parseMessageData(data) {
|
|
1060
|
+
const { thread_id, ...rest } = data;
|
|
1061
|
+
return {
|
|
1062
|
+
...rest,
|
|
1063
|
+
threadId: thread_id,
|
|
1064
|
+
content: typeof data.content === "string" ? (() => {
|
|
1065
|
+
try {
|
|
1066
|
+
return JSON.parse(data.content);
|
|
1067
|
+
} catch {
|
|
1068
|
+
return data.content;
|
|
1069
|
+
}
|
|
1070
|
+
})() : data.content,
|
|
1071
|
+
createdAt: new Date(data.createdAt),
|
|
1072
|
+
updatedAt: new Date(data.updatedAt)
|
|
1073
|
+
};
|
|
1074
|
+
}
|
|
1075
|
+
async updateMessages(args) {
|
|
1076
|
+
const { messages } = args;
|
|
1077
|
+
this.logger.debug("Updating messages", { count: messages.length });
|
|
1078
|
+
if (!messages.length) {
|
|
1079
|
+
return [];
|
|
1268
1080
|
}
|
|
1081
|
+
const updatedMessages = [];
|
|
1082
|
+
const affectedThreadIds = /* @__PURE__ */ new Set();
|
|
1269
1083
|
try {
|
|
1270
|
-
const
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1084
|
+
for (const updateData of messages) {
|
|
1085
|
+
const { id, ...updates } = updateData;
|
|
1086
|
+
const existingMessage = await this.#db.load({ tableName: TABLE_MESSAGES, keys: { id } });
|
|
1087
|
+
if (!existingMessage) {
|
|
1088
|
+
this.logger.warn("Message not found for update", { id });
|
|
1089
|
+
continue;
|
|
1090
|
+
}
|
|
1091
|
+
const existingMsg = this.parseMessageData(existingMessage);
|
|
1092
|
+
const originalThreadId = existingMsg.threadId;
|
|
1093
|
+
affectedThreadIds.add(originalThreadId);
|
|
1094
|
+
const updatePayload = {};
|
|
1095
|
+
if ("role" in updates && updates.role !== void 0) updatePayload.role = updates.role;
|
|
1096
|
+
if ("type" in updates && updates.type !== void 0) updatePayload.type = updates.type;
|
|
1097
|
+
if ("resourceId" in updates && updates.resourceId !== void 0) updatePayload.resourceId = updates.resourceId;
|
|
1098
|
+
if ("threadId" in updates && updates.threadId !== void 0 && updates.threadId !== null) {
|
|
1099
|
+
updatePayload.thread_id = updates.threadId;
|
|
1100
|
+
affectedThreadIds.add(updates.threadId);
|
|
1101
|
+
}
|
|
1102
|
+
if (updates.content) {
|
|
1103
|
+
const existingContent = existingMsg.content;
|
|
1104
|
+
let newContent = { ...existingContent };
|
|
1105
|
+
if (updates.content.metadata !== void 0) {
|
|
1106
|
+
newContent.metadata = {
|
|
1107
|
+
...existingContent.metadata || {},
|
|
1108
|
+
...updates.content.metadata || {}
|
|
1109
|
+
};
|
|
1110
|
+
}
|
|
1111
|
+
if (updates.content.content !== void 0) {
|
|
1112
|
+
newContent.content = updates.content.content;
|
|
1113
|
+
}
|
|
1114
|
+
if ("parts" in updates.content && updates.content.parts !== void 0) {
|
|
1115
|
+
newContent.parts = updates.content.parts;
|
|
1116
|
+
}
|
|
1117
|
+
updatePayload.content = JSON.stringify(newContent);
|
|
1118
|
+
}
|
|
1119
|
+
await this.#db.insert({ tableName: TABLE_MESSAGES, record: { id, ...updatePayload } });
|
|
1120
|
+
const updatedMessage = await this.#db.load({ tableName: TABLE_MESSAGES, keys: { id } });
|
|
1121
|
+
if (updatedMessage) {
|
|
1122
|
+
updatedMessages.push(this.parseMessageData(updatedMessage));
|
|
1277
1123
|
}
|
|
1278
1124
|
}
|
|
1279
|
-
|
|
1280
|
-
|
|
1125
|
+
for (const threadId of affectedThreadIds) {
|
|
1126
|
+
await this.#db.insert({
|
|
1127
|
+
tableName: TABLE_THREADS,
|
|
1128
|
+
record: { id: threadId, updatedAt: Date.now() }
|
|
1129
|
+
});
|
|
1130
|
+
}
|
|
1131
|
+
return updatedMessages;
|
|
1281
1132
|
} catch (error) {
|
|
1282
1133
|
throw new MastraError(
|
|
1283
1134
|
{
|
|
1284
|
-
id: "
|
|
1135
|
+
id: createStorageErrorId("LANCE", "UPDATE_MESSAGES", "FAILED"),
|
|
1285
1136
|
domain: ErrorDomain.STORAGE,
|
|
1286
1137
|
category: ErrorCategory.THIRD_PARTY,
|
|
1287
|
-
details: {
|
|
1138
|
+
details: { count: messages.length }
|
|
1288
1139
|
},
|
|
1289
1140
|
error
|
|
1290
1141
|
);
|
|
1291
1142
|
}
|
|
1292
1143
|
}
|
|
1293
|
-
async
|
|
1144
|
+
async getResourceById({ resourceId }) {
|
|
1294
1145
|
try {
|
|
1295
|
-
|
|
1296
|
-
|
|
1146
|
+
const resource = await this.#db.load({ tableName: TABLE_RESOURCES, keys: { id: resourceId } });
|
|
1147
|
+
if (!resource) {
|
|
1148
|
+
return null;
|
|
1297
1149
|
}
|
|
1298
|
-
|
|
1299
|
-
|
|
1150
|
+
let createdAt;
|
|
1151
|
+
let updatedAt;
|
|
1152
|
+
try {
|
|
1153
|
+
if (resource.createdAt instanceof Date) {
|
|
1154
|
+
createdAt = resource.createdAt;
|
|
1155
|
+
} else if (typeof resource.createdAt === "string") {
|
|
1156
|
+
createdAt = new Date(resource.createdAt);
|
|
1157
|
+
} else if (typeof resource.createdAt === "number") {
|
|
1158
|
+
createdAt = new Date(resource.createdAt);
|
|
1159
|
+
} else {
|
|
1160
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1161
|
+
}
|
|
1162
|
+
if (isNaN(createdAt.getTime())) {
|
|
1163
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1164
|
+
}
|
|
1165
|
+
} catch {
|
|
1166
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1300
1167
|
}
|
|
1301
|
-
|
|
1302
|
-
|
|
1168
|
+
try {
|
|
1169
|
+
if (resource.updatedAt instanceof Date) {
|
|
1170
|
+
updatedAt = resource.updatedAt;
|
|
1171
|
+
} else if (typeof resource.updatedAt === "string") {
|
|
1172
|
+
updatedAt = new Date(resource.updatedAt);
|
|
1173
|
+
} else if (typeof resource.updatedAt === "number") {
|
|
1174
|
+
updatedAt = new Date(resource.updatedAt);
|
|
1175
|
+
} else {
|
|
1176
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1177
|
+
}
|
|
1178
|
+
if (isNaN(updatedAt.getTime())) {
|
|
1179
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1180
|
+
}
|
|
1181
|
+
} catch {
|
|
1182
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1303
1183
|
}
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
const processedRecord = { ...record };
|
|
1321
|
-
for (const key in processedRecord) {
|
|
1322
|
-
if (processedRecord[key] == null) continue;
|
|
1323
|
-
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
1324
|
-
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
1325
|
-
}
|
|
1184
|
+
let workingMemory = resource.workingMemory;
|
|
1185
|
+
if (workingMemory === null || workingMemory === void 0) {
|
|
1186
|
+
workingMemory = void 0;
|
|
1187
|
+
} else if (workingMemory === "") {
|
|
1188
|
+
workingMemory = "";
|
|
1189
|
+
} else if (typeof workingMemory === "object") {
|
|
1190
|
+
workingMemory = JSON.stringify(workingMemory);
|
|
1191
|
+
}
|
|
1192
|
+
let metadata = resource.metadata;
|
|
1193
|
+
if (metadata === "" || metadata === null || metadata === void 0) {
|
|
1194
|
+
metadata = void 0;
|
|
1195
|
+
} else if (typeof metadata === "string") {
|
|
1196
|
+
try {
|
|
1197
|
+
metadata = JSON.parse(metadata);
|
|
1198
|
+
} catch {
|
|
1199
|
+
metadata = metadata;
|
|
1326
1200
|
}
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1201
|
+
}
|
|
1202
|
+
return {
|
|
1203
|
+
...resource,
|
|
1204
|
+
createdAt,
|
|
1205
|
+
updatedAt,
|
|
1206
|
+
workingMemory,
|
|
1207
|
+
metadata
|
|
1208
|
+
};
|
|
1330
1209
|
} catch (error) {
|
|
1331
1210
|
throw new MastraError(
|
|
1332
1211
|
{
|
|
1333
|
-
id: "
|
|
1212
|
+
id: createStorageErrorId("LANCE", "GET_RESOURCE_BY_ID", "FAILED"),
|
|
1334
1213
|
domain: ErrorDomain.STORAGE,
|
|
1335
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1336
|
-
details: { tableName }
|
|
1214
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1337
1215
|
},
|
|
1338
1216
|
error
|
|
1339
1217
|
);
|
|
1340
1218
|
}
|
|
1341
1219
|
}
|
|
1342
|
-
async
|
|
1343
|
-
try {
|
|
1344
|
-
if (!this.client) {
|
|
1345
|
-
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1346
|
-
}
|
|
1347
|
-
if (!tableName) {
|
|
1348
|
-
throw new Error("tableName is required for load.");
|
|
1349
|
-
}
|
|
1350
|
-
if (!keys || Object.keys(keys).length === 0) {
|
|
1351
|
-
throw new Error("keys are required and cannot be empty for load.");
|
|
1352
|
-
}
|
|
1353
|
-
} catch (validationError) {
|
|
1354
|
-
throw new MastraError(
|
|
1355
|
-
{
|
|
1356
|
-
id: "STORAGE_LANCE_STORAGE_LOAD_INVALID_ARGS",
|
|
1357
|
-
domain: ErrorDomain.STORAGE,
|
|
1358
|
-
category: ErrorCategory.USER,
|
|
1359
|
-
text: validationError.message,
|
|
1360
|
-
details: { tableName }
|
|
1361
|
-
},
|
|
1362
|
-
validationError
|
|
1363
|
-
);
|
|
1364
|
-
}
|
|
1220
|
+
async saveResource({ resource }) {
|
|
1365
1221
|
try {
|
|
1366
|
-
const
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
return `${quotedKey} IS NULL`;
|
|
1378
|
-
} else {
|
|
1379
|
-
return `${quotedKey} = ${value}`;
|
|
1380
|
-
}
|
|
1381
|
-
}).join(" AND ");
|
|
1382
|
-
this.logger.debug("where clause generated: " + filterConditions);
|
|
1383
|
-
query.where(filterConditions);
|
|
1384
|
-
}
|
|
1385
|
-
const result = await query.limit(1).toArray();
|
|
1386
|
-
if (result.length === 0) {
|
|
1387
|
-
this.logger.debug("No record found");
|
|
1388
|
-
return null;
|
|
1389
|
-
}
|
|
1390
|
-
return processResultWithTypeConversion(result[0], tableSchema);
|
|
1222
|
+
const record = {
|
|
1223
|
+
...resource,
|
|
1224
|
+
metadata: resource.metadata ? JSON.stringify(resource.metadata) : "",
|
|
1225
|
+
createdAt: resource.createdAt.getTime(),
|
|
1226
|
+
// Store as timestamp (milliseconds)
|
|
1227
|
+
updatedAt: resource.updatedAt.getTime()
|
|
1228
|
+
// Store as timestamp (milliseconds)
|
|
1229
|
+
};
|
|
1230
|
+
const table = await this.client.openTable(TABLE_RESOURCES);
|
|
1231
|
+
await table.add([record], { mode: "append" });
|
|
1232
|
+
return resource;
|
|
1391
1233
|
} catch (error) {
|
|
1392
|
-
if (error instanceof MastraError) throw error;
|
|
1393
1234
|
throw new MastraError(
|
|
1394
1235
|
{
|
|
1395
|
-
id: "
|
|
1236
|
+
id: createStorageErrorId("LANCE", "SAVE_RESOURCE", "FAILED"),
|
|
1396
1237
|
domain: ErrorDomain.STORAGE,
|
|
1397
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1398
|
-
details: { tableName, keyCount: Object.keys(keys).length, firstKey: Object.keys(keys)[0] ?? "" }
|
|
1238
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1399
1239
|
},
|
|
1400
1240
|
error
|
|
1401
1241
|
);
|
|
1402
1242
|
}
|
|
1403
1243
|
}
|
|
1244
|
+
async updateResource({
|
|
1245
|
+
resourceId,
|
|
1246
|
+
workingMemory,
|
|
1247
|
+
metadata
|
|
1248
|
+
}) {
|
|
1249
|
+
const maxRetries = 3;
|
|
1250
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
1251
|
+
try {
|
|
1252
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1253
|
+
if (!existingResource) {
|
|
1254
|
+
const newResource = {
|
|
1255
|
+
id: resourceId,
|
|
1256
|
+
workingMemory,
|
|
1257
|
+
metadata: metadata || {},
|
|
1258
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1259
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1260
|
+
};
|
|
1261
|
+
return this.saveResource({ resource: newResource });
|
|
1262
|
+
}
|
|
1263
|
+
const updatedResource = {
|
|
1264
|
+
...existingResource,
|
|
1265
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1266
|
+
metadata: {
|
|
1267
|
+
...existingResource.metadata,
|
|
1268
|
+
...metadata
|
|
1269
|
+
},
|
|
1270
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1271
|
+
};
|
|
1272
|
+
const record = {
|
|
1273
|
+
id: resourceId,
|
|
1274
|
+
workingMemory: updatedResource.workingMemory || "",
|
|
1275
|
+
metadata: updatedResource.metadata ? JSON.stringify(updatedResource.metadata) : "",
|
|
1276
|
+
updatedAt: updatedResource.updatedAt.getTime()
|
|
1277
|
+
// Store as timestamp (milliseconds)
|
|
1278
|
+
};
|
|
1279
|
+
const table = await this.client.openTable(TABLE_RESOURCES);
|
|
1280
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
1281
|
+
return updatedResource;
|
|
1282
|
+
} catch (error) {
|
|
1283
|
+
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
1284
|
+
const delay = Math.pow(2, attempt) * 10;
|
|
1285
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1286
|
+
continue;
|
|
1287
|
+
}
|
|
1288
|
+
throw new MastraError(
|
|
1289
|
+
{
|
|
1290
|
+
id: createStorageErrorId("LANCE", "UPDATE_RESOURCE", "FAILED"),
|
|
1291
|
+
domain: ErrorDomain.STORAGE,
|
|
1292
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1293
|
+
},
|
|
1294
|
+
error
|
|
1295
|
+
);
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
throw new Error("Unexpected end of retry loop");
|
|
1299
|
+
}
|
|
1404
1300
|
};
|
|
1405
1301
|
var StoreScoresLance = class extends ScoresStorage {
|
|
1406
1302
|
client;
|
|
1407
|
-
|
|
1303
|
+
#db;
|
|
1304
|
+
constructor(config) {
|
|
1408
1305
|
super();
|
|
1306
|
+
const client = resolveLanceConfig(config);
|
|
1409
1307
|
this.client = client;
|
|
1308
|
+
this.#db = new LanceDB({ client });
|
|
1309
|
+
}
|
|
1310
|
+
async init() {
|
|
1311
|
+
await this.#db.createTable({ tableName: TABLE_SCORERS, schema: SCORERS_SCHEMA });
|
|
1312
|
+
await this.#db.alterTable({
|
|
1313
|
+
tableName: TABLE_SCORERS,
|
|
1314
|
+
schema: SCORERS_SCHEMA,
|
|
1315
|
+
ifNotExists: ["spanId", "requestContext"]
|
|
1316
|
+
});
|
|
1317
|
+
}
|
|
1318
|
+
async dangerouslyClearAll() {
|
|
1319
|
+
await this.#db.clearTable({ tableName: TABLE_SCORERS });
|
|
1410
1320
|
}
|
|
1411
1321
|
async saveScore(score) {
|
|
1412
1322
|
let validatedScore;
|
|
@@ -1415,37 +1325,47 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1415
1325
|
} catch (error) {
|
|
1416
1326
|
throw new MastraError(
|
|
1417
1327
|
{
|
|
1418
|
-
id: "
|
|
1328
|
+
id: createStorageErrorId("LANCE", "SAVE_SCORE", "VALIDATION_FAILED"),
|
|
1419
1329
|
text: "Failed to save score in LanceStorage",
|
|
1420
1330
|
domain: ErrorDomain.STORAGE,
|
|
1421
|
-
category: ErrorCategory.
|
|
1331
|
+
category: ErrorCategory.USER,
|
|
1332
|
+
details: {
|
|
1333
|
+
scorer: typeof score.scorer?.id === "string" ? score.scorer.id : String(score.scorer?.id ?? "unknown"),
|
|
1334
|
+
entityId: score.entityId ?? "unknown",
|
|
1335
|
+
entityType: score.entityType ?? "unknown",
|
|
1336
|
+
traceId: score.traceId ?? "",
|
|
1337
|
+
spanId: score.spanId ?? ""
|
|
1338
|
+
}
|
|
1422
1339
|
},
|
|
1423
1340
|
error
|
|
1424
1341
|
);
|
|
1425
1342
|
}
|
|
1343
|
+
const id = crypto.randomUUID();
|
|
1344
|
+
const now = /* @__PURE__ */ new Date();
|
|
1426
1345
|
try {
|
|
1427
|
-
const id = crypto.randomUUID();
|
|
1428
1346
|
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1429
1347
|
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1430
1348
|
const allowedFields = new Set(schema.fields.map((f) => f.name));
|
|
1431
1349
|
const filteredScore = {};
|
|
1432
|
-
Object.keys(validatedScore)
|
|
1350
|
+
for (const key of Object.keys(validatedScore)) {
|
|
1433
1351
|
if (allowedFields.has(key)) {
|
|
1434
|
-
filteredScore[key] =
|
|
1352
|
+
filteredScore[key] = validatedScore[key];
|
|
1435
1353
|
}
|
|
1436
|
-
}
|
|
1354
|
+
}
|
|
1437
1355
|
for (const key in filteredScore) {
|
|
1438
1356
|
if (filteredScore[key] !== null && typeof filteredScore[key] === "object" && !(filteredScore[key] instanceof Date)) {
|
|
1439
1357
|
filteredScore[key] = JSON.stringify(filteredScore[key]);
|
|
1440
1358
|
}
|
|
1441
1359
|
}
|
|
1442
1360
|
filteredScore.id = id;
|
|
1361
|
+
filteredScore.createdAt = now;
|
|
1362
|
+
filteredScore.updatedAt = now;
|
|
1443
1363
|
await table.add([filteredScore], { mode: "append" });
|
|
1444
|
-
return { score };
|
|
1364
|
+
return { score: { ...validatedScore, id, createdAt: now, updatedAt: now } };
|
|
1445
1365
|
} catch (error) {
|
|
1446
1366
|
throw new MastraError(
|
|
1447
1367
|
{
|
|
1448
|
-
id: "
|
|
1368
|
+
id: createStorageErrorId("LANCE", "SAVE_SCORE", "FAILED"),
|
|
1449
1369
|
text: "Failed to save score in LanceStorage",
|
|
1450
1370
|
domain: ErrorDomain.STORAGE,
|
|
1451
1371
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1461,12 +1381,11 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1461
1381
|
const query = table.query().where(`id = '${id}'`).limit(1);
|
|
1462
1382
|
const records = await query.toArray();
|
|
1463
1383
|
if (records.length === 0) return null;
|
|
1464
|
-
|
|
1465
|
-
return processResultWithTypeConversion(records[0], schema);
|
|
1384
|
+
return await this.transformScoreRow(records[0]);
|
|
1466
1385
|
} catch (error) {
|
|
1467
1386
|
throw new MastraError(
|
|
1468
1387
|
{
|
|
1469
|
-
id: "
|
|
1388
|
+
id: createStorageErrorId("LANCE", "GET_SCORE_BY_ID", "FAILED"),
|
|
1470
1389
|
text: "Failed to get score by id in LanceStorage",
|
|
1471
1390
|
domain: ErrorDomain.STORAGE,
|
|
1472
1391
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1476,7 +1395,23 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1476
1395
|
);
|
|
1477
1396
|
}
|
|
1478
1397
|
}
|
|
1479
|
-
|
|
1398
|
+
/**
|
|
1399
|
+
* LanceDB-specific score row transformation.
|
|
1400
|
+
*
|
|
1401
|
+
* Note: This implementation does NOT use coreTransformScoreRow because:
|
|
1402
|
+
* 1. LanceDB stores schema information in the table itself (requires async fetch)
|
|
1403
|
+
* 2. Uses processResultWithTypeConversion utility for LanceDB-specific type handling
|
|
1404
|
+
*/
|
|
1405
|
+
async transformScoreRow(row) {
|
|
1406
|
+
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1407
|
+
const transformed = processResultWithTypeConversion(row, schema);
|
|
1408
|
+
return {
|
|
1409
|
+
...transformed,
|
|
1410
|
+
createdAt: row.createdAt,
|
|
1411
|
+
updatedAt: row.updatedAt
|
|
1412
|
+
};
|
|
1413
|
+
}
|
|
1414
|
+
async listScoresByScorerId({
|
|
1480
1415
|
scorerId,
|
|
1481
1416
|
pagination,
|
|
1482
1417
|
entityId,
|
|
@@ -1484,9 +1419,10 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1484
1419
|
source
|
|
1485
1420
|
}) {
|
|
1486
1421
|
try {
|
|
1422
|
+
const { page, perPage: perPageInput } = pagination;
|
|
1423
|
+
const perPage = normalizePerPage(perPageInput, 100);
|
|
1424
|
+
const { offset: start, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
1487
1425
|
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1488
|
-
const { page = 0, perPage = 10 } = pagination || {};
|
|
1489
|
-
const offset = page * perPage;
|
|
1490
1426
|
let query = table.query().where(`\`scorerId\` = '${scorerId}'`);
|
|
1491
1427
|
if (source) {
|
|
1492
1428
|
query = query.where(`\`source\` = '${source}'`);
|
|
@@ -1497,30 +1433,38 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1497
1433
|
if (entityType) {
|
|
1498
1434
|
query = query.where(`\`entityType\` = '${entityType}'`);
|
|
1499
1435
|
}
|
|
1500
|
-
query = query.limit(perPage);
|
|
1501
|
-
if (offset > 0) query.offset(offset);
|
|
1502
|
-
const records = await query.toArray();
|
|
1503
|
-
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1504
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1505
1436
|
let totalQuery = table.query().where(`\`scorerId\` = '${scorerId}'`);
|
|
1506
1437
|
if (source) {
|
|
1507
1438
|
totalQuery = totalQuery.where(`\`source\` = '${source}'`);
|
|
1508
1439
|
}
|
|
1440
|
+
if (entityId) {
|
|
1441
|
+
totalQuery = totalQuery.where(`\`entityId\` = '${entityId}'`);
|
|
1442
|
+
}
|
|
1443
|
+
if (entityType) {
|
|
1444
|
+
totalQuery = totalQuery.where(`\`entityType\` = '${entityType}'`);
|
|
1445
|
+
}
|
|
1509
1446
|
const allRecords = await totalQuery.toArray();
|
|
1510
1447
|
const total = allRecords.length;
|
|
1448
|
+
const end = perPageInput === false ? total : start + perPage;
|
|
1449
|
+
if (perPageInput !== false) {
|
|
1450
|
+
query = query.limit(perPage);
|
|
1451
|
+
if (start > 0) query = query.offset(start);
|
|
1452
|
+
}
|
|
1453
|
+
const records = await query.toArray();
|
|
1454
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1511
1455
|
return {
|
|
1512
1456
|
pagination: {
|
|
1513
1457
|
page,
|
|
1514
|
-
perPage,
|
|
1458
|
+
perPage: perPageForResponse,
|
|
1515
1459
|
total,
|
|
1516
|
-
hasMore:
|
|
1460
|
+
hasMore: end < total
|
|
1517
1461
|
},
|
|
1518
1462
|
scores
|
|
1519
1463
|
};
|
|
1520
1464
|
} catch (error) {
|
|
1521
1465
|
throw new MastraError(
|
|
1522
1466
|
{
|
|
1523
|
-
id: "
|
|
1467
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_SCORER_ID", "FAILED"),
|
|
1524
1468
|
text: "Failed to get scores by scorerId in LanceStorage",
|
|
1525
1469
|
domain: ErrorDomain.STORAGE,
|
|
1526
1470
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1530,34 +1474,38 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1530
1474
|
);
|
|
1531
1475
|
}
|
|
1532
1476
|
}
|
|
1533
|
-
async
|
|
1477
|
+
async listScoresByRunId({
|
|
1534
1478
|
runId,
|
|
1535
1479
|
pagination
|
|
1536
1480
|
}) {
|
|
1537
1481
|
try {
|
|
1482
|
+
const { page, perPage: perPageInput } = pagination;
|
|
1483
|
+
const perPage = normalizePerPage(perPageInput, 100);
|
|
1484
|
+
const { offset: start, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
1538
1485
|
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1539
|
-
const { page = 0, perPage = 10 } = pagination || {};
|
|
1540
|
-
const offset = page * perPage;
|
|
1541
|
-
const query = table.query().where(`\`runId\` = '${runId}'`).limit(perPage);
|
|
1542
|
-
if (offset > 0) query.offset(offset);
|
|
1543
|
-
const records = await query.toArray();
|
|
1544
|
-
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1545
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1546
1486
|
const allRecords = await table.query().where(`\`runId\` = '${runId}'`).toArray();
|
|
1547
1487
|
const total = allRecords.length;
|
|
1488
|
+
const end = perPageInput === false ? total : start + perPage;
|
|
1489
|
+
let query = table.query().where(`\`runId\` = '${runId}'`);
|
|
1490
|
+
if (perPageInput !== false) {
|
|
1491
|
+
query = query.limit(perPage);
|
|
1492
|
+
if (start > 0) query = query.offset(start);
|
|
1493
|
+
}
|
|
1494
|
+
const records = await query.toArray();
|
|
1495
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1548
1496
|
return {
|
|
1549
1497
|
pagination: {
|
|
1550
1498
|
page,
|
|
1551
|
-
perPage,
|
|
1499
|
+
perPage: perPageForResponse,
|
|
1552
1500
|
total,
|
|
1553
|
-
hasMore:
|
|
1501
|
+
hasMore: end < total
|
|
1554
1502
|
},
|
|
1555
1503
|
scores
|
|
1556
1504
|
};
|
|
1557
1505
|
} catch (error) {
|
|
1558
1506
|
throw new MastraError(
|
|
1559
1507
|
{
|
|
1560
|
-
id: "
|
|
1508
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_RUN_ID", "FAILED"),
|
|
1561
1509
|
text: "Failed to get scores by runId in LanceStorage",
|
|
1562
1510
|
domain: ErrorDomain.STORAGE,
|
|
1563
1511
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1567,35 +1515,39 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1567
1515
|
);
|
|
1568
1516
|
}
|
|
1569
1517
|
}
|
|
1570
|
-
async
|
|
1518
|
+
async listScoresByEntityId({
|
|
1571
1519
|
entityId,
|
|
1572
1520
|
entityType,
|
|
1573
1521
|
pagination
|
|
1574
1522
|
}) {
|
|
1575
1523
|
try {
|
|
1524
|
+
const { page, perPage: perPageInput } = pagination;
|
|
1525
|
+
const perPage = normalizePerPage(perPageInput, 100);
|
|
1526
|
+
const { offset: start, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
1576
1527
|
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1577
|
-
const { page = 0, perPage = 10 } = pagination || {};
|
|
1578
|
-
const offset = page * perPage;
|
|
1579
|
-
const query = table.query().where(`\`entityId\` = '${entityId}' AND \`entityType\` = '${entityType}'`).limit(perPage);
|
|
1580
|
-
if (offset > 0) query.offset(offset);
|
|
1581
|
-
const records = await query.toArray();
|
|
1582
|
-
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1583
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1584
1528
|
const allRecords = await table.query().where(`\`entityId\` = '${entityId}' AND \`entityType\` = '${entityType}'`).toArray();
|
|
1585
1529
|
const total = allRecords.length;
|
|
1530
|
+
const end = perPageInput === false ? total : start + perPage;
|
|
1531
|
+
let query = table.query().where(`\`entityId\` = '${entityId}' AND \`entityType\` = '${entityType}'`);
|
|
1532
|
+
if (perPageInput !== false) {
|
|
1533
|
+
query = query.limit(perPage);
|
|
1534
|
+
if (start > 0) query = query.offset(start);
|
|
1535
|
+
}
|
|
1536
|
+
const records = await query.toArray();
|
|
1537
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1586
1538
|
return {
|
|
1587
1539
|
pagination: {
|
|
1588
1540
|
page,
|
|
1589
|
-
perPage,
|
|
1541
|
+
perPage: perPageForResponse,
|
|
1590
1542
|
total,
|
|
1591
|
-
hasMore:
|
|
1543
|
+
hasMore: end < total
|
|
1592
1544
|
},
|
|
1593
1545
|
scores
|
|
1594
1546
|
};
|
|
1595
1547
|
} catch (error) {
|
|
1596
1548
|
throw new MastraError(
|
|
1597
1549
|
{
|
|
1598
|
-
id: "
|
|
1550
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_ENTITY_ID", "FAILED"),
|
|
1599
1551
|
text: "Failed to get scores by entityId and entityType in LanceStorage",
|
|
1600
1552
|
domain: ErrorDomain.STORAGE,
|
|
1601
1553
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1605,35 +1557,39 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1605
1557
|
);
|
|
1606
1558
|
}
|
|
1607
1559
|
}
|
|
1608
|
-
async
|
|
1560
|
+
async listScoresBySpan({
|
|
1609
1561
|
traceId,
|
|
1610
1562
|
spanId,
|
|
1611
1563
|
pagination
|
|
1612
1564
|
}) {
|
|
1613
1565
|
try {
|
|
1566
|
+
const { page, perPage: perPageInput } = pagination;
|
|
1567
|
+
const perPage = normalizePerPage(perPageInput, 100);
|
|
1568
|
+
const { offset: start, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
1614
1569
|
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1615
|
-
const { page = 0, perPage = 10 } = pagination || {};
|
|
1616
|
-
const offset = page * perPage;
|
|
1617
|
-
const query = table.query().where(`\`traceId\` = '${traceId}' AND \`spanId\` = '${spanId}'`).limit(perPage);
|
|
1618
|
-
if (offset > 0) query.offset(offset);
|
|
1619
|
-
const records = await query.toArray();
|
|
1620
|
-
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1621
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1622
1570
|
const allRecords = await table.query().where(`\`traceId\` = '${traceId}' AND \`spanId\` = '${spanId}'`).toArray();
|
|
1623
1571
|
const total = allRecords.length;
|
|
1572
|
+
const end = perPageInput === false ? total : start + perPage;
|
|
1573
|
+
let query = table.query().where(`\`traceId\` = '${traceId}' AND \`spanId\` = '${spanId}'`);
|
|
1574
|
+
if (perPageInput !== false) {
|
|
1575
|
+
query = query.limit(perPage);
|
|
1576
|
+
if (start > 0) query = query.offset(start);
|
|
1577
|
+
}
|
|
1578
|
+
const records = await query.toArray();
|
|
1579
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1624
1580
|
return {
|
|
1625
1581
|
pagination: {
|
|
1626
1582
|
page,
|
|
1627
|
-
perPage,
|
|
1583
|
+
perPage: perPageForResponse,
|
|
1628
1584
|
total,
|
|
1629
|
-
hasMore:
|
|
1585
|
+
hasMore: end < total
|
|
1630
1586
|
},
|
|
1631
1587
|
scores
|
|
1632
1588
|
};
|
|
1633
1589
|
} catch (error) {
|
|
1634
1590
|
throw new MastraError(
|
|
1635
1591
|
{
|
|
1636
|
-
id: "
|
|
1592
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_SPAN", "FAILED"),
|
|
1637
1593
|
text: "Failed to get scores by traceId and spanId in LanceStorage",
|
|
1638
1594
|
domain: ErrorDomain.STORAGE,
|
|
1639
1595
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1644,6 +1600,9 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1644
1600
|
}
|
|
1645
1601
|
}
|
|
1646
1602
|
};
|
|
1603
|
+
function escapeSql(str) {
|
|
1604
|
+
return str.replace(/'/g, "''");
|
|
1605
|
+
}
|
|
1647
1606
|
function parseWorkflowRun(row) {
|
|
1648
1607
|
let parsedSnapshot = row.snapshot;
|
|
1649
1608
|
if (typeof parsedSnapshot === "string") {
|
|
@@ -1664,56 +1623,104 @@ function parseWorkflowRun(row) {
|
|
|
1664
1623
|
}
|
|
1665
1624
|
var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
1666
1625
|
client;
|
|
1667
|
-
|
|
1626
|
+
#db;
|
|
1627
|
+
constructor(config) {
|
|
1668
1628
|
super();
|
|
1629
|
+
const client = resolveLanceConfig(config);
|
|
1669
1630
|
this.client = client;
|
|
1631
|
+
this.#db = new LanceDB({ client });
|
|
1632
|
+
}
|
|
1633
|
+
async init() {
|
|
1634
|
+
const schema = TABLE_SCHEMAS[TABLE_WORKFLOW_SNAPSHOT];
|
|
1635
|
+
await this.#db.createTable({ tableName: TABLE_WORKFLOW_SNAPSHOT, schema });
|
|
1636
|
+
await this.#db.alterTable({
|
|
1637
|
+
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
1638
|
+
schema,
|
|
1639
|
+
ifNotExists: ["resourceId"]
|
|
1640
|
+
});
|
|
1641
|
+
}
|
|
1642
|
+
async dangerouslyClearAll() {
|
|
1643
|
+
await this.#db.clearTable({ tableName: TABLE_WORKFLOW_SNAPSHOT });
|
|
1670
1644
|
}
|
|
1671
|
-
updateWorkflowResults({
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1645
|
+
async updateWorkflowResults({
|
|
1646
|
+
workflowName,
|
|
1647
|
+
runId,
|
|
1648
|
+
stepId,
|
|
1649
|
+
result,
|
|
1650
|
+
requestContext
|
|
1677
1651
|
}) {
|
|
1678
|
-
|
|
1652
|
+
let snapshot = await this.loadWorkflowSnapshot({ workflowName, runId });
|
|
1653
|
+
if (!snapshot) {
|
|
1654
|
+
snapshot = {
|
|
1655
|
+
context: {},
|
|
1656
|
+
activePaths: [],
|
|
1657
|
+
timestamp: Date.now(),
|
|
1658
|
+
suspendedPaths: {},
|
|
1659
|
+
activeStepsPath: {},
|
|
1660
|
+
resumeLabels: {},
|
|
1661
|
+
serializedStepGraph: [],
|
|
1662
|
+
status: "pending",
|
|
1663
|
+
value: {},
|
|
1664
|
+
waitingPaths: {},
|
|
1665
|
+
runId,
|
|
1666
|
+
requestContext: {}
|
|
1667
|
+
};
|
|
1668
|
+
}
|
|
1669
|
+
snapshot.context[stepId] = result;
|
|
1670
|
+
snapshot.requestContext = { ...snapshot.requestContext, ...requestContext };
|
|
1671
|
+
await this.persistWorkflowSnapshot({ workflowName, runId, snapshot });
|
|
1672
|
+
return snapshot.context;
|
|
1679
1673
|
}
|
|
1680
|
-
updateWorkflowState({
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1674
|
+
async updateWorkflowState({
|
|
1675
|
+
workflowName,
|
|
1676
|
+
runId,
|
|
1677
|
+
opts
|
|
1684
1678
|
}) {
|
|
1685
|
-
|
|
1679
|
+
const snapshot = await this.loadWorkflowSnapshot({ workflowName, runId });
|
|
1680
|
+
if (!snapshot) {
|
|
1681
|
+
return void 0;
|
|
1682
|
+
}
|
|
1683
|
+
if (!snapshot.context) {
|
|
1684
|
+
throw new Error(`Snapshot not found for runId ${runId}`);
|
|
1685
|
+
}
|
|
1686
|
+
const updatedSnapshot = { ...snapshot, ...opts };
|
|
1687
|
+
await this.persistWorkflowSnapshot({ workflowName, runId, snapshot: updatedSnapshot });
|
|
1688
|
+
return updatedSnapshot;
|
|
1686
1689
|
}
|
|
1687
1690
|
async persistWorkflowSnapshot({
|
|
1688
1691
|
workflowName,
|
|
1689
1692
|
runId,
|
|
1690
1693
|
resourceId,
|
|
1691
|
-
snapshot
|
|
1694
|
+
snapshot,
|
|
1695
|
+
createdAt,
|
|
1696
|
+
updatedAt
|
|
1692
1697
|
}) {
|
|
1693
1698
|
try {
|
|
1694
1699
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1695
|
-
const query = table.query().where(`workflow_name = '${workflowName}' AND run_id = '${runId}'`);
|
|
1700
|
+
const query = table.query().where(`workflow_name = '${escapeSql(workflowName)}' AND run_id = '${escapeSql(runId)}'`);
|
|
1696
1701
|
const records = await query.toArray();
|
|
1697
|
-
let
|
|
1698
|
-
const now = Date.now();
|
|
1702
|
+
let createdAtValue;
|
|
1703
|
+
const now = createdAt?.getTime() ?? Date.now();
|
|
1699
1704
|
if (records.length > 0) {
|
|
1700
|
-
|
|
1705
|
+
createdAtValue = records[0].createdAt ?? now;
|
|
1701
1706
|
} else {
|
|
1702
|
-
|
|
1707
|
+
createdAtValue = now;
|
|
1703
1708
|
}
|
|
1709
|
+
const { status, value, ...rest } = snapshot;
|
|
1704
1710
|
const record = {
|
|
1705
1711
|
workflow_name: workflowName,
|
|
1706
1712
|
run_id: runId,
|
|
1707
1713
|
resourceId,
|
|
1708
|
-
snapshot: JSON.stringify(
|
|
1709
|
-
|
|
1710
|
-
|
|
1714
|
+
snapshot: JSON.stringify({ status, value, ...rest }),
|
|
1715
|
+
// this is to ensure status is always just before value, for when querying the db by status
|
|
1716
|
+
createdAt: createdAtValue,
|
|
1717
|
+
updatedAt: updatedAt ?? now
|
|
1711
1718
|
};
|
|
1712
1719
|
await table.mergeInsert(["workflow_name", "run_id"]).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
1713
1720
|
} catch (error) {
|
|
1714
1721
|
throw new MastraError(
|
|
1715
1722
|
{
|
|
1716
|
-
id: "
|
|
1723
|
+
id: createStorageErrorId("LANCE", "PERSIST_WORKFLOW_SNAPSHOT", "FAILED"),
|
|
1717
1724
|
domain: ErrorDomain.STORAGE,
|
|
1718
1725
|
category: ErrorCategory.THIRD_PARTY,
|
|
1719
1726
|
details: { workflowName, runId }
|
|
@@ -1728,13 +1735,13 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1728
1735
|
}) {
|
|
1729
1736
|
try {
|
|
1730
1737
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1731
|
-
const query = table.query().where(`workflow_name = '${workflowName}' AND run_id = '${runId}'`);
|
|
1738
|
+
const query = table.query().where(`workflow_name = '${escapeSql(workflowName)}' AND run_id = '${escapeSql(runId)}'`);
|
|
1732
1739
|
const records = await query.toArray();
|
|
1733
1740
|
return records.length > 0 ? JSON.parse(records[0].snapshot) : null;
|
|
1734
1741
|
} catch (error) {
|
|
1735
1742
|
throw new MastraError(
|
|
1736
1743
|
{
|
|
1737
|
-
id: "
|
|
1744
|
+
id: createStorageErrorId("LANCE", "LOAD_WORKFLOW_SNAPSHOT", "FAILED"),
|
|
1738
1745
|
domain: ErrorDomain.STORAGE,
|
|
1739
1746
|
category: ErrorCategory.THIRD_PARTY,
|
|
1740
1747
|
details: { workflowName, runId }
|
|
@@ -1746,9 +1753,9 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1746
1753
|
async getWorkflowRunById(args) {
|
|
1747
1754
|
try {
|
|
1748
1755
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1749
|
-
let whereClause = `run_id = '${args.runId}'`;
|
|
1756
|
+
let whereClause = `run_id = '${escapeSql(args.runId)}'`;
|
|
1750
1757
|
if (args.workflowName) {
|
|
1751
|
-
whereClause += ` AND workflow_name = '${args.workflowName}'`;
|
|
1758
|
+
whereClause += ` AND workflow_name = '${escapeSql(args.workflowName)}'`;
|
|
1752
1759
|
}
|
|
1753
1760
|
const query = table.query().where(whereClause);
|
|
1754
1761
|
const records = await query.toArray();
|
|
@@ -1758,7 +1765,7 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1758
1765
|
} catch (error) {
|
|
1759
1766
|
throw new MastraError(
|
|
1760
1767
|
{
|
|
1761
|
-
id: "
|
|
1768
|
+
id: createStorageErrorId("LANCE", "GET_WORKFLOW_RUN_BY_ID", "FAILED"),
|
|
1762
1769
|
domain: ErrorDomain.STORAGE,
|
|
1763
1770
|
category: ErrorCategory.THIRD_PARTY,
|
|
1764
1771
|
details: { runId: args.runId, workflowName: args.workflowName ?? "" }
|
|
@@ -1767,16 +1774,37 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1767
1774
|
);
|
|
1768
1775
|
}
|
|
1769
1776
|
}
|
|
1777
|
+
async deleteWorkflowRunById({ runId, workflowName }) {
|
|
1778
|
+
try {
|
|
1779
|
+
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1780
|
+
const whereClause = `run_id = '${escapeSql(runId)}' AND workflow_name = '${escapeSql(workflowName)}'`;
|
|
1781
|
+
await table.delete(whereClause);
|
|
1782
|
+
} catch (error) {
|
|
1783
|
+
throw new MastraError(
|
|
1784
|
+
{
|
|
1785
|
+
id: createStorageErrorId("LANCE", "DELETE_WORKFLOW_RUN_BY_ID", "FAILED"),
|
|
1786
|
+
domain: ErrorDomain.STORAGE,
|
|
1787
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1788
|
+
details: { runId, workflowName }
|
|
1789
|
+
},
|
|
1790
|
+
error
|
|
1791
|
+
);
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1770
1794
|
async listWorkflowRuns(args) {
|
|
1771
1795
|
try {
|
|
1772
1796
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1773
1797
|
let query = table.query();
|
|
1774
1798
|
const conditions = [];
|
|
1775
1799
|
if (args?.workflowName) {
|
|
1776
|
-
conditions.push(`workflow_name = '${args.workflowName
|
|
1800
|
+
conditions.push(`workflow_name = '${escapeSql(args.workflowName)}'`);
|
|
1801
|
+
}
|
|
1802
|
+
if (args?.status) {
|
|
1803
|
+
const escapedStatus = args.status.replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/%/g, "\\%").replace(/_/g, "\\_");
|
|
1804
|
+
conditions.push(`\`snapshot\` LIKE '%"status":"${escapedStatus}","value"%'`);
|
|
1777
1805
|
}
|
|
1778
1806
|
if (args?.resourceId) {
|
|
1779
|
-
conditions.push(`\`resourceId\` = '${args.resourceId}'`);
|
|
1807
|
+
conditions.push(`\`resourceId\` = '${escapeSql(args.resourceId)}'`);
|
|
1780
1808
|
}
|
|
1781
1809
|
if (args?.fromDate instanceof Date) {
|
|
1782
1810
|
conditions.push(`\`createdAt\` >= ${args.fromDate.getTime()}`);
|
|
@@ -1791,11 +1819,22 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1791
1819
|
} else {
|
|
1792
1820
|
total = await table.countRows();
|
|
1793
1821
|
}
|
|
1794
|
-
if (args?.
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1822
|
+
if (args?.perPage !== void 0 && args?.page !== void 0) {
|
|
1823
|
+
const normalizedPerPage = normalizePerPage(args.perPage, Number.MAX_SAFE_INTEGER);
|
|
1824
|
+
if (args.page < 0 || !Number.isInteger(args.page)) {
|
|
1825
|
+
throw new MastraError(
|
|
1826
|
+
{
|
|
1827
|
+
id: createStorageErrorId("LANCE", "LIST_WORKFLOW_RUNS", "INVALID_PAGINATION"),
|
|
1828
|
+
domain: ErrorDomain.STORAGE,
|
|
1829
|
+
category: ErrorCategory.USER,
|
|
1830
|
+
details: { page: args.page, perPage: args.perPage }
|
|
1831
|
+
},
|
|
1832
|
+
new Error(`Invalid pagination parameters: page=${args.page}, perPage=${args.perPage}`)
|
|
1833
|
+
);
|
|
1834
|
+
}
|
|
1835
|
+
const offset = args.page * normalizedPerPage;
|
|
1836
|
+
query.limit(normalizedPerPage);
|
|
1837
|
+
query.offset(offset);
|
|
1799
1838
|
}
|
|
1800
1839
|
const records = await query.toArray();
|
|
1801
1840
|
return {
|
|
@@ -1805,7 +1844,7 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1805
1844
|
} catch (error) {
|
|
1806
1845
|
throw new MastraError(
|
|
1807
1846
|
{
|
|
1808
|
-
id: "
|
|
1847
|
+
id: createStorageErrorId("LANCE", "LIST_WORKFLOW_RUNS", "FAILED"),
|
|
1809
1848
|
domain: ErrorDomain.STORAGE,
|
|
1810
1849
|
category: ErrorCategory.THIRD_PARTY,
|
|
1811
1850
|
details: { resourceId: args?.resourceId ?? "", workflowName: args?.workflowName ?? "" }
|
|
@@ -1822,279 +1861,95 @@ var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
|
1822
1861
|
lanceClient;
|
|
1823
1862
|
/**
|
|
1824
1863
|
* Creates a new instance of LanceStorage
|
|
1864
|
+
* @param id The unique identifier for this storage instance
|
|
1865
|
+
* @param name The name for this storage instance
|
|
1825
1866
|
* @param uri The URI to connect to LanceDB
|
|
1826
|
-
* @param
|
|
1867
|
+
* @param connectionOptions connection options for LanceDB
|
|
1868
|
+
* @param storageOptions storage options including disableInit
|
|
1827
1869
|
*
|
|
1828
1870
|
* Usage:
|
|
1829
1871
|
*
|
|
1830
1872
|
* Connect to a local database
|
|
1831
1873
|
* ```ts
|
|
1832
|
-
* const store = await LanceStorage.create('/path/to/db');
|
|
1874
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', '/path/to/db');
|
|
1833
1875
|
* ```
|
|
1834
1876
|
*
|
|
1835
1877
|
* Connect to a LanceDB cloud database
|
|
1836
1878
|
* ```ts
|
|
1837
|
-
* const store = await LanceStorage.create('db://host:port');
|
|
1879
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 'db://host:port');
|
|
1838
1880
|
* ```
|
|
1839
1881
|
*
|
|
1840
1882
|
* Connect to a cloud database
|
|
1841
1883
|
* ```ts
|
|
1842
|
-
* const store = await LanceStorage.create('s3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
1884
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 's3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
1885
|
+
* ```
|
|
1886
|
+
*
|
|
1887
|
+
* Disable auto-init for runtime (after CI/CD has run migrations)
|
|
1888
|
+
* ```ts
|
|
1889
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', '/path/to/db', undefined, { disableInit: true });
|
|
1843
1890
|
* ```
|
|
1844
1891
|
*/
|
|
1845
|
-
static async create(name, uri,
|
|
1846
|
-
const instance = new _LanceStorage(name);
|
|
1892
|
+
static async create(id, name, uri, connectionOptions, storageOptions) {
|
|
1893
|
+
const instance = new _LanceStorage(id, name, storageOptions?.disableInit);
|
|
1847
1894
|
try {
|
|
1848
|
-
instance.lanceClient = await connect(uri,
|
|
1849
|
-
const operations = new StoreOperationsLance({ client: instance.lanceClient });
|
|
1895
|
+
instance.lanceClient = await connect(uri, connectionOptions);
|
|
1850
1896
|
instance.stores = {
|
|
1851
|
-
operations: new StoreOperationsLance({ client: instance.lanceClient }),
|
|
1852
1897
|
workflows: new StoreWorkflowsLance({ client: instance.lanceClient }),
|
|
1853
|
-
scores: new StoreScoresLance({ client: instance.lanceClient }),
|
|
1854
|
-
memory: new StoreMemoryLance({ client: instance.lanceClient
|
|
1855
|
-
};
|
|
1856
|
-
return instance;
|
|
1857
|
-
} catch (e) {
|
|
1858
|
-
throw new MastraError(
|
|
1859
|
-
{
|
|
1860
|
-
id: "
|
|
1861
|
-
domain: ErrorDomain.STORAGE,
|
|
1862
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
1863
|
-
text: `Failed to connect to LanceDB: ${e.message || e}`,
|
|
1864
|
-
details: { uri, optionsProvided: !!
|
|
1865
|
-
},
|
|
1866
|
-
e
|
|
1867
|
-
);
|
|
1868
|
-
}
|
|
1869
|
-
}
|
|
1870
|
-
/**
|
|
1871
|
-
* @internal
|
|
1872
|
-
* Private constructor to enforce using the create factory method
|
|
1873
|
-
*/
|
|
1874
|
-
constructor(name) {
|
|
1875
|
-
super({ name });
|
|
1876
|
-
const operations = new StoreOperationsLance({ client: this.lanceClient });
|
|
1877
|
-
this.stores = {
|
|
1878
|
-
operations: new StoreOperationsLance({ client: this.lanceClient }),
|
|
1879
|
-
workflows: new StoreWorkflowsLance({ client: this.lanceClient }),
|
|
1880
|
-
scores: new StoreScoresLance({ client: this.lanceClient }),
|
|
1881
|
-
memory: new StoreMemoryLance({ client: this.lanceClient, operations })
|
|
1882
|
-
};
|
|
1883
|
-
}
|
|
1884
|
-
async createTable({
|
|
1885
|
-
tableName,
|
|
1886
|
-
schema
|
|
1887
|
-
}) {
|
|
1888
|
-
return this.stores.operations.createTable({ tableName, schema });
|
|
1889
|
-
}
|
|
1890
|
-
async dropTable({ tableName }) {
|
|
1891
|
-
return this.stores.operations.dropTable({ tableName });
|
|
1892
|
-
}
|
|
1893
|
-
async alterTable({
|
|
1894
|
-
tableName,
|
|
1895
|
-
schema,
|
|
1896
|
-
ifNotExists
|
|
1897
|
-
}) {
|
|
1898
|
-
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
1899
|
-
}
|
|
1900
|
-
async clearTable({ tableName }) {
|
|
1901
|
-
return this.stores.operations.clearTable({ tableName });
|
|
1902
|
-
}
|
|
1903
|
-
async insert({ tableName, record }) {
|
|
1904
|
-
return this.stores.operations.insert({ tableName, record });
|
|
1905
|
-
}
|
|
1906
|
-
async batchInsert({ tableName, records }) {
|
|
1907
|
-
return this.stores.operations.batchInsert({ tableName, records });
|
|
1908
|
-
}
|
|
1909
|
-
async load({ tableName, keys }) {
|
|
1910
|
-
return this.stores.operations.load({ tableName, keys });
|
|
1911
|
-
}
|
|
1912
|
-
async getThreadById({ threadId }) {
|
|
1913
|
-
return this.stores.memory.getThreadById({ threadId });
|
|
1914
|
-
}
|
|
1915
|
-
async getThreadsByResourceId({ resourceId }) {
|
|
1916
|
-
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
1898
|
+
scores: new StoreScoresLance({ client: instance.lanceClient }),
|
|
1899
|
+
memory: new StoreMemoryLance({ client: instance.lanceClient })
|
|
1900
|
+
};
|
|
1901
|
+
return instance;
|
|
1902
|
+
} catch (e) {
|
|
1903
|
+
throw new MastraError(
|
|
1904
|
+
{
|
|
1905
|
+
id: createStorageErrorId("LANCE", "CONNECT", "FAILED"),
|
|
1906
|
+
domain: ErrorDomain.STORAGE,
|
|
1907
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1908
|
+
text: `Failed to connect to LanceDB: ${e.message || e}`,
|
|
1909
|
+
details: { uri, optionsProvided: !!connectionOptions }
|
|
1910
|
+
},
|
|
1911
|
+
e
|
|
1912
|
+
);
|
|
1913
|
+
}
|
|
1917
1914
|
}
|
|
1918
1915
|
/**
|
|
1919
|
-
*
|
|
1920
|
-
*
|
|
1921
|
-
*
|
|
1916
|
+
* Creates a new instance of LanceStorage from a pre-configured LanceDB connection.
|
|
1917
|
+
* Use this when you need to configure the connection before initialization.
|
|
1918
|
+
*
|
|
1919
|
+
* @param id The unique identifier for this storage instance
|
|
1920
|
+
* @param name The name for this storage instance
|
|
1921
|
+
* @param client Pre-configured LanceDB connection
|
|
1922
|
+
* @param options Storage options including disableInit
|
|
1923
|
+
*
|
|
1924
|
+
* @example
|
|
1925
|
+
* ```typescript
|
|
1926
|
+
* import { connect } from '@lancedb/lancedb';
|
|
1927
|
+
*
|
|
1928
|
+
* const client = await connect('/path/to/db', {
|
|
1929
|
+
* // Custom connection options
|
|
1930
|
+
* });
|
|
1931
|
+
*
|
|
1932
|
+
* const store = LanceStorage.fromClient('my-id', 'MyStorage', client);
|
|
1933
|
+
* ```
|
|
1922
1934
|
*/
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
}) {
|
|
1931
|
-
return this.stores.memory.updateThread({ id, title, metadata });
|
|
1932
|
-
}
|
|
1933
|
-
async deleteThread({ threadId }) {
|
|
1934
|
-
return this.stores.memory.deleteThread({ threadId });
|
|
1935
|
-
}
|
|
1936
|
-
get supports() {
|
|
1937
|
-
return {
|
|
1938
|
-
selectByIncludeResourceScope: true,
|
|
1939
|
-
resourceWorkingMemory: true,
|
|
1940
|
-
hasColumn: true,
|
|
1941
|
-
createTable: true,
|
|
1942
|
-
deleteMessages: false,
|
|
1943
|
-
getScoresBySpan: true
|
|
1935
|
+
static fromClient(id, name, client, options) {
|
|
1936
|
+
const instance = new _LanceStorage(id, name, options?.disableInit);
|
|
1937
|
+
instance.lanceClient = client;
|
|
1938
|
+
instance.stores = {
|
|
1939
|
+
workflows: new StoreWorkflowsLance({ client }),
|
|
1940
|
+
scores: new StoreScoresLance({ client }),
|
|
1941
|
+
memory: new StoreMemoryLance({ client })
|
|
1944
1942
|
};
|
|
1945
|
-
|
|
1946
|
-
async getResourceById({ resourceId }) {
|
|
1947
|
-
return this.stores.memory.getResourceById({ resourceId });
|
|
1948
|
-
}
|
|
1949
|
-
async saveResource({ resource }) {
|
|
1950
|
-
return this.stores.memory.saveResource({ resource });
|
|
1951
|
-
}
|
|
1952
|
-
async updateResource({
|
|
1953
|
-
resourceId,
|
|
1954
|
-
workingMemory,
|
|
1955
|
-
metadata
|
|
1956
|
-
}) {
|
|
1957
|
-
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
1943
|
+
return instance;
|
|
1958
1944
|
}
|
|
1959
1945
|
/**
|
|
1960
|
-
*
|
|
1961
|
-
*
|
|
1962
|
-
*
|
|
1963
|
-
* @returns The processed array with context messages included
|
|
1946
|
+
* @internal
|
|
1947
|
+
* Private constructor to enforce using the create factory method.
|
|
1948
|
+
* Note: stores is initialized in create() after the lanceClient is connected.
|
|
1964
1949
|
*/
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
return records;
|
|
1969
|
-
}
|
|
1970
|
-
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
1971
|
-
records.forEach((message, index) => {
|
|
1972
|
-
messageIndexMap.set(message.id, index);
|
|
1973
|
-
});
|
|
1974
|
-
const additionalIndices = /* @__PURE__ */ new Set();
|
|
1975
|
-
for (const item of messagesWithContext) {
|
|
1976
|
-
const messageIndex = messageIndexMap.get(item.id);
|
|
1977
|
-
if (messageIndex !== void 0) {
|
|
1978
|
-
if (item.withPreviousMessages) {
|
|
1979
|
-
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
1980
|
-
for (let i = startIdx; i < messageIndex; i++) {
|
|
1981
|
-
additionalIndices.add(i);
|
|
1982
|
-
}
|
|
1983
|
-
}
|
|
1984
|
-
if (item.withNextMessages) {
|
|
1985
|
-
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
1986
|
-
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
1987
|
-
additionalIndices.add(i);
|
|
1988
|
-
}
|
|
1989
|
-
}
|
|
1990
|
-
}
|
|
1991
|
-
}
|
|
1992
|
-
if (additionalIndices.size === 0) {
|
|
1993
|
-
return records;
|
|
1994
|
-
}
|
|
1995
|
-
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
1996
|
-
const allIndices = /* @__PURE__ */ new Set();
|
|
1997
|
-
records.forEach((record, index) => {
|
|
1998
|
-
if (originalMatchIds.has(record.id)) {
|
|
1999
|
-
allIndices.add(index);
|
|
2000
|
-
}
|
|
2001
|
-
});
|
|
2002
|
-
additionalIndices.forEach((index) => {
|
|
2003
|
-
allIndices.add(index);
|
|
2004
|
-
});
|
|
2005
|
-
return Array.from(allIndices).sort((a, b) => a - b).map((index) => records[index]);
|
|
2006
|
-
}
|
|
2007
|
-
async getMessages({
|
|
2008
|
-
threadId,
|
|
2009
|
-
resourceId,
|
|
2010
|
-
selectBy,
|
|
2011
|
-
format,
|
|
2012
|
-
threadConfig
|
|
2013
|
-
}) {
|
|
2014
|
-
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format, threadConfig });
|
|
2015
|
-
}
|
|
2016
|
-
async saveMessages(args) {
|
|
2017
|
-
return this.stores.memory.saveMessages(args);
|
|
2018
|
-
}
|
|
2019
|
-
async getThreadsByResourceIdPaginated(args) {
|
|
2020
|
-
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2021
|
-
}
|
|
2022
|
-
async getMessagesPaginated(args) {
|
|
2023
|
-
return this.stores.memory.getMessagesPaginated(args);
|
|
2024
|
-
}
|
|
2025
|
-
async updateMessages(_args) {
|
|
2026
|
-
return this.stores.memory.updateMessages(_args);
|
|
2027
|
-
}
|
|
2028
|
-
async listWorkflowRuns(args) {
|
|
2029
|
-
return this.stores.workflows.listWorkflowRuns(args);
|
|
2030
|
-
}
|
|
2031
|
-
async getWorkflowRunById(args) {
|
|
2032
|
-
return this.stores.workflows.getWorkflowRunById(args);
|
|
2033
|
-
}
|
|
2034
|
-
async updateWorkflowResults({
|
|
2035
|
-
workflowName,
|
|
2036
|
-
runId,
|
|
2037
|
-
stepId,
|
|
2038
|
-
result,
|
|
2039
|
-
requestContext
|
|
2040
|
-
}) {
|
|
2041
|
-
return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, requestContext });
|
|
2042
|
-
}
|
|
2043
|
-
async updateWorkflowState({
|
|
2044
|
-
workflowName,
|
|
2045
|
-
runId,
|
|
2046
|
-
opts
|
|
2047
|
-
}) {
|
|
2048
|
-
return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
|
|
2049
|
-
}
|
|
2050
|
-
async persistWorkflowSnapshot({
|
|
2051
|
-
workflowName,
|
|
2052
|
-
runId,
|
|
2053
|
-
resourceId,
|
|
2054
|
-
snapshot
|
|
2055
|
-
}) {
|
|
2056
|
-
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot });
|
|
2057
|
-
}
|
|
2058
|
-
async loadWorkflowSnapshot({
|
|
2059
|
-
workflowName,
|
|
2060
|
-
runId
|
|
2061
|
-
}) {
|
|
2062
|
-
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2063
|
-
}
|
|
2064
|
-
async getScoreById({ id: _id }) {
|
|
2065
|
-
return this.stores.scores.getScoreById({ id: _id });
|
|
2066
|
-
}
|
|
2067
|
-
async getScoresByScorerId({
|
|
2068
|
-
scorerId,
|
|
2069
|
-
source,
|
|
2070
|
-
entityId,
|
|
2071
|
-
entityType,
|
|
2072
|
-
pagination
|
|
2073
|
-
}) {
|
|
2074
|
-
return this.stores.scores.getScoresByScorerId({ scorerId, source, pagination, entityId, entityType });
|
|
2075
|
-
}
|
|
2076
|
-
async saveScore(_score) {
|
|
2077
|
-
return this.stores.scores.saveScore(_score);
|
|
2078
|
-
}
|
|
2079
|
-
async getScoresByRunId({
|
|
2080
|
-
runId,
|
|
2081
|
-
pagination
|
|
2082
|
-
}) {
|
|
2083
|
-
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
2084
|
-
}
|
|
2085
|
-
async getScoresByEntityId({
|
|
2086
|
-
entityId,
|
|
2087
|
-
entityType,
|
|
2088
|
-
pagination
|
|
2089
|
-
}) {
|
|
2090
|
-
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
2091
|
-
}
|
|
2092
|
-
async getScoresBySpan({
|
|
2093
|
-
traceId,
|
|
2094
|
-
spanId,
|
|
2095
|
-
pagination
|
|
2096
|
-
}) {
|
|
2097
|
-
return this.stores.scores.getScoresBySpan({ traceId, spanId, pagination });
|
|
1950
|
+
constructor(id, name, disableInit) {
|
|
1951
|
+
super({ id, name, disableInit });
|
|
1952
|
+
this.stores = {};
|
|
2098
1953
|
}
|
|
2099
1954
|
};
|
|
2100
1955
|
var LanceFilterTranslator = class extends BaseFilterTranslator {
|
|
@@ -2443,14 +2298,14 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2443
2298
|
* ```
|
|
2444
2299
|
*/
|
|
2445
2300
|
static async create(uri, options) {
|
|
2446
|
-
const instance = new _LanceVectorStore();
|
|
2301
|
+
const instance = new _LanceVectorStore(options?.id || crypto.randomUUID());
|
|
2447
2302
|
try {
|
|
2448
2303
|
instance.lanceClient = await connect(uri, options);
|
|
2449
2304
|
return instance;
|
|
2450
2305
|
} catch (e) {
|
|
2451
2306
|
throw new MastraError(
|
|
2452
2307
|
{
|
|
2453
|
-
id: "
|
|
2308
|
+
id: createVectorErrorId("LANCE", "CONNECT", "FAILED"),
|
|
2454
2309
|
domain: ErrorDomain.STORAGE,
|
|
2455
2310
|
category: ErrorCategory.THIRD_PARTY,
|
|
2456
2311
|
details: { uri }
|
|
@@ -2463,8 +2318,8 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2463
2318
|
* @internal
|
|
2464
2319
|
* Private constructor to enforce using the create factory method
|
|
2465
2320
|
*/
|
|
2466
|
-
constructor() {
|
|
2467
|
-
super();
|
|
2321
|
+
constructor(id) {
|
|
2322
|
+
super({ id });
|
|
2468
2323
|
}
|
|
2469
2324
|
close() {
|
|
2470
2325
|
if (this.lanceClient) {
|
|
@@ -2493,7 +2348,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2493
2348
|
} catch (error) {
|
|
2494
2349
|
throw new MastraError(
|
|
2495
2350
|
{
|
|
2496
|
-
id: "
|
|
2351
|
+
id: createVectorErrorId("LANCE", "QUERY", "INVALID_ARGS"),
|
|
2497
2352
|
domain: ErrorDomain.STORAGE,
|
|
2498
2353
|
category: ErrorCategory.USER,
|
|
2499
2354
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2541,7 +2396,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2541
2396
|
} catch (error) {
|
|
2542
2397
|
throw new MastraError(
|
|
2543
2398
|
{
|
|
2544
|
-
id: "
|
|
2399
|
+
id: createVectorErrorId("LANCE", "QUERY", "FAILED"),
|
|
2545
2400
|
domain: ErrorDomain.STORAGE,
|
|
2546
2401
|
category: ErrorCategory.THIRD_PARTY,
|
|
2547
2402
|
details: { tableName, includeVector, columnsCount: columns?.length, includeAllColumns }
|
|
@@ -2593,7 +2448,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2593
2448
|
} catch (error) {
|
|
2594
2449
|
throw new MastraError(
|
|
2595
2450
|
{
|
|
2596
|
-
id: "
|
|
2451
|
+
id: createVectorErrorId("LANCE", "UPSERT", "INVALID_ARGS"),
|
|
2597
2452
|
domain: ErrorDomain.STORAGE,
|
|
2598
2453
|
category: ErrorCategory.USER,
|
|
2599
2454
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2629,7 +2484,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2629
2484
|
} catch (error) {
|
|
2630
2485
|
throw new MastraError(
|
|
2631
2486
|
{
|
|
2632
|
-
id: "
|
|
2487
|
+
id: createVectorErrorId("LANCE", "UPSERT", "FAILED"),
|
|
2633
2488
|
domain: ErrorDomain.STORAGE,
|
|
2634
2489
|
category: ErrorCategory.THIRD_PARTY,
|
|
2635
2490
|
details: { tableName, vectorCount: vectors.length, metadataCount: metadata.length, idsCount: ids.length }
|
|
@@ -2656,7 +2511,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2656
2511
|
async createTable(tableName, data, options) {
|
|
2657
2512
|
if (!this.lanceClient) {
|
|
2658
2513
|
throw new MastraError({
|
|
2659
|
-
id: "
|
|
2514
|
+
id: createVectorErrorId("LANCE", "CREATE_TABLE", "INVALID_ARGS"),
|
|
2660
2515
|
domain: ErrorDomain.STORAGE,
|
|
2661
2516
|
category: ErrorCategory.USER,
|
|
2662
2517
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2671,7 +2526,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2671
2526
|
} catch (error) {
|
|
2672
2527
|
throw new MastraError(
|
|
2673
2528
|
{
|
|
2674
|
-
id: "
|
|
2529
|
+
id: createVectorErrorId("LANCE", "CREATE_TABLE", "FAILED"),
|
|
2675
2530
|
domain: ErrorDomain.STORAGE,
|
|
2676
2531
|
category: ErrorCategory.THIRD_PARTY,
|
|
2677
2532
|
details: { tableName }
|
|
@@ -2683,7 +2538,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2683
2538
|
async listTables() {
|
|
2684
2539
|
if (!this.lanceClient) {
|
|
2685
2540
|
throw new MastraError({
|
|
2686
|
-
id: "
|
|
2541
|
+
id: createVectorErrorId("LANCE", "LIST_TABLES", "INVALID_ARGS"),
|
|
2687
2542
|
domain: ErrorDomain.STORAGE,
|
|
2688
2543
|
category: ErrorCategory.USER,
|
|
2689
2544
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2695,7 +2550,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2695
2550
|
} catch (error) {
|
|
2696
2551
|
throw new MastraError(
|
|
2697
2552
|
{
|
|
2698
|
-
id: "
|
|
2553
|
+
id: createVectorErrorId("LANCE", "LIST_TABLES", "FAILED"),
|
|
2699
2554
|
domain: ErrorDomain.STORAGE,
|
|
2700
2555
|
category: ErrorCategory.THIRD_PARTY
|
|
2701
2556
|
},
|
|
@@ -2706,7 +2561,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2706
2561
|
async getTableSchema(tableName) {
|
|
2707
2562
|
if (!this.lanceClient) {
|
|
2708
2563
|
throw new MastraError({
|
|
2709
|
-
id: "
|
|
2564
|
+
id: createVectorErrorId("LANCE", "GET_TABLE_SCHEMA", "INVALID_ARGS"),
|
|
2710
2565
|
domain: ErrorDomain.STORAGE,
|
|
2711
2566
|
category: ErrorCategory.USER,
|
|
2712
2567
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2719,7 +2574,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2719
2574
|
} catch (error) {
|
|
2720
2575
|
throw new MastraError(
|
|
2721
2576
|
{
|
|
2722
|
-
id: "
|
|
2577
|
+
id: createVectorErrorId("LANCE", "GET_TABLE_SCHEMA", "FAILED"),
|
|
2723
2578
|
domain: ErrorDomain.STORAGE,
|
|
2724
2579
|
category: ErrorCategory.THIRD_PARTY,
|
|
2725
2580
|
details: { tableName }
|
|
@@ -2754,7 +2609,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2754
2609
|
} catch (err) {
|
|
2755
2610
|
throw new MastraError(
|
|
2756
2611
|
{
|
|
2757
|
-
id: "
|
|
2612
|
+
id: createVectorErrorId("LANCE", "CREATE_INDEX", "INVALID_ARGS"),
|
|
2758
2613
|
domain: ErrorDomain.STORAGE,
|
|
2759
2614
|
category: ErrorCategory.USER,
|
|
2760
2615
|
details: { tableName: tableName || "", indexName, dimension, metric }
|
|
@@ -2799,7 +2654,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2799
2654
|
} catch (error) {
|
|
2800
2655
|
throw new MastraError(
|
|
2801
2656
|
{
|
|
2802
|
-
id: "
|
|
2657
|
+
id: createVectorErrorId("LANCE", "CREATE_INDEX", "FAILED"),
|
|
2803
2658
|
domain: ErrorDomain.STORAGE,
|
|
2804
2659
|
category: ErrorCategory.THIRD_PARTY,
|
|
2805
2660
|
details: { tableName: tableName || "", indexName, dimension }
|
|
@@ -2811,7 +2666,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2811
2666
|
async listIndexes() {
|
|
2812
2667
|
if (!this.lanceClient) {
|
|
2813
2668
|
throw new MastraError({
|
|
2814
|
-
id: "
|
|
2669
|
+
id: createVectorErrorId("LANCE", "LIST_INDEXES", "INVALID_ARGS"),
|
|
2815
2670
|
domain: ErrorDomain.STORAGE,
|
|
2816
2671
|
category: ErrorCategory.USER,
|
|
2817
2672
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2830,7 +2685,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2830
2685
|
} catch (error) {
|
|
2831
2686
|
throw new MastraError(
|
|
2832
2687
|
{
|
|
2833
|
-
id: "
|
|
2688
|
+
id: createVectorErrorId("LANCE", "LIST_INDEXES", "FAILED"),
|
|
2834
2689
|
domain: ErrorDomain.STORAGE,
|
|
2835
2690
|
category: ErrorCategory.THIRD_PARTY
|
|
2836
2691
|
},
|
|
@@ -2849,7 +2704,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2849
2704
|
} catch (err) {
|
|
2850
2705
|
throw new MastraError(
|
|
2851
2706
|
{
|
|
2852
|
-
id: "
|
|
2707
|
+
id: createVectorErrorId("LANCE", "DESCRIBE_INDEX", "INVALID_ARGS"),
|
|
2853
2708
|
domain: ErrorDomain.STORAGE,
|
|
2854
2709
|
category: ErrorCategory.USER,
|
|
2855
2710
|
details: { indexName }
|
|
@@ -2884,7 +2739,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2884
2739
|
} catch (error) {
|
|
2885
2740
|
throw new MastraError(
|
|
2886
2741
|
{
|
|
2887
|
-
id: "
|
|
2742
|
+
id: createVectorErrorId("LANCE", "DESCRIBE_INDEX", "FAILED"),
|
|
2888
2743
|
domain: ErrorDomain.STORAGE,
|
|
2889
2744
|
category: ErrorCategory.THIRD_PARTY,
|
|
2890
2745
|
details: { indexName }
|
|
@@ -2904,7 +2759,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2904
2759
|
} catch (err) {
|
|
2905
2760
|
throw new MastraError(
|
|
2906
2761
|
{
|
|
2907
|
-
id: "
|
|
2762
|
+
id: createVectorErrorId("LANCE", "DELETE_INDEX", "INVALID_ARGS"),
|
|
2908
2763
|
domain: ErrorDomain.STORAGE,
|
|
2909
2764
|
category: ErrorCategory.USER,
|
|
2910
2765
|
details: { indexName }
|
|
@@ -2927,7 +2782,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2927
2782
|
} catch (error) {
|
|
2928
2783
|
throw new MastraError(
|
|
2929
2784
|
{
|
|
2930
|
-
id: "
|
|
2785
|
+
id: createVectorErrorId("LANCE", "DELETE_INDEX", "FAILED"),
|
|
2931
2786
|
domain: ErrorDomain.STORAGE,
|
|
2932
2787
|
category: ErrorCategory.THIRD_PARTY,
|
|
2933
2788
|
details: { indexName }
|
|
@@ -2942,7 +2797,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2942
2797
|
async deleteAllTables() {
|
|
2943
2798
|
if (!this.lanceClient) {
|
|
2944
2799
|
throw new MastraError({
|
|
2945
|
-
id: "
|
|
2800
|
+
id: createVectorErrorId("LANCE", "DELETE_ALL_TABLES", "INVALID_ARGS"),
|
|
2946
2801
|
domain: ErrorDomain.STORAGE,
|
|
2947
2802
|
category: ErrorCategory.USER,
|
|
2948
2803
|
details: { methodName: "deleteAllTables" },
|
|
@@ -2954,7 +2809,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2954
2809
|
} catch (error) {
|
|
2955
2810
|
throw new MastraError(
|
|
2956
2811
|
{
|
|
2957
|
-
id: "
|
|
2812
|
+
id: createVectorErrorId("LANCE", "DELETE_ALL_TABLES", "FAILED"),
|
|
2958
2813
|
domain: ErrorDomain.STORAGE,
|
|
2959
2814
|
category: ErrorCategory.THIRD_PARTY,
|
|
2960
2815
|
details: { methodName: "deleteAllTables" }
|
|
@@ -2966,7 +2821,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2966
2821
|
async deleteTable(tableName) {
|
|
2967
2822
|
if (!this.lanceClient) {
|
|
2968
2823
|
throw new MastraError({
|
|
2969
|
-
id: "
|
|
2824
|
+
id: createVectorErrorId("LANCE", "DELETE_TABLE", "INVALID_ARGS"),
|
|
2970
2825
|
domain: ErrorDomain.STORAGE,
|
|
2971
2826
|
category: ErrorCategory.USER,
|
|
2972
2827
|
details: { tableName },
|
|
@@ -2978,7 +2833,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2978
2833
|
} catch (error) {
|
|
2979
2834
|
throw new MastraError(
|
|
2980
2835
|
{
|
|
2981
|
-
id: "
|
|
2836
|
+
id: createVectorErrorId("LANCE", "DELETE_TABLE", "FAILED"),
|
|
2982
2837
|
domain: ErrorDomain.STORAGE,
|
|
2983
2838
|
category: ErrorCategory.THIRD_PARTY,
|
|
2984
2839
|
details: { tableName }
|
|
@@ -2987,7 +2842,44 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2987
2842
|
);
|
|
2988
2843
|
}
|
|
2989
2844
|
}
|
|
2990
|
-
async updateVector(
|
|
2845
|
+
async updateVector(params) {
|
|
2846
|
+
const { indexName, update } = params;
|
|
2847
|
+
if ("id" in params && "filter" in params && params.id && params.filter) {
|
|
2848
|
+
throw new MastraError({
|
|
2849
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "MUTUALLY_EXCLUSIVE"),
|
|
2850
|
+
domain: ErrorDomain.STORAGE,
|
|
2851
|
+
category: ErrorCategory.USER,
|
|
2852
|
+
text: "id and filter are mutually exclusive",
|
|
2853
|
+
details: { indexName }
|
|
2854
|
+
});
|
|
2855
|
+
}
|
|
2856
|
+
if (!("id" in params || "filter" in params) || !params.id && !params.filter) {
|
|
2857
|
+
throw new MastraError({
|
|
2858
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "NO_TARGET"),
|
|
2859
|
+
domain: ErrorDomain.STORAGE,
|
|
2860
|
+
category: ErrorCategory.USER,
|
|
2861
|
+
text: "Either id or filter must be provided",
|
|
2862
|
+
details: { indexName }
|
|
2863
|
+
});
|
|
2864
|
+
}
|
|
2865
|
+
if ("filter" in params && params.filter && Object.keys(params.filter).length === 0) {
|
|
2866
|
+
throw new MastraError({
|
|
2867
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "EMPTY_FILTER"),
|
|
2868
|
+
domain: ErrorDomain.STORAGE,
|
|
2869
|
+
category: ErrorCategory.USER,
|
|
2870
|
+
text: "Cannot update with empty filter",
|
|
2871
|
+
details: { indexName }
|
|
2872
|
+
});
|
|
2873
|
+
}
|
|
2874
|
+
if (!update.vector && !update.metadata) {
|
|
2875
|
+
throw new MastraError({
|
|
2876
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "NO_PAYLOAD"),
|
|
2877
|
+
domain: ErrorDomain.STORAGE,
|
|
2878
|
+
category: ErrorCategory.USER,
|
|
2879
|
+
text: "No updates provided",
|
|
2880
|
+
details: { indexName }
|
|
2881
|
+
});
|
|
2882
|
+
}
|
|
2991
2883
|
try {
|
|
2992
2884
|
if (!this.lanceClient) {
|
|
2993
2885
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
@@ -2995,21 +2887,6 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2995
2887
|
if (!indexName) {
|
|
2996
2888
|
throw new Error("indexName is required");
|
|
2997
2889
|
}
|
|
2998
|
-
if (!id) {
|
|
2999
|
-
throw new Error("id is required");
|
|
3000
|
-
}
|
|
3001
|
-
} catch (err) {
|
|
3002
|
-
throw new MastraError(
|
|
3003
|
-
{
|
|
3004
|
-
id: "STORAGE_LANCE_VECTOR_UPDATE_VECTOR_FAILED_INVALID_ARGS",
|
|
3005
|
-
domain: ErrorDomain.STORAGE,
|
|
3006
|
-
category: ErrorCategory.USER,
|
|
3007
|
-
details: { indexName, id }
|
|
3008
|
-
},
|
|
3009
|
-
err
|
|
3010
|
-
);
|
|
3011
|
-
}
|
|
3012
|
-
try {
|
|
3013
2890
|
const tables = await this.lanceClient.tableNames();
|
|
3014
2891
|
for (const tableName of tables) {
|
|
3015
2892
|
this.logger.debug("Checking table:" + tableName);
|
|
@@ -3019,39 +2896,66 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3019
2896
|
const hasColumn = schema.fields.some((field) => field.name === indexName);
|
|
3020
2897
|
if (hasColumn) {
|
|
3021
2898
|
this.logger.debug(`Found column ${indexName} in table ${tableName}`);
|
|
3022
|
-
|
|
3023
|
-
if (
|
|
3024
|
-
|
|
2899
|
+
let whereClause;
|
|
2900
|
+
if ("id" in params && params.id) {
|
|
2901
|
+
whereClause = `id = '${params.id}'`;
|
|
2902
|
+
} else if ("filter" in params && params.filter) {
|
|
2903
|
+
const translator = new LanceFilterTranslator();
|
|
2904
|
+
const processFilterKeys = (filter) => {
|
|
2905
|
+
const processedFilter = {};
|
|
2906
|
+
Object.entries(filter).forEach(([key, value]) => {
|
|
2907
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
2908
|
+
Object.entries(value).forEach(([nestedKey, nestedValue]) => {
|
|
2909
|
+
processedFilter[`metadata_${key}_${nestedKey}`] = nestedValue;
|
|
2910
|
+
});
|
|
2911
|
+
} else {
|
|
2912
|
+
processedFilter[`metadata_${key}`] = value;
|
|
2913
|
+
}
|
|
2914
|
+
});
|
|
2915
|
+
return processedFilter;
|
|
2916
|
+
};
|
|
2917
|
+
const prefixedFilter = processFilterKeys(params.filter);
|
|
2918
|
+
whereClause = translator.translate(prefixedFilter) || "";
|
|
2919
|
+
if (!whereClause) {
|
|
2920
|
+
throw new Error("Failed to translate filter to SQL");
|
|
2921
|
+
}
|
|
2922
|
+
} else {
|
|
2923
|
+
throw new Error("Either id or filter must be provided");
|
|
3025
2924
|
}
|
|
3026
|
-
const
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
2925
|
+
const existingRecords = await table.query().where(whereClause).select(schema.fields.map((field) => field.name)).toArray();
|
|
2926
|
+
if (existingRecords.length === 0) {
|
|
2927
|
+
this.logger.info(`No records found matching criteria in table ${tableName}`);
|
|
2928
|
+
return;
|
|
2929
|
+
}
|
|
2930
|
+
const updatedRecords = existingRecords.map((record) => {
|
|
2931
|
+
const rowData = {};
|
|
2932
|
+
Object.entries(record).forEach(([key, value]) => {
|
|
2933
|
+
if (key !== "_distance") {
|
|
2934
|
+
if (key === indexName) {
|
|
2935
|
+
if (update.vector) {
|
|
2936
|
+
rowData[key] = update.vector;
|
|
3037
2937
|
} else {
|
|
3038
|
-
|
|
2938
|
+
if (Array.isArray(value)) {
|
|
2939
|
+
rowData[key] = [...value];
|
|
2940
|
+
} else if (typeof value === "object" && value !== null) {
|
|
2941
|
+
rowData[key] = Array.from(value);
|
|
2942
|
+
} else {
|
|
2943
|
+
rowData[key] = value;
|
|
2944
|
+
}
|
|
3039
2945
|
}
|
|
2946
|
+
} else {
|
|
2947
|
+
rowData[key] = value;
|
|
3040
2948
|
}
|
|
3041
|
-
} else {
|
|
3042
|
-
rowData[key] = value;
|
|
3043
2949
|
}
|
|
2950
|
+
});
|
|
2951
|
+
if (update.metadata) {
|
|
2952
|
+
Object.entries(update.metadata).forEach(([key, value]) => {
|
|
2953
|
+
rowData[`metadata_${key}`] = value;
|
|
2954
|
+
});
|
|
3044
2955
|
}
|
|
2956
|
+
return rowData;
|
|
3045
2957
|
});
|
|
3046
|
-
|
|
3047
|
-
rowData[indexName] = update.vector;
|
|
3048
|
-
}
|
|
3049
|
-
if (update.metadata) {
|
|
3050
|
-
Object.entries(update.metadata).forEach(([key, value]) => {
|
|
3051
|
-
rowData[`metadata_${key}`] = value;
|
|
3052
|
-
});
|
|
3053
|
-
}
|
|
3054
|
-
await table.add([rowData], { mode: "overwrite" });
|
|
2958
|
+
await table.add(updatedRecords, { mode: "overwrite" });
|
|
3055
2959
|
return;
|
|
3056
2960
|
}
|
|
3057
2961
|
} catch (err) {
|
|
@@ -3061,12 +2965,19 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3061
2965
|
}
|
|
3062
2966
|
throw new Error(`No table found with column/index '${indexName}'`);
|
|
3063
2967
|
} catch (error) {
|
|
2968
|
+
if (error instanceof MastraError) throw error;
|
|
3064
2969
|
throw new MastraError(
|
|
3065
2970
|
{
|
|
3066
|
-
id: "
|
|
2971
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "FAILED"),
|
|
3067
2972
|
domain: ErrorDomain.STORAGE,
|
|
3068
2973
|
category: ErrorCategory.THIRD_PARTY,
|
|
3069
|
-
details: {
|
|
2974
|
+
details: {
|
|
2975
|
+
indexName,
|
|
2976
|
+
..."id" in params && params.id && { id: params.id },
|
|
2977
|
+
..."filter" in params && params.filter && { filter: JSON.stringify(params.filter) },
|
|
2978
|
+
hasVector: !!update.vector,
|
|
2979
|
+
hasMetadata: !!update.metadata
|
|
2980
|
+
}
|
|
3070
2981
|
},
|
|
3071
2982
|
error
|
|
3072
2983
|
);
|
|
@@ -3086,10 +2997,13 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3086
2997
|
} catch (err) {
|
|
3087
2998
|
throw new MastraError(
|
|
3088
2999
|
{
|
|
3089
|
-
id: "
|
|
3000
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTOR", "INVALID_ARGS"),
|
|
3090
3001
|
domain: ErrorDomain.STORAGE,
|
|
3091
3002
|
category: ErrorCategory.USER,
|
|
3092
|
-
details: {
|
|
3003
|
+
details: {
|
|
3004
|
+
indexName,
|
|
3005
|
+
...id && { id }
|
|
3006
|
+
}
|
|
3093
3007
|
},
|
|
3094
3008
|
err
|
|
3095
3009
|
);
|
|
@@ -3116,10 +3030,13 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3116
3030
|
} catch (error) {
|
|
3117
3031
|
throw new MastraError(
|
|
3118
3032
|
{
|
|
3119
|
-
id: "
|
|
3033
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTOR", "FAILED"),
|
|
3120
3034
|
domain: ErrorDomain.STORAGE,
|
|
3121
3035
|
category: ErrorCategory.THIRD_PARTY,
|
|
3122
|
-
details: {
|
|
3036
|
+
details: {
|
|
3037
|
+
indexName,
|
|
3038
|
+
...id && { id }
|
|
3039
|
+
}
|
|
3123
3040
|
},
|
|
3124
3041
|
error
|
|
3125
3042
|
);
|
|
@@ -3150,8 +3067,111 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3150
3067
|
});
|
|
3151
3068
|
return result;
|
|
3152
3069
|
}
|
|
3070
|
+
async deleteVectors({ indexName, filter, ids }) {
|
|
3071
|
+
if (ids && filter) {
|
|
3072
|
+
throw new MastraError({
|
|
3073
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "MUTUALLY_EXCLUSIVE"),
|
|
3074
|
+
domain: ErrorDomain.STORAGE,
|
|
3075
|
+
category: ErrorCategory.USER,
|
|
3076
|
+
text: "ids and filter are mutually exclusive",
|
|
3077
|
+
details: { indexName }
|
|
3078
|
+
});
|
|
3079
|
+
}
|
|
3080
|
+
if (!ids && !filter) {
|
|
3081
|
+
throw new MastraError({
|
|
3082
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "NO_TARGET"),
|
|
3083
|
+
domain: ErrorDomain.STORAGE,
|
|
3084
|
+
category: ErrorCategory.USER,
|
|
3085
|
+
text: "Either filter or ids must be provided",
|
|
3086
|
+
details: { indexName }
|
|
3087
|
+
});
|
|
3088
|
+
}
|
|
3089
|
+
if (ids && ids.length === 0) {
|
|
3090
|
+
throw new MastraError({
|
|
3091
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "EMPTY_IDS"),
|
|
3092
|
+
domain: ErrorDomain.STORAGE,
|
|
3093
|
+
category: ErrorCategory.USER,
|
|
3094
|
+
text: "Cannot delete with empty ids array",
|
|
3095
|
+
details: { indexName }
|
|
3096
|
+
});
|
|
3097
|
+
}
|
|
3098
|
+
if (filter && Object.keys(filter).length === 0) {
|
|
3099
|
+
throw new MastraError({
|
|
3100
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "EMPTY_FILTER"),
|
|
3101
|
+
domain: ErrorDomain.STORAGE,
|
|
3102
|
+
category: ErrorCategory.USER,
|
|
3103
|
+
text: "Cannot delete with empty filter",
|
|
3104
|
+
details: { indexName }
|
|
3105
|
+
});
|
|
3106
|
+
}
|
|
3107
|
+
try {
|
|
3108
|
+
if (!this.lanceClient) {
|
|
3109
|
+
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
3110
|
+
}
|
|
3111
|
+
if (!indexName) {
|
|
3112
|
+
throw new Error("indexName is required");
|
|
3113
|
+
}
|
|
3114
|
+
const tables = await this.lanceClient.tableNames();
|
|
3115
|
+
for (const tableName of tables) {
|
|
3116
|
+
this.logger.debug("Checking table:" + tableName);
|
|
3117
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
3118
|
+
try {
|
|
3119
|
+
const schema = await table.schema();
|
|
3120
|
+
const hasColumn = schema.fields.some((field) => field.name === indexName);
|
|
3121
|
+
if (hasColumn) {
|
|
3122
|
+
this.logger.debug(`Found column ${indexName} in table ${tableName}`);
|
|
3123
|
+
if (ids) {
|
|
3124
|
+
const idsConditions = ids.map((id) => `id = '${id}'`).join(" OR ");
|
|
3125
|
+
await table.delete(idsConditions);
|
|
3126
|
+
} else if (filter) {
|
|
3127
|
+
const translator = new LanceFilterTranslator();
|
|
3128
|
+
const processFilterKeys = (filter2) => {
|
|
3129
|
+
const processedFilter = {};
|
|
3130
|
+
Object.entries(filter2).forEach(([key, value]) => {
|
|
3131
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
3132
|
+
Object.entries(value).forEach(([nestedKey, nestedValue]) => {
|
|
3133
|
+
processedFilter[`metadata_${key}_${nestedKey}`] = nestedValue;
|
|
3134
|
+
});
|
|
3135
|
+
} else {
|
|
3136
|
+
processedFilter[`metadata_${key}`] = value;
|
|
3137
|
+
}
|
|
3138
|
+
});
|
|
3139
|
+
return processedFilter;
|
|
3140
|
+
};
|
|
3141
|
+
const prefixedFilter = processFilterKeys(filter);
|
|
3142
|
+
const whereClause = translator.translate(prefixedFilter);
|
|
3143
|
+
if (!whereClause) {
|
|
3144
|
+
throw new Error("Failed to translate filter to SQL");
|
|
3145
|
+
}
|
|
3146
|
+
await table.delete(whereClause);
|
|
3147
|
+
}
|
|
3148
|
+
return;
|
|
3149
|
+
}
|
|
3150
|
+
} catch (err) {
|
|
3151
|
+
this.logger.error(`Error checking schema for table ${tableName}:` + err);
|
|
3152
|
+
continue;
|
|
3153
|
+
}
|
|
3154
|
+
}
|
|
3155
|
+
throw new Error(`No table found with column/index '${indexName}'`);
|
|
3156
|
+
} catch (error) {
|
|
3157
|
+
if (error instanceof MastraError) throw error;
|
|
3158
|
+
throw new MastraError(
|
|
3159
|
+
{
|
|
3160
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "FAILED"),
|
|
3161
|
+
domain: ErrorDomain.STORAGE,
|
|
3162
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
3163
|
+
details: {
|
|
3164
|
+
indexName,
|
|
3165
|
+
...filter && { filter: JSON.stringify(filter) },
|
|
3166
|
+
...ids && { idsCount: ids.length }
|
|
3167
|
+
}
|
|
3168
|
+
},
|
|
3169
|
+
error
|
|
3170
|
+
);
|
|
3171
|
+
}
|
|
3172
|
+
}
|
|
3153
3173
|
};
|
|
3154
3174
|
|
|
3155
|
-
export { LanceStorage, LanceVectorStore };
|
|
3175
|
+
export { LanceStorage, LanceVectorStore, StoreMemoryLance, StoreScoresLance, StoreWorkflowsLance };
|
|
3156
3176
|
//# sourceMappingURL=index.js.map
|
|
3157
3177
|
//# sourceMappingURL=index.js.map
|