@mastra/lance 1.0.0-beta.1 → 1.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +498 -0
- package/dist/docs/README.md +33 -0
- package/dist/docs/SKILL.md +34 -0
- package/dist/docs/SOURCE_MAP.json +6 -0
- package/dist/docs/rag/01-vector-databases.md +643 -0
- package/dist/docs/storage/01-reference.md +113 -0
- package/dist/docs/vectors/01-reference.md +149 -0
- package/dist/index.cjs +1594 -1346
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1593 -1348
- package/dist/index.js.map +1 -1
- package/dist/storage/{domains/operations → db}/index.d.ts +21 -2
- package/dist/storage/db/index.d.ts.map +1 -0
- package/dist/storage/db/utils.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +6 -7
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/dist/storage/domains/scores/index.d.ts +20 -23
- package/dist/storage/domains/scores/index.d.ts.map +1 -1
- package/dist/storage/domains/workflows/index.d.ts +17 -14
- package/dist/storage/domains/workflows/index.d.ts.map +1 -1
- package/dist/storage/index.d.ts +96 -178
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/vector/filter.d.ts +5 -5
- package/dist/vector/index.d.ts +18 -7
- package/dist/vector/index.d.ts.map +1 -1
- package/package.json +9 -9
- package/dist/storage/domains/operations/index.d.ts.map +0 -1
- package/dist/storage/domains/utils.d.ts.map +0 -1
- /package/dist/storage/{domains → db}/utils.d.ts +0 -0
package/dist/index.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { connect, Index } from '@lancedb/lancedb';
|
|
2
2
|
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
|
-
import {
|
|
3
|
+
import { MemoryStorage, TABLE_SCHEMAS, TABLE_THREADS, TABLE_MESSAGES, TABLE_RESOURCES, createStorageErrorId, normalizePerPage, calculatePagination, ScoresStorage, SCORERS_SCHEMA, TABLE_SCORERS, WorkflowsStorage, ensureDate, TABLE_WORKFLOW_SNAPSHOT, MastraStorage, createVectorErrorId, getDefaultValue } from '@mastra/core/storage';
|
|
4
4
|
import { MessageList } from '@mastra/core/agent';
|
|
5
|
+
import { MastraBase } from '@mastra/core/base';
|
|
5
6
|
import { Utf8, Float64, Binary, Float32, Int32, Field, Schema } from 'apache-arrow';
|
|
6
7
|
import { saveScorePayloadSchema } from '@mastra/core/evals';
|
|
7
8
|
import { MastraVector } from '@mastra/core/vector';
|
|
@@ -89,7 +90,7 @@ async function getTableSchema({
|
|
|
89
90
|
} catch (validationError) {
|
|
90
91
|
throw new MastraError(
|
|
91
92
|
{
|
|
92
|
-
id: "
|
|
93
|
+
id: createStorageErrorId("LANCE", "GET_TABLE_SCHEMA", "INVALID_ARGS"),
|
|
93
94
|
domain: ErrorDomain.STORAGE,
|
|
94
95
|
category: ErrorCategory.USER,
|
|
95
96
|
text: validationError.message,
|
|
@@ -112,7 +113,7 @@ async function getTableSchema({
|
|
|
112
113
|
} catch (error) {
|
|
113
114
|
throw new MastraError(
|
|
114
115
|
{
|
|
115
|
-
id: "
|
|
116
|
+
id: createStorageErrorId("LANCE", "GET_TABLE_SCHEMA", "FAILED"),
|
|
116
117
|
domain: ErrorDomain.STORAGE,
|
|
117
118
|
category: ErrorCategory.THIRD_PARTY,
|
|
118
119
|
details: { tableName }
|
|
@@ -122,1118 +123,1200 @@ async function getTableSchema({
|
|
|
122
123
|
}
|
|
123
124
|
}
|
|
124
125
|
|
|
125
|
-
// src/storage/
|
|
126
|
-
|
|
126
|
+
// src/storage/db/index.ts
|
|
127
|
+
function resolveLanceConfig(config) {
|
|
128
|
+
return config.client;
|
|
129
|
+
}
|
|
130
|
+
var LanceDB = class extends MastraBase {
|
|
127
131
|
client;
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
super();
|
|
132
|
+
constructor({ client }) {
|
|
133
|
+
super({ name: "lance-db" });
|
|
131
134
|
this.client = client;
|
|
132
|
-
this.operations = operations;
|
|
133
135
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
136
|
+
getDefaultValue(type) {
|
|
137
|
+
switch (type) {
|
|
138
|
+
case "text":
|
|
139
|
+
return "''";
|
|
140
|
+
case "timestamp":
|
|
141
|
+
return "CURRENT_TIMESTAMP";
|
|
142
|
+
case "integer":
|
|
143
|
+
case "bigint":
|
|
144
|
+
return "0";
|
|
145
|
+
case "jsonb":
|
|
146
|
+
return "'{}'";
|
|
147
|
+
case "uuid":
|
|
148
|
+
return "''";
|
|
149
|
+
default:
|
|
150
|
+
return getDefaultValue(type);
|
|
151
|
+
}
|
|
137
152
|
}
|
|
138
|
-
async
|
|
153
|
+
async hasColumn(tableName, columnName) {
|
|
154
|
+
const table = await this.client.openTable(tableName);
|
|
155
|
+
const schema = await table.schema();
|
|
156
|
+
return schema.fields.some((field) => field.name === columnName);
|
|
157
|
+
}
|
|
158
|
+
translateSchema(schema) {
|
|
159
|
+
const fields = Object.entries(schema).map(([name, column]) => {
|
|
160
|
+
let arrowType;
|
|
161
|
+
switch (column.type.toLowerCase()) {
|
|
162
|
+
case "text":
|
|
163
|
+
case "uuid":
|
|
164
|
+
arrowType = new Utf8();
|
|
165
|
+
break;
|
|
166
|
+
case "int":
|
|
167
|
+
case "integer":
|
|
168
|
+
arrowType = new Int32();
|
|
169
|
+
break;
|
|
170
|
+
case "bigint":
|
|
171
|
+
arrowType = new Float64();
|
|
172
|
+
break;
|
|
173
|
+
case "float":
|
|
174
|
+
arrowType = new Float32();
|
|
175
|
+
break;
|
|
176
|
+
case "jsonb":
|
|
177
|
+
case "json":
|
|
178
|
+
arrowType = new Utf8();
|
|
179
|
+
break;
|
|
180
|
+
case "binary":
|
|
181
|
+
arrowType = new Binary();
|
|
182
|
+
break;
|
|
183
|
+
case "timestamp":
|
|
184
|
+
arrowType = new Float64();
|
|
185
|
+
break;
|
|
186
|
+
default:
|
|
187
|
+
arrowType = new Utf8();
|
|
188
|
+
}
|
|
189
|
+
return new Field(name, arrowType, column.nullable ?? true);
|
|
190
|
+
});
|
|
191
|
+
return new Schema(fields);
|
|
192
|
+
}
|
|
193
|
+
async createTable({
|
|
194
|
+
tableName,
|
|
195
|
+
schema
|
|
196
|
+
}) {
|
|
139
197
|
try {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
198
|
+
if (!this.client) {
|
|
199
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
200
|
+
}
|
|
201
|
+
if (!tableName) {
|
|
202
|
+
throw new Error("tableName is required for createTable.");
|
|
203
|
+
}
|
|
204
|
+
if (!schema) {
|
|
205
|
+
throw new Error("schema is required for createTable.");
|
|
143
206
|
}
|
|
144
|
-
return {
|
|
145
|
-
...thread,
|
|
146
|
-
createdAt: new Date(thread.createdAt),
|
|
147
|
-
updatedAt: new Date(thread.updatedAt)
|
|
148
|
-
};
|
|
149
207
|
} catch (error) {
|
|
150
208
|
throw new MastraError(
|
|
151
209
|
{
|
|
152
|
-
id: "
|
|
210
|
+
id: createStorageErrorId("LANCE", "CREATE_TABLE", "INVALID_ARGS"),
|
|
153
211
|
domain: ErrorDomain.STORAGE,
|
|
154
|
-
category: ErrorCategory.
|
|
212
|
+
category: ErrorCategory.USER,
|
|
213
|
+
details: { tableName }
|
|
155
214
|
},
|
|
156
215
|
error
|
|
157
216
|
);
|
|
158
217
|
}
|
|
159
|
-
}
|
|
160
|
-
/**
|
|
161
|
-
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
162
|
-
* @param thread - The thread to save
|
|
163
|
-
* @returns The saved thread
|
|
164
|
-
*/
|
|
165
|
-
async saveThread({ thread }) {
|
|
166
218
|
try {
|
|
167
|
-
const
|
|
168
|
-
|
|
169
|
-
await table.add([record], { mode: "append" });
|
|
170
|
-
return thread;
|
|
219
|
+
const arrowSchema = this.translateSchema(schema);
|
|
220
|
+
await this.client.createEmptyTable(tableName, arrowSchema);
|
|
171
221
|
} catch (error) {
|
|
222
|
+
if (error.message?.includes("already exists")) {
|
|
223
|
+
this.logger.debug(`Table '${tableName}' already exists, skipping create`);
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
172
226
|
throw new MastraError(
|
|
173
227
|
{
|
|
174
|
-
id: "
|
|
228
|
+
id: createStorageErrorId("LANCE", "CREATE_TABLE", "FAILED"),
|
|
175
229
|
domain: ErrorDomain.STORAGE,
|
|
176
|
-
category: ErrorCategory.THIRD_PARTY
|
|
230
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
231
|
+
details: { tableName }
|
|
177
232
|
},
|
|
178
233
|
error
|
|
179
234
|
);
|
|
180
235
|
}
|
|
181
236
|
}
|
|
182
|
-
async
|
|
183
|
-
id,
|
|
184
|
-
title,
|
|
185
|
-
metadata
|
|
186
|
-
}) {
|
|
187
|
-
const maxRetries = 5;
|
|
188
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
189
|
-
try {
|
|
190
|
-
const current = await this.getThreadById({ threadId: id });
|
|
191
|
-
if (!current) {
|
|
192
|
-
throw new Error(`Thread with id ${id} not found`);
|
|
193
|
-
}
|
|
194
|
-
const mergedMetadata = { ...current.metadata, ...metadata };
|
|
195
|
-
const record = {
|
|
196
|
-
id,
|
|
197
|
-
title,
|
|
198
|
-
metadata: JSON.stringify(mergedMetadata),
|
|
199
|
-
updatedAt: (/* @__PURE__ */ new Date()).getTime()
|
|
200
|
-
};
|
|
201
|
-
const table = await this.client.openTable(TABLE_THREADS);
|
|
202
|
-
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
203
|
-
const updatedThread = await this.getThreadById({ threadId: id });
|
|
204
|
-
if (!updatedThread) {
|
|
205
|
-
throw new Error(`Failed to retrieve updated thread ${id}`);
|
|
206
|
-
}
|
|
207
|
-
return updatedThread;
|
|
208
|
-
} catch (error) {
|
|
209
|
-
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
210
|
-
const delay = Math.pow(2, attempt) * 10;
|
|
211
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
212
|
-
continue;
|
|
213
|
-
}
|
|
214
|
-
throw new MastraError(
|
|
215
|
-
{
|
|
216
|
-
id: "LANCE_STORE_UPDATE_THREAD_FAILED",
|
|
217
|
-
domain: ErrorDomain.STORAGE,
|
|
218
|
-
category: ErrorCategory.THIRD_PARTY
|
|
219
|
-
},
|
|
220
|
-
error
|
|
221
|
-
);
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
throw new MastraError(
|
|
225
|
-
{
|
|
226
|
-
id: "LANCE_STORE_UPDATE_THREAD_FAILED",
|
|
227
|
-
domain: ErrorDomain.STORAGE,
|
|
228
|
-
category: ErrorCategory.THIRD_PARTY
|
|
229
|
-
},
|
|
230
|
-
new Error("All retries exhausted")
|
|
231
|
-
);
|
|
232
|
-
}
|
|
233
|
-
async deleteThread({ threadId }) {
|
|
237
|
+
async dropTable({ tableName }) {
|
|
234
238
|
try {
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
239
|
+
if (!this.client) {
|
|
240
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
241
|
+
}
|
|
242
|
+
if (!tableName) {
|
|
243
|
+
throw new Error("tableName is required for dropTable.");
|
|
244
|
+
}
|
|
245
|
+
} catch (validationError) {
|
|
240
246
|
throw new MastraError(
|
|
241
247
|
{
|
|
242
|
-
id: "
|
|
248
|
+
id: createStorageErrorId("LANCE", "DROP_TABLE", "INVALID_ARGS"),
|
|
243
249
|
domain: ErrorDomain.STORAGE,
|
|
244
|
-
category: ErrorCategory.
|
|
250
|
+
category: ErrorCategory.USER,
|
|
251
|
+
text: validationError.message,
|
|
252
|
+
details: { tableName }
|
|
245
253
|
},
|
|
246
|
-
|
|
254
|
+
validationError
|
|
247
255
|
);
|
|
248
256
|
}
|
|
249
|
-
}
|
|
250
|
-
normalizeMessage(message) {
|
|
251
|
-
const { thread_id, ...rest } = message;
|
|
252
|
-
return {
|
|
253
|
-
...rest,
|
|
254
|
-
threadId: thread_id,
|
|
255
|
-
content: typeof message.content === "string" ? (() => {
|
|
256
|
-
try {
|
|
257
|
-
return JSON.parse(message.content);
|
|
258
|
-
} catch {
|
|
259
|
-
return message.content;
|
|
260
|
-
}
|
|
261
|
-
})() : message.content
|
|
262
|
-
};
|
|
263
|
-
}
|
|
264
|
-
async listMessagesById({ messageIds }) {
|
|
265
|
-
if (messageIds.length === 0) return { messages: [] };
|
|
266
257
|
try {
|
|
267
|
-
|
|
268
|
-
const quotedIds = messageIds.map((id) => `'${id}'`).join(", ");
|
|
269
|
-
const allRecords = await table.query().where(`id IN (${quotedIds})`).toArray();
|
|
270
|
-
const messages = processResultWithTypeConversion(
|
|
271
|
-
allRecords,
|
|
272
|
-
await getTableSchema({ tableName: TABLE_MESSAGES, client: this.client })
|
|
273
|
-
);
|
|
274
|
-
const list = new MessageList().add(
|
|
275
|
-
messages.map(this.normalizeMessage),
|
|
276
|
-
"memory"
|
|
277
|
-
);
|
|
278
|
-
return { messages: list.get.all.db() };
|
|
258
|
+
await this.client.dropTable(tableName);
|
|
279
259
|
} catch (error) {
|
|
260
|
+
if (error.toString().includes("was not found") || error.message?.includes("Table not found")) {
|
|
261
|
+
this.logger.debug(`Table '${tableName}' does not exist, skipping drop`);
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
280
264
|
throw new MastraError(
|
|
281
265
|
{
|
|
282
|
-
id: "
|
|
266
|
+
id: createStorageErrorId("LANCE", "DROP_TABLE", "FAILED"),
|
|
283
267
|
domain: ErrorDomain.STORAGE,
|
|
284
268
|
category: ErrorCategory.THIRD_PARTY,
|
|
285
|
-
details: {
|
|
286
|
-
messageIds: JSON.stringify(messageIds)
|
|
287
|
-
}
|
|
269
|
+
details: { tableName }
|
|
288
270
|
},
|
|
289
271
|
error
|
|
290
272
|
);
|
|
291
273
|
}
|
|
292
274
|
}
|
|
293
|
-
async
|
|
294
|
-
|
|
295
|
-
|
|
275
|
+
async alterTable({
|
|
276
|
+
tableName,
|
|
277
|
+
schema,
|
|
278
|
+
ifNotExists
|
|
279
|
+
}) {
|
|
280
|
+
try {
|
|
281
|
+
if (!this.client) {
|
|
282
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
283
|
+
}
|
|
284
|
+
if (!tableName) {
|
|
285
|
+
throw new Error("tableName is required for alterTable.");
|
|
286
|
+
}
|
|
287
|
+
if (!schema) {
|
|
288
|
+
throw new Error("schema is required for alterTable.");
|
|
289
|
+
}
|
|
290
|
+
if (!ifNotExists || ifNotExists.length === 0) {
|
|
291
|
+
this.logger.debug("No columns specified to add in alterTable, skipping.");
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
} catch (validationError) {
|
|
296
295
|
throw new MastraError(
|
|
297
296
|
{
|
|
298
|
-
id: "
|
|
297
|
+
id: createStorageErrorId("LANCE", "ALTER_TABLE", "INVALID_ARGS"),
|
|
299
298
|
domain: ErrorDomain.STORAGE,
|
|
300
|
-
category: ErrorCategory.
|
|
301
|
-
|
|
299
|
+
category: ErrorCategory.USER,
|
|
300
|
+
text: validationError.message,
|
|
301
|
+
details: { tableName }
|
|
302
302
|
},
|
|
303
|
-
|
|
303
|
+
validationError
|
|
304
304
|
);
|
|
305
305
|
}
|
|
306
|
-
const perPage = normalizePerPage(perPageInput, 40);
|
|
307
|
-
const { offset, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
308
306
|
try {
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
}
|
|
320
|
-
const
|
|
321
|
-
|
|
322
|
-
const conditions = [`thread_id = '${this.escapeSql(threadId)}'`];
|
|
323
|
-
if (resourceId) {
|
|
324
|
-
conditions.push(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
325
|
-
}
|
|
326
|
-
if (filter?.dateRange?.start) {
|
|
327
|
-
const startTime = filter.dateRange.start instanceof Date ? filter.dateRange.start.getTime() : new Date(filter.dateRange.start).getTime();
|
|
328
|
-
conditions.push(`\`createdAt\` >= ${startTime}`);
|
|
329
|
-
}
|
|
330
|
-
if (filter?.dateRange?.end) {
|
|
331
|
-
const endTime = filter.dateRange.end instanceof Date ? filter.dateRange.end.getTime() : new Date(filter.dateRange.end).getTime();
|
|
332
|
-
conditions.push(`\`createdAt\` <= ${endTime}`);
|
|
333
|
-
}
|
|
334
|
-
const whereClause = conditions.join(" AND ");
|
|
335
|
-
const total = await table.countRows(whereClause);
|
|
336
|
-
const query = table.query().where(whereClause);
|
|
337
|
-
let allRecords = await query.toArray();
|
|
338
|
-
allRecords.sort((a, b) => {
|
|
339
|
-
const aValue = field === "createdAt" ? a.createdAt : a[field];
|
|
340
|
-
const bValue = field === "createdAt" ? b.createdAt : b[field];
|
|
341
|
-
if (aValue == null && bValue == null) return 0;
|
|
342
|
-
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
343
|
-
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
344
|
-
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
345
|
-
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
346
|
-
}
|
|
347
|
-
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
348
|
-
});
|
|
349
|
-
const paginatedRecords = allRecords.slice(offset, offset + perPage);
|
|
350
|
-
const messages = paginatedRecords.map((row) => this.normalizeMessage(row));
|
|
351
|
-
if (total === 0 && messages.length === 0 && (!include || include.length === 0)) {
|
|
307
|
+
const table = await this.client.openTable(tableName);
|
|
308
|
+
const currentSchema = await table.schema();
|
|
309
|
+
const existingFields = new Set(currentSchema.fields.map((f) => f.name));
|
|
310
|
+
const typeMap = {
|
|
311
|
+
text: "string",
|
|
312
|
+
integer: "int",
|
|
313
|
+
bigint: "bigint",
|
|
314
|
+
timestamp: "timestamp",
|
|
315
|
+
jsonb: "string",
|
|
316
|
+
uuid: "string"
|
|
317
|
+
};
|
|
318
|
+
const columnsToAdd = ifNotExists.filter((col) => schema[col] && !existingFields.has(col)).map((col) => {
|
|
319
|
+
const colDef = schema[col];
|
|
352
320
|
return {
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
page,
|
|
356
|
-
perPage: perPageForResponse,
|
|
357
|
-
hasMore: false
|
|
321
|
+
name: col,
|
|
322
|
+
valueSql: colDef?.nullable ? `cast(NULL as ${typeMap[colDef.type ?? "text"]})` : `cast(${this.getDefaultValue(colDef?.type ?? "text")} as ${typeMap[colDef?.type ?? "text"]})`
|
|
358
323
|
};
|
|
359
|
-
}
|
|
360
|
-
const messageIds = new Set(messages.map((m) => m.id));
|
|
361
|
-
if (include && include.length > 0) {
|
|
362
|
-
const threadIds = [...new Set(include.map((item) => item.threadId || threadId))];
|
|
363
|
-
const allThreadMessages = [];
|
|
364
|
-
for (const tid of threadIds) {
|
|
365
|
-
const threadQuery = table.query().where(`thread_id = '${tid}'`);
|
|
366
|
-
let threadRecords = await threadQuery.toArray();
|
|
367
|
-
allThreadMessages.push(...threadRecords);
|
|
368
|
-
}
|
|
369
|
-
allThreadMessages.sort((a, b) => a.createdAt - b.createdAt);
|
|
370
|
-
const contextMessages = this.processMessagesWithContext(allThreadMessages, include);
|
|
371
|
-
const includedMessages = contextMessages.map((row) => this.normalizeMessage(row));
|
|
372
|
-
for (const includeMsg of includedMessages) {
|
|
373
|
-
if (!messageIds.has(includeMsg.id)) {
|
|
374
|
-
messages.push(includeMsg);
|
|
375
|
-
messageIds.add(includeMsg.id);
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
const list = new MessageList().add(messages, "memory");
|
|
380
|
-
let finalMessages = list.get.all.db();
|
|
381
|
-
finalMessages = finalMessages.sort((a, b) => {
|
|
382
|
-
const aValue = field === "createdAt" ? new Date(a.createdAt).getTime() : a[field];
|
|
383
|
-
const bValue = field === "createdAt" ? new Date(b.createdAt).getTime() : b[field];
|
|
384
|
-
if (aValue == null && bValue == null) return 0;
|
|
385
|
-
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
386
|
-
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
387
|
-
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
388
|
-
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
389
|
-
}
|
|
390
|
-
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
391
324
|
});
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
return {
|
|
397
|
-
messages: finalMessages,
|
|
398
|
-
total,
|
|
399
|
-
page,
|
|
400
|
-
perPage: perPageForResponse,
|
|
401
|
-
hasMore
|
|
402
|
-
};
|
|
325
|
+
if (columnsToAdd.length > 0) {
|
|
326
|
+
await table.addColumns(columnsToAdd);
|
|
327
|
+
this.logger?.info?.(`Added columns [${columnsToAdd.map((c) => c.name).join(", ")}] to table ${tableName}`);
|
|
328
|
+
}
|
|
403
329
|
} catch (error) {
|
|
404
|
-
|
|
330
|
+
throw new MastraError(
|
|
405
331
|
{
|
|
406
|
-
id: "
|
|
332
|
+
id: createStorageErrorId("LANCE", "ALTER_TABLE", "FAILED"),
|
|
407
333
|
domain: ErrorDomain.STORAGE,
|
|
408
334
|
category: ErrorCategory.THIRD_PARTY,
|
|
409
|
-
details: {
|
|
410
|
-
threadId,
|
|
411
|
-
resourceId: resourceId ?? ""
|
|
412
|
-
}
|
|
335
|
+
details: { tableName }
|
|
413
336
|
},
|
|
414
337
|
error
|
|
415
338
|
);
|
|
416
|
-
this.logger?.error?.(mastraError.toString());
|
|
417
|
-
this.logger?.trackException?.(mastraError);
|
|
418
|
-
return {
|
|
419
|
-
messages: [],
|
|
420
|
-
total: 0,
|
|
421
|
-
page,
|
|
422
|
-
perPage: perPageForResponse,
|
|
423
|
-
hasMore: false
|
|
424
|
-
};
|
|
425
339
|
}
|
|
426
340
|
}
|
|
427
|
-
async
|
|
341
|
+
async clearTable({ tableName }) {
|
|
428
342
|
try {
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
return { messages: [] };
|
|
432
|
-
}
|
|
433
|
-
const threadId = messages[0]?.threadId;
|
|
434
|
-
if (!threadId) {
|
|
435
|
-
throw new Error("Thread ID is required");
|
|
343
|
+
if (!this.client) {
|
|
344
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
436
345
|
}
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
throw new Error("Message ID is required");
|
|
440
|
-
}
|
|
441
|
-
if (!message.threadId) {
|
|
442
|
-
throw new Error("Thread ID is required for all messages");
|
|
443
|
-
}
|
|
444
|
-
if (message.resourceId === null || message.resourceId === void 0) {
|
|
445
|
-
throw new Error("Resource ID cannot be null or undefined");
|
|
446
|
-
}
|
|
447
|
-
if (!message.content) {
|
|
448
|
-
throw new Error("Message content is required");
|
|
449
|
-
}
|
|
346
|
+
if (!tableName) {
|
|
347
|
+
throw new Error("tableName is required for clearTable.");
|
|
450
348
|
}
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
const
|
|
465
|
-
await
|
|
466
|
-
const list = new MessageList().add(messages, "memory");
|
|
467
|
-
return { messages: list.get.all.db() };
|
|
349
|
+
} catch (validationError) {
|
|
350
|
+
throw new MastraError(
|
|
351
|
+
{
|
|
352
|
+
id: createStorageErrorId("LANCE", "CLEAR_TABLE", "INVALID_ARGS"),
|
|
353
|
+
domain: ErrorDomain.STORAGE,
|
|
354
|
+
category: ErrorCategory.USER,
|
|
355
|
+
text: validationError.message,
|
|
356
|
+
details: { tableName }
|
|
357
|
+
},
|
|
358
|
+
validationError
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
try {
|
|
362
|
+
const table = await this.client.openTable(tableName);
|
|
363
|
+
await table.delete("1=1");
|
|
468
364
|
} catch (error) {
|
|
469
365
|
throw new MastraError(
|
|
470
366
|
{
|
|
471
|
-
id: "
|
|
367
|
+
id: createStorageErrorId("LANCE", "CLEAR_TABLE", "FAILED"),
|
|
472
368
|
domain: ErrorDomain.STORAGE,
|
|
473
|
-
category: ErrorCategory.THIRD_PARTY
|
|
369
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
370
|
+
details: { tableName }
|
|
474
371
|
},
|
|
475
372
|
error
|
|
476
373
|
);
|
|
477
374
|
}
|
|
478
375
|
}
|
|
479
|
-
async
|
|
376
|
+
async insert({ tableName, record }) {
|
|
480
377
|
try {
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
if (page < 0) {
|
|
484
|
-
throw new MastraError(
|
|
485
|
-
{
|
|
486
|
-
id: "STORAGE_LANCE_LIST_THREADS_BY_RESOURCE_ID_INVALID_PAGE",
|
|
487
|
-
domain: ErrorDomain.STORAGE,
|
|
488
|
-
category: ErrorCategory.USER,
|
|
489
|
-
details: { page }
|
|
490
|
-
},
|
|
491
|
-
new Error("page must be >= 0")
|
|
492
|
-
);
|
|
378
|
+
if (!this.client) {
|
|
379
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
493
380
|
}
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
});
|
|
511
|
-
const paginatedRecords = records.slice(offset, offset + perPage);
|
|
512
|
-
const schema = await getTableSchema({ tableName: TABLE_THREADS, client: this.client });
|
|
513
|
-
const threads = paginatedRecords.map(
|
|
514
|
-
(record) => processResultWithTypeConversion(record, schema)
|
|
381
|
+
if (!tableName) {
|
|
382
|
+
throw new Error("tableName is required for insert.");
|
|
383
|
+
}
|
|
384
|
+
if (!record || Object.keys(record).length === 0) {
|
|
385
|
+
throw new Error("record is required and cannot be empty for insert.");
|
|
386
|
+
}
|
|
387
|
+
} catch (validationError) {
|
|
388
|
+
throw new MastraError(
|
|
389
|
+
{
|
|
390
|
+
id: createStorageErrorId("LANCE", "INSERT", "INVALID_ARGS"),
|
|
391
|
+
domain: ErrorDomain.STORAGE,
|
|
392
|
+
category: ErrorCategory.USER,
|
|
393
|
+
text: validationError.message,
|
|
394
|
+
details: { tableName }
|
|
395
|
+
},
|
|
396
|
+
validationError
|
|
515
397
|
);
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
398
|
+
}
|
|
399
|
+
try {
|
|
400
|
+
const table = await this.client.openTable(tableName);
|
|
401
|
+
const primaryId = getPrimaryKeys(tableName);
|
|
402
|
+
const processedRecord = { ...record };
|
|
403
|
+
for (const key in processedRecord) {
|
|
404
|
+
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
405
|
+
this.logger.debug("Converting object to JSON string: ", processedRecord[key]);
|
|
406
|
+
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([processedRecord]);
|
|
523
410
|
} catch (error) {
|
|
524
411
|
throw new MastraError(
|
|
525
412
|
{
|
|
526
|
-
id: "
|
|
413
|
+
id: createStorageErrorId("LANCE", "INSERT", "FAILED"),
|
|
527
414
|
domain: ErrorDomain.STORAGE,
|
|
528
|
-
category: ErrorCategory.THIRD_PARTY
|
|
415
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
416
|
+
details: { tableName }
|
|
529
417
|
},
|
|
530
418
|
error
|
|
531
419
|
);
|
|
532
420
|
}
|
|
533
421
|
}
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
* @returns The processed array with context messages included
|
|
539
|
-
*/
|
|
540
|
-
processMessagesWithContext(records, include) {
|
|
541
|
-
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
542
|
-
if (messagesWithContext.length === 0) {
|
|
543
|
-
return records;
|
|
544
|
-
}
|
|
545
|
-
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
546
|
-
records.forEach((message, index) => {
|
|
547
|
-
messageIndexMap.set(message.id, index);
|
|
548
|
-
});
|
|
549
|
-
const additionalIndices = /* @__PURE__ */ new Set();
|
|
550
|
-
for (const item of messagesWithContext) {
|
|
551
|
-
const messageIndex = messageIndexMap.get(item.id);
|
|
552
|
-
if (messageIndex !== void 0) {
|
|
553
|
-
if (item.withPreviousMessages) {
|
|
554
|
-
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
555
|
-
for (let i = startIdx; i < messageIndex; i++) {
|
|
556
|
-
additionalIndices.add(i);
|
|
557
|
-
}
|
|
558
|
-
}
|
|
559
|
-
if (item.withNextMessages) {
|
|
560
|
-
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
561
|
-
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
562
|
-
additionalIndices.add(i);
|
|
563
|
-
}
|
|
564
|
-
}
|
|
422
|
+
async batchInsert({ tableName, records }) {
|
|
423
|
+
try {
|
|
424
|
+
if (!this.client) {
|
|
425
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
565
426
|
}
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
return records;
|
|
569
|
-
}
|
|
570
|
-
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
571
|
-
const allIndices = /* @__PURE__ */ new Set();
|
|
572
|
-
records.forEach((record, index) => {
|
|
573
|
-
if (originalMatchIds.has(record.id)) {
|
|
574
|
-
allIndices.add(index);
|
|
427
|
+
if (!tableName) {
|
|
428
|
+
throw new Error("tableName is required for batchInsert.");
|
|
575
429
|
}
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
})
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
content: typeof data.content === "string" ? (() => {
|
|
591
|
-
try {
|
|
592
|
-
return JSON.parse(data.content);
|
|
593
|
-
} catch {
|
|
594
|
-
return data.content;
|
|
595
|
-
}
|
|
596
|
-
})() : data.content,
|
|
597
|
-
createdAt: new Date(data.createdAt),
|
|
598
|
-
updatedAt: new Date(data.updatedAt)
|
|
599
|
-
};
|
|
600
|
-
}
|
|
601
|
-
async updateMessages(args) {
|
|
602
|
-
const { messages } = args;
|
|
603
|
-
this.logger.debug("Updating messages", { count: messages.length });
|
|
604
|
-
if (!messages.length) {
|
|
605
|
-
return [];
|
|
430
|
+
if (!records || records.length === 0) {
|
|
431
|
+
throw new Error("records array is required and cannot be empty for batchInsert.");
|
|
432
|
+
}
|
|
433
|
+
} catch (validationError) {
|
|
434
|
+
throw new MastraError(
|
|
435
|
+
{
|
|
436
|
+
id: createStorageErrorId("LANCE", "BATCH_INSERT", "INVALID_ARGS"),
|
|
437
|
+
domain: ErrorDomain.STORAGE,
|
|
438
|
+
category: ErrorCategory.USER,
|
|
439
|
+
text: validationError.message,
|
|
440
|
+
details: { tableName }
|
|
441
|
+
},
|
|
442
|
+
validationError
|
|
443
|
+
);
|
|
606
444
|
}
|
|
607
|
-
const updatedMessages = [];
|
|
608
|
-
const affectedThreadIds = /* @__PURE__ */ new Set();
|
|
609
445
|
try {
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
continue;
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
const originalThreadId = existingMsg.threadId;
|
|
619
|
-
affectedThreadIds.add(originalThreadId);
|
|
620
|
-
const updatePayload = {};
|
|
621
|
-
if ("role" in updates && updates.role !== void 0) updatePayload.role = updates.role;
|
|
622
|
-
if ("type" in updates && updates.type !== void 0) updatePayload.type = updates.type;
|
|
623
|
-
if ("resourceId" in updates && updates.resourceId !== void 0) updatePayload.resourceId = updates.resourceId;
|
|
624
|
-
if ("threadId" in updates && updates.threadId !== void 0 && updates.threadId !== null) {
|
|
625
|
-
updatePayload.thread_id = updates.threadId;
|
|
626
|
-
affectedThreadIds.add(updates.threadId);
|
|
627
|
-
}
|
|
628
|
-
if (updates.content) {
|
|
629
|
-
const existingContent = existingMsg.content;
|
|
630
|
-
let newContent = { ...existingContent };
|
|
631
|
-
if (updates.content.metadata !== void 0) {
|
|
632
|
-
newContent.metadata = {
|
|
633
|
-
...existingContent.metadata || {},
|
|
634
|
-
...updates.content.metadata || {}
|
|
635
|
-
};
|
|
636
|
-
}
|
|
637
|
-
if (updates.content.content !== void 0) {
|
|
638
|
-
newContent.content = updates.content.content;
|
|
639
|
-
}
|
|
640
|
-
if ("parts" in updates.content && updates.content.parts !== void 0) {
|
|
641
|
-
newContent.parts = updates.content.parts;
|
|
446
|
+
const table = await this.client.openTable(tableName);
|
|
447
|
+
const primaryId = getPrimaryKeys(tableName);
|
|
448
|
+
const processedRecords = records.map((record) => {
|
|
449
|
+
const processedRecord = { ...record };
|
|
450
|
+
for (const key in processedRecord) {
|
|
451
|
+
if (processedRecord[key] == null) continue;
|
|
452
|
+
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
453
|
+
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
642
454
|
}
|
|
643
|
-
updatePayload.content = JSON.stringify(newContent);
|
|
644
|
-
}
|
|
645
|
-
await this.operations.insert({ tableName: TABLE_MESSAGES, record: { id, ...updatePayload } });
|
|
646
|
-
const updatedMessage = await this.operations.load({ tableName: TABLE_MESSAGES, keys: { id } });
|
|
647
|
-
if (updatedMessage) {
|
|
648
|
-
updatedMessages.push(this.parseMessageData(updatedMessage));
|
|
649
455
|
}
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
tableName: TABLE_THREADS,
|
|
654
|
-
record: { id: threadId, updatedAt: Date.now() }
|
|
655
|
-
});
|
|
656
|
-
}
|
|
657
|
-
return updatedMessages;
|
|
456
|
+
return processedRecord;
|
|
457
|
+
});
|
|
458
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(processedRecords);
|
|
658
459
|
} catch (error) {
|
|
659
460
|
throw new MastraError(
|
|
660
461
|
{
|
|
661
|
-
id: "
|
|
462
|
+
id: createStorageErrorId("LANCE", "BATCH_INSERT", "FAILED"),
|
|
662
463
|
domain: ErrorDomain.STORAGE,
|
|
663
464
|
category: ErrorCategory.THIRD_PARTY,
|
|
664
|
-
details: {
|
|
465
|
+
details: { tableName }
|
|
665
466
|
},
|
|
666
467
|
error
|
|
667
468
|
);
|
|
668
469
|
}
|
|
669
470
|
}
|
|
670
|
-
async
|
|
471
|
+
async load({ tableName, keys }) {
|
|
671
472
|
try {
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
return null;
|
|
675
|
-
}
|
|
676
|
-
let createdAt;
|
|
677
|
-
let updatedAt;
|
|
678
|
-
try {
|
|
679
|
-
if (resource.createdAt instanceof Date) {
|
|
680
|
-
createdAt = resource.createdAt;
|
|
681
|
-
} else if (typeof resource.createdAt === "string") {
|
|
682
|
-
createdAt = new Date(resource.createdAt);
|
|
683
|
-
} else if (typeof resource.createdAt === "number") {
|
|
684
|
-
createdAt = new Date(resource.createdAt);
|
|
685
|
-
} else {
|
|
686
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
687
|
-
}
|
|
688
|
-
if (isNaN(createdAt.getTime())) {
|
|
689
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
690
|
-
}
|
|
691
|
-
} catch {
|
|
692
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
693
|
-
}
|
|
694
|
-
try {
|
|
695
|
-
if (resource.updatedAt instanceof Date) {
|
|
696
|
-
updatedAt = resource.updatedAt;
|
|
697
|
-
} else if (typeof resource.updatedAt === "string") {
|
|
698
|
-
updatedAt = new Date(resource.updatedAt);
|
|
699
|
-
} else if (typeof resource.updatedAt === "number") {
|
|
700
|
-
updatedAt = new Date(resource.updatedAt);
|
|
701
|
-
} else {
|
|
702
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
703
|
-
}
|
|
704
|
-
if (isNaN(updatedAt.getTime())) {
|
|
705
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
706
|
-
}
|
|
707
|
-
} catch {
|
|
708
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
473
|
+
if (!this.client) {
|
|
474
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
709
475
|
}
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
workingMemory = void 0;
|
|
713
|
-
} else if (workingMemory === "") {
|
|
714
|
-
workingMemory = "";
|
|
715
|
-
} else if (typeof workingMemory === "object") {
|
|
716
|
-
workingMemory = JSON.stringify(workingMemory);
|
|
476
|
+
if (!tableName) {
|
|
477
|
+
throw new Error("tableName is required for load.");
|
|
717
478
|
}
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
metadata = void 0;
|
|
721
|
-
} else if (typeof metadata === "string") {
|
|
722
|
-
try {
|
|
723
|
-
metadata = JSON.parse(metadata);
|
|
724
|
-
} catch {
|
|
725
|
-
metadata = metadata;
|
|
726
|
-
}
|
|
479
|
+
if (!keys || Object.keys(keys).length === 0) {
|
|
480
|
+
throw new Error("keys are required and cannot be empty for load.");
|
|
727
481
|
}
|
|
728
|
-
|
|
729
|
-
...resource,
|
|
730
|
-
createdAt,
|
|
731
|
-
updatedAt,
|
|
732
|
-
workingMemory,
|
|
733
|
-
metadata
|
|
734
|
-
};
|
|
735
|
-
} catch (error) {
|
|
482
|
+
} catch (validationError) {
|
|
736
483
|
throw new MastraError(
|
|
737
484
|
{
|
|
738
|
-
id: "
|
|
485
|
+
id: createStorageErrorId("LANCE", "LOAD", "INVALID_ARGS"),
|
|
739
486
|
domain: ErrorDomain.STORAGE,
|
|
740
|
-
category: ErrorCategory.
|
|
487
|
+
category: ErrorCategory.USER,
|
|
488
|
+
text: validationError.message,
|
|
489
|
+
details: { tableName }
|
|
741
490
|
},
|
|
742
|
-
|
|
491
|
+
validationError
|
|
743
492
|
);
|
|
744
493
|
}
|
|
745
|
-
}
|
|
746
|
-
async saveResource({ resource }) {
|
|
747
494
|
try {
|
|
748
|
-
const
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
495
|
+
const table = await this.client.openTable(tableName);
|
|
496
|
+
const tableSchema = await getTableSchema({ tableName, client: this.client });
|
|
497
|
+
const query = table.query();
|
|
498
|
+
if (Object.keys(keys).length > 0) {
|
|
499
|
+
validateKeyTypes(keys, tableSchema);
|
|
500
|
+
const filterConditions = Object.entries(keys).map(([key, value]) => {
|
|
501
|
+
const isCamelCase = /^[a-z][a-zA-Z]*$/.test(key) && /[A-Z]/.test(key);
|
|
502
|
+
const quotedKey = isCamelCase ? `\`${key}\`` : key;
|
|
503
|
+
if (typeof value === "string") {
|
|
504
|
+
return `${quotedKey} = '${value}'`;
|
|
505
|
+
} else if (value === null) {
|
|
506
|
+
return `${quotedKey} IS NULL`;
|
|
507
|
+
} else {
|
|
508
|
+
return `${quotedKey} = ${value}`;
|
|
509
|
+
}
|
|
510
|
+
}).join(" AND ");
|
|
511
|
+
this.logger.debug("where clause generated: " + filterConditions);
|
|
512
|
+
query.where(filterConditions);
|
|
513
|
+
}
|
|
514
|
+
const result = await query.limit(1).toArray();
|
|
515
|
+
if (result.length === 0) {
|
|
516
|
+
this.logger.debug("No record found");
|
|
517
|
+
return null;
|
|
518
|
+
}
|
|
519
|
+
return processResultWithTypeConversion(result[0], tableSchema);
|
|
759
520
|
} catch (error) {
|
|
521
|
+
if (error instanceof MastraError) throw error;
|
|
760
522
|
throw new MastraError(
|
|
761
523
|
{
|
|
762
|
-
id: "
|
|
524
|
+
id: createStorageErrorId("LANCE", "LOAD", "FAILED"),
|
|
763
525
|
domain: ErrorDomain.STORAGE,
|
|
764
|
-
category: ErrorCategory.THIRD_PARTY
|
|
526
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
527
|
+
details: { tableName, keyCount: Object.keys(keys).length, firstKey: Object.keys(keys)[0] ?? "" }
|
|
765
528
|
},
|
|
766
529
|
error
|
|
767
530
|
);
|
|
768
531
|
}
|
|
769
532
|
}
|
|
770
|
-
async updateResource({
|
|
771
|
-
resourceId,
|
|
772
|
-
workingMemory,
|
|
773
|
-
metadata
|
|
774
|
-
}) {
|
|
775
|
-
const maxRetries = 3;
|
|
776
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
777
|
-
try {
|
|
778
|
-
const existingResource = await this.getResourceById({ resourceId });
|
|
779
|
-
if (!existingResource) {
|
|
780
|
-
const newResource = {
|
|
781
|
-
id: resourceId,
|
|
782
|
-
workingMemory,
|
|
783
|
-
metadata: metadata || {},
|
|
784
|
-
createdAt: /* @__PURE__ */ new Date(),
|
|
785
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
786
|
-
};
|
|
787
|
-
return this.saveResource({ resource: newResource });
|
|
788
|
-
}
|
|
789
|
-
const updatedResource = {
|
|
790
|
-
...existingResource,
|
|
791
|
-
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
792
|
-
metadata: {
|
|
793
|
-
...existingResource.metadata,
|
|
794
|
-
...metadata
|
|
795
|
-
},
|
|
796
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
797
|
-
};
|
|
798
|
-
const record = {
|
|
799
|
-
id: resourceId,
|
|
800
|
-
workingMemory: updatedResource.workingMemory || "",
|
|
801
|
-
metadata: updatedResource.metadata ? JSON.stringify(updatedResource.metadata) : "",
|
|
802
|
-
updatedAt: updatedResource.updatedAt.getTime()
|
|
803
|
-
// Store as timestamp (milliseconds)
|
|
804
|
-
};
|
|
805
|
-
const table = await this.client.openTable(TABLE_RESOURCES);
|
|
806
|
-
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
807
|
-
return updatedResource;
|
|
808
|
-
} catch (error) {
|
|
809
|
-
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
810
|
-
const delay = Math.pow(2, attempt) * 10;
|
|
811
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
812
|
-
continue;
|
|
813
|
-
}
|
|
814
|
-
throw new MastraError(
|
|
815
|
-
{
|
|
816
|
-
id: "LANCE_STORE_UPDATE_RESOURCE_FAILED",
|
|
817
|
-
domain: ErrorDomain.STORAGE,
|
|
818
|
-
category: ErrorCategory.THIRD_PARTY
|
|
819
|
-
},
|
|
820
|
-
error
|
|
821
|
-
);
|
|
822
|
-
}
|
|
823
|
-
}
|
|
824
|
-
throw new Error("Unexpected end of retry loop");
|
|
825
|
-
}
|
|
826
533
|
};
|
|
827
|
-
|
|
534
|
+
|
|
535
|
+
// src/storage/domains/memory/index.ts
|
|
536
|
+
var StoreMemoryLance = class extends MemoryStorage {
|
|
828
537
|
client;
|
|
829
|
-
|
|
538
|
+
#db;
|
|
539
|
+
constructor(config) {
|
|
830
540
|
super();
|
|
541
|
+
const client = resolveLanceConfig(config);
|
|
831
542
|
this.client = client;
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
case "jsonb":
|
|
843
|
-
return "'{}'";
|
|
844
|
-
case "uuid":
|
|
845
|
-
return "''";
|
|
846
|
-
default:
|
|
847
|
-
return super.getDefaultValue(type);
|
|
848
|
-
}
|
|
849
|
-
}
|
|
850
|
-
async hasColumn(tableName, columnName) {
|
|
851
|
-
const table = await this.client.openTable(tableName);
|
|
852
|
-
const schema = await table.schema();
|
|
853
|
-
return schema.fields.some((field) => field.name === columnName);
|
|
854
|
-
}
|
|
855
|
-
translateSchema(schema) {
|
|
856
|
-
const fields = Object.entries(schema).map(([name, column]) => {
|
|
857
|
-
let arrowType;
|
|
858
|
-
switch (column.type.toLowerCase()) {
|
|
859
|
-
case "text":
|
|
860
|
-
case "uuid":
|
|
861
|
-
arrowType = new Utf8();
|
|
862
|
-
break;
|
|
863
|
-
case "int":
|
|
864
|
-
case "integer":
|
|
865
|
-
arrowType = new Int32();
|
|
866
|
-
break;
|
|
867
|
-
case "bigint":
|
|
868
|
-
arrowType = new Float64();
|
|
869
|
-
break;
|
|
870
|
-
case "float":
|
|
871
|
-
arrowType = new Float32();
|
|
872
|
-
break;
|
|
873
|
-
case "jsonb":
|
|
874
|
-
case "json":
|
|
875
|
-
arrowType = new Utf8();
|
|
876
|
-
break;
|
|
877
|
-
case "binary":
|
|
878
|
-
arrowType = new Binary();
|
|
879
|
-
break;
|
|
880
|
-
case "timestamp":
|
|
881
|
-
arrowType = new Float64();
|
|
882
|
-
break;
|
|
883
|
-
default:
|
|
884
|
-
arrowType = new Utf8();
|
|
885
|
-
}
|
|
886
|
-
return new Field(name, arrowType, column.nullable ?? true);
|
|
543
|
+
this.#db = new LanceDB({ client });
|
|
544
|
+
}
|
|
545
|
+
async init() {
|
|
546
|
+
await this.#db.createTable({ tableName: TABLE_THREADS, schema: TABLE_SCHEMAS[TABLE_THREADS] });
|
|
547
|
+
await this.#db.createTable({ tableName: TABLE_MESSAGES, schema: TABLE_SCHEMAS[TABLE_MESSAGES] });
|
|
548
|
+
await this.#db.createTable({ tableName: TABLE_RESOURCES, schema: TABLE_SCHEMAS[TABLE_RESOURCES] });
|
|
549
|
+
await this.#db.alterTable({
|
|
550
|
+
tableName: TABLE_MESSAGES,
|
|
551
|
+
schema: TABLE_SCHEMAS[TABLE_MESSAGES],
|
|
552
|
+
ifNotExists: ["resourceId"]
|
|
887
553
|
});
|
|
888
|
-
return new Schema(fields);
|
|
889
554
|
}
|
|
890
|
-
async
|
|
891
|
-
tableName
|
|
892
|
-
|
|
893
|
-
|
|
555
|
+
async dangerouslyClearAll() {
|
|
556
|
+
await this.#db.clearTable({ tableName: TABLE_THREADS });
|
|
557
|
+
await this.#db.clearTable({ tableName: TABLE_MESSAGES });
|
|
558
|
+
await this.#db.clearTable({ tableName: TABLE_RESOURCES });
|
|
559
|
+
}
|
|
560
|
+
async deleteMessages(messageIds) {
|
|
561
|
+
if (!messageIds || messageIds.length === 0) {
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
this.logger.debug("Deleting messages", { count: messageIds.length });
|
|
894
565
|
try {
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
566
|
+
const threadIds = /* @__PURE__ */ new Set();
|
|
567
|
+
for (const messageId of messageIds) {
|
|
568
|
+
const message = await this.#db.load({ tableName: TABLE_MESSAGES, keys: { id: messageId } });
|
|
569
|
+
if (message?.thread_id) {
|
|
570
|
+
threadIds.add(message.thread_id);
|
|
571
|
+
}
|
|
900
572
|
}
|
|
901
|
-
|
|
902
|
-
|
|
573
|
+
const messagesTable = await this.client.openTable(TABLE_MESSAGES);
|
|
574
|
+
const idConditions = messageIds.map((id) => `id = '${this.escapeSql(id)}'`).join(" OR ");
|
|
575
|
+
await messagesTable.delete(idConditions);
|
|
576
|
+
const now = (/* @__PURE__ */ new Date()).getTime();
|
|
577
|
+
const threadsTable = await this.client.openTable(TABLE_THREADS);
|
|
578
|
+
for (const threadId of threadIds) {
|
|
579
|
+
const thread = await this.getThreadById({ threadId });
|
|
580
|
+
if (thread) {
|
|
581
|
+
const record = {
|
|
582
|
+
id: threadId,
|
|
583
|
+
resourceId: thread.resourceId,
|
|
584
|
+
title: thread.title,
|
|
585
|
+
metadata: JSON.stringify(thread.metadata),
|
|
586
|
+
createdAt: new Date(thread.createdAt).getTime(),
|
|
587
|
+
updatedAt: now
|
|
588
|
+
};
|
|
589
|
+
await threadsTable.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
590
|
+
}
|
|
903
591
|
}
|
|
904
592
|
} catch (error) {
|
|
905
593
|
throw new MastraError(
|
|
906
594
|
{
|
|
907
|
-
id: "
|
|
595
|
+
id: createStorageErrorId("LANCE", "DELETE_MESSAGES", "FAILED"),
|
|
908
596
|
domain: ErrorDomain.STORAGE,
|
|
909
|
-
category: ErrorCategory.
|
|
910
|
-
details: {
|
|
597
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
598
|
+
details: { count: messageIds.length }
|
|
911
599
|
},
|
|
912
600
|
error
|
|
913
601
|
);
|
|
914
602
|
}
|
|
603
|
+
}
|
|
604
|
+
// Utility to escape single quotes in SQL strings
|
|
605
|
+
escapeSql(str) {
|
|
606
|
+
return str.replace(/'/g, "''");
|
|
607
|
+
}
|
|
608
|
+
async getThreadById({ threadId }) {
|
|
915
609
|
try {
|
|
916
|
-
const
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
if (error.message?.includes("already exists")) {
|
|
920
|
-
this.logger.debug(`Table '${tableName}' already exists, skipping create`);
|
|
921
|
-
return;
|
|
610
|
+
const thread = await this.#db.load({ tableName: TABLE_THREADS, keys: { id: threadId } });
|
|
611
|
+
if (!thread) {
|
|
612
|
+
return null;
|
|
922
613
|
}
|
|
614
|
+
return {
|
|
615
|
+
...thread,
|
|
616
|
+
createdAt: new Date(thread.createdAt),
|
|
617
|
+
updatedAt: new Date(thread.updatedAt)
|
|
618
|
+
};
|
|
619
|
+
} catch (error) {
|
|
923
620
|
throw new MastraError(
|
|
924
621
|
{
|
|
925
|
-
id: "
|
|
622
|
+
id: createStorageErrorId("LANCE", "GET_THREAD_BY_ID", "FAILED"),
|
|
926
623
|
domain: ErrorDomain.STORAGE,
|
|
927
|
-
category: ErrorCategory.THIRD_PARTY
|
|
928
|
-
details: { tableName }
|
|
624
|
+
category: ErrorCategory.THIRD_PARTY
|
|
929
625
|
},
|
|
930
626
|
error
|
|
931
627
|
);
|
|
932
628
|
}
|
|
933
629
|
}
|
|
934
|
-
|
|
630
|
+
/**
|
|
631
|
+
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
632
|
+
* @param thread - The thread to save
|
|
633
|
+
* @returns The saved thread
|
|
634
|
+
*/
|
|
635
|
+
async saveThread({ thread }) {
|
|
935
636
|
try {
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
}
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
}
|
|
942
|
-
} catch (validationError) {
|
|
637
|
+
const record = { ...thread, metadata: JSON.stringify(thread.metadata) };
|
|
638
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
639
|
+
await table.add([record], { mode: "append" });
|
|
640
|
+
return thread;
|
|
641
|
+
} catch (error) {
|
|
943
642
|
throw new MastraError(
|
|
944
643
|
{
|
|
945
|
-
id: "
|
|
644
|
+
id: createStorageErrorId("LANCE", "SAVE_THREAD", "FAILED"),
|
|
946
645
|
domain: ErrorDomain.STORAGE,
|
|
947
|
-
category: ErrorCategory.
|
|
948
|
-
text: validationError.message,
|
|
949
|
-
details: { tableName }
|
|
646
|
+
category: ErrorCategory.THIRD_PARTY
|
|
950
647
|
},
|
|
951
|
-
|
|
648
|
+
error
|
|
952
649
|
);
|
|
953
650
|
}
|
|
651
|
+
}
|
|
652
|
+
async updateThread({
|
|
653
|
+
id,
|
|
654
|
+
title,
|
|
655
|
+
metadata
|
|
656
|
+
}) {
|
|
657
|
+
const maxRetries = 5;
|
|
658
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
659
|
+
try {
|
|
660
|
+
const current = await this.getThreadById({ threadId: id });
|
|
661
|
+
if (!current) {
|
|
662
|
+
throw new Error(`Thread with id ${id} not found`);
|
|
663
|
+
}
|
|
664
|
+
const mergedMetadata = { ...current.metadata, ...metadata };
|
|
665
|
+
const record = {
|
|
666
|
+
id,
|
|
667
|
+
title,
|
|
668
|
+
metadata: JSON.stringify(mergedMetadata),
|
|
669
|
+
updatedAt: (/* @__PURE__ */ new Date()).getTime()
|
|
670
|
+
};
|
|
671
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
672
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
673
|
+
const updatedThread = await this.getThreadById({ threadId: id });
|
|
674
|
+
if (!updatedThread) {
|
|
675
|
+
throw new Error(`Failed to retrieve updated thread ${id}`);
|
|
676
|
+
}
|
|
677
|
+
return updatedThread;
|
|
678
|
+
} catch (error) {
|
|
679
|
+
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
680
|
+
const delay = Math.pow(2, attempt) * 10;
|
|
681
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
682
|
+
continue;
|
|
683
|
+
}
|
|
684
|
+
throw new MastraError(
|
|
685
|
+
{
|
|
686
|
+
id: createStorageErrorId("LANCE", "UPDATE_THREAD", "FAILED"),
|
|
687
|
+
domain: ErrorDomain.STORAGE,
|
|
688
|
+
category: ErrorCategory.THIRD_PARTY
|
|
689
|
+
},
|
|
690
|
+
error
|
|
691
|
+
);
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
throw new MastraError(
|
|
695
|
+
{
|
|
696
|
+
id: createStorageErrorId("LANCE", "UPDATE_THREAD", "FAILED"),
|
|
697
|
+
domain: ErrorDomain.STORAGE,
|
|
698
|
+
category: ErrorCategory.THIRD_PARTY
|
|
699
|
+
},
|
|
700
|
+
new Error("All retries exhausted")
|
|
701
|
+
);
|
|
702
|
+
}
|
|
703
|
+
async deleteThread({ threadId }) {
|
|
954
704
|
try {
|
|
955
|
-
await this.client.
|
|
705
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
706
|
+
await table.delete(`id = '${threadId}'`);
|
|
707
|
+
const messagesTable = await this.client.openTable(TABLE_MESSAGES);
|
|
708
|
+
await messagesTable.delete(`thread_id = '${threadId}'`);
|
|
956
709
|
} catch (error) {
|
|
957
|
-
if (error.toString().includes("was not found") || error.message?.includes("Table not found")) {
|
|
958
|
-
this.logger.debug(`Table '${tableName}' does not exist, skipping drop`);
|
|
959
|
-
return;
|
|
960
|
-
}
|
|
961
710
|
throw new MastraError(
|
|
962
711
|
{
|
|
963
|
-
id: "
|
|
712
|
+
id: createStorageErrorId("LANCE", "DELETE_THREAD", "FAILED"),
|
|
964
713
|
domain: ErrorDomain.STORAGE,
|
|
965
|
-
category: ErrorCategory.THIRD_PARTY
|
|
966
|
-
details: { tableName }
|
|
714
|
+
category: ErrorCategory.THIRD_PARTY
|
|
967
715
|
},
|
|
968
716
|
error
|
|
969
717
|
);
|
|
970
718
|
}
|
|
971
719
|
}
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
720
|
+
normalizeMessage(message) {
|
|
721
|
+
const { thread_id, ...rest } = message;
|
|
722
|
+
return {
|
|
723
|
+
...rest,
|
|
724
|
+
threadId: thread_id,
|
|
725
|
+
content: typeof message.content === "string" ? (() => {
|
|
726
|
+
try {
|
|
727
|
+
return JSON.parse(message.content);
|
|
728
|
+
} catch {
|
|
729
|
+
return message.content;
|
|
730
|
+
}
|
|
731
|
+
})() : message.content
|
|
732
|
+
};
|
|
733
|
+
}
|
|
734
|
+
async listMessagesById({ messageIds }) {
|
|
735
|
+
if (messageIds.length === 0) return { messages: [] };
|
|
977
736
|
try {
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
}
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
} catch (validationError) {
|
|
737
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
738
|
+
const quotedIds = messageIds.map((id) => `'${id}'`).join(", ");
|
|
739
|
+
const allRecords = await table.query().where(`id IN (${quotedIds})`).toArray();
|
|
740
|
+
const messages = processResultWithTypeConversion(
|
|
741
|
+
allRecords,
|
|
742
|
+
await getTableSchema({ tableName: TABLE_MESSAGES, client: this.client })
|
|
743
|
+
);
|
|
744
|
+
const list = new MessageList().add(
|
|
745
|
+
messages.map(this.normalizeMessage),
|
|
746
|
+
"memory"
|
|
747
|
+
);
|
|
748
|
+
return { messages: list.get.all.db() };
|
|
749
|
+
} catch (error) {
|
|
992
750
|
throw new MastraError(
|
|
993
751
|
{
|
|
994
|
-
id: "
|
|
752
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES_BY_ID", "FAILED"),
|
|
995
753
|
domain: ErrorDomain.STORAGE,
|
|
996
|
-
category: ErrorCategory.
|
|
997
|
-
|
|
998
|
-
|
|
754
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
755
|
+
details: {
|
|
756
|
+
messageIds: JSON.stringify(messageIds)
|
|
757
|
+
}
|
|
999
758
|
},
|
|
1000
|
-
|
|
759
|
+
error
|
|
1001
760
|
);
|
|
1002
761
|
}
|
|
762
|
+
}
|
|
763
|
+
async listMessages(args) {
|
|
764
|
+
const { threadId, resourceId, include, filter, perPage: perPageInput, page = 0, orderBy } = args;
|
|
765
|
+
const threadIds = Array.isArray(threadId) ? threadId : [threadId];
|
|
766
|
+
if (threadIds.length === 0 || threadIds.some((id) => !id.trim())) {
|
|
767
|
+
throw new MastraError(
|
|
768
|
+
{
|
|
769
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES", "INVALID_THREAD_ID"),
|
|
770
|
+
domain: ErrorDomain.STORAGE,
|
|
771
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
772
|
+
details: { threadId: Array.isArray(threadId) ? threadId.join(",") : threadId }
|
|
773
|
+
},
|
|
774
|
+
new Error("threadId must be a non-empty string or array of non-empty strings")
|
|
775
|
+
);
|
|
776
|
+
}
|
|
777
|
+
const perPage = normalizePerPage(perPageInput, 40);
|
|
778
|
+
const { offset, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
1003
779
|
try {
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
}
|
|
1015
|
-
const
|
|
1016
|
-
|
|
780
|
+
if (page < 0) {
|
|
781
|
+
throw new MastraError(
|
|
782
|
+
{
|
|
783
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES", "INVALID_PAGE"),
|
|
784
|
+
domain: ErrorDomain.STORAGE,
|
|
785
|
+
category: ErrorCategory.USER,
|
|
786
|
+
details: { page }
|
|
787
|
+
},
|
|
788
|
+
new Error("page must be >= 0")
|
|
789
|
+
);
|
|
790
|
+
}
|
|
791
|
+
const { field, direction } = this.parseOrderBy(orderBy, "ASC");
|
|
792
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
793
|
+
const threadCondition = threadIds.length === 1 ? `thread_id = '${this.escapeSql(threadIds[0])}'` : `thread_id IN (${threadIds.map((t) => `'${this.escapeSql(t)}'`).join(", ")})`;
|
|
794
|
+
const conditions = [threadCondition];
|
|
795
|
+
if (resourceId) {
|
|
796
|
+
conditions.push(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
797
|
+
}
|
|
798
|
+
if (filter?.dateRange?.start) {
|
|
799
|
+
const startTime = filter.dateRange.start instanceof Date ? filter.dateRange.start.getTime() : new Date(filter.dateRange.start).getTime();
|
|
800
|
+
const startOp = filter.dateRange.startExclusive ? ">" : ">=";
|
|
801
|
+
conditions.push(`\`createdAt\` ${startOp} ${startTime}`);
|
|
802
|
+
}
|
|
803
|
+
if (filter?.dateRange?.end) {
|
|
804
|
+
const endTime = filter.dateRange.end instanceof Date ? filter.dateRange.end.getTime() : new Date(filter.dateRange.end).getTime();
|
|
805
|
+
const endOp = filter.dateRange.endExclusive ? "<" : "<=";
|
|
806
|
+
conditions.push(`\`createdAt\` ${endOp} ${endTime}`);
|
|
807
|
+
}
|
|
808
|
+
const whereClause = conditions.join(" AND ");
|
|
809
|
+
const total = await table.countRows(whereClause);
|
|
810
|
+
const query = table.query().where(whereClause);
|
|
811
|
+
let allRecords = await query.toArray();
|
|
812
|
+
allRecords.sort((a, b) => {
|
|
813
|
+
const aValue = field === "createdAt" ? a.createdAt : a[field];
|
|
814
|
+
const bValue = field === "createdAt" ? b.createdAt : b[field];
|
|
815
|
+
if (aValue == null && bValue == null) return 0;
|
|
816
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
817
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
818
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
819
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
820
|
+
}
|
|
821
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
822
|
+
});
|
|
823
|
+
const paginatedRecords = allRecords.slice(offset, offset + perPage);
|
|
824
|
+
const messages = paginatedRecords.map((row) => this.normalizeMessage(row));
|
|
825
|
+
if (total === 0 && messages.length === 0 && (!include || include.length === 0)) {
|
|
1017
826
|
return {
|
|
1018
|
-
|
|
1019
|
-
|
|
827
|
+
messages: [],
|
|
828
|
+
total: 0,
|
|
829
|
+
page,
|
|
830
|
+
perPage: perPageForResponse,
|
|
831
|
+
hasMore: false
|
|
1020
832
|
};
|
|
1021
|
-
});
|
|
1022
|
-
if (columnsToAdd.length > 0) {
|
|
1023
|
-
await table.addColumns(columnsToAdd);
|
|
1024
|
-
this.logger?.info?.(`Added columns [${columnsToAdd.map((c) => c.name).join(", ")}] to table ${tableName}`);
|
|
1025
833
|
}
|
|
834
|
+
const messageIds = new Set(messages.map((m) => m.id));
|
|
835
|
+
if (include && include.length > 0) {
|
|
836
|
+
const threadIds2 = [...new Set(include.map((item) => item.threadId || threadId))];
|
|
837
|
+
const allThreadMessages = [];
|
|
838
|
+
for (const tid of threadIds2) {
|
|
839
|
+
const threadQuery = table.query().where(`thread_id = '${tid}'`);
|
|
840
|
+
let threadRecords = await threadQuery.toArray();
|
|
841
|
+
allThreadMessages.push(...threadRecords);
|
|
842
|
+
}
|
|
843
|
+
allThreadMessages.sort((a, b) => a.createdAt - b.createdAt);
|
|
844
|
+
const contextMessages = this.processMessagesWithContext(allThreadMessages, include);
|
|
845
|
+
const includedMessages = contextMessages.map((row) => this.normalizeMessage(row));
|
|
846
|
+
for (const includeMsg of includedMessages) {
|
|
847
|
+
if (!messageIds.has(includeMsg.id)) {
|
|
848
|
+
messages.push(includeMsg);
|
|
849
|
+
messageIds.add(includeMsg.id);
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
const list = new MessageList().add(messages, "memory");
|
|
854
|
+
let finalMessages = list.get.all.db();
|
|
855
|
+
finalMessages = finalMessages.sort((a, b) => {
|
|
856
|
+
const aValue = field === "createdAt" ? new Date(a.createdAt).getTime() : a[field];
|
|
857
|
+
const bValue = field === "createdAt" ? new Date(b.createdAt).getTime() : b[field];
|
|
858
|
+
if (aValue == null && bValue == null) return 0;
|
|
859
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
860
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
861
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
862
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
863
|
+
}
|
|
864
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
865
|
+
});
|
|
866
|
+
const returnedThreadMessageIds = new Set(finalMessages.filter((m) => m.threadId === threadId).map((m) => m.id));
|
|
867
|
+
const allThreadMessagesReturned = returnedThreadMessageIds.size >= total;
|
|
868
|
+
const fetchedAll = perPageInput === false || allThreadMessagesReturned;
|
|
869
|
+
const hasMore = !fetchedAll && offset + perPage < total;
|
|
870
|
+
return {
|
|
871
|
+
messages: finalMessages,
|
|
872
|
+
total,
|
|
873
|
+
page,
|
|
874
|
+
perPage: perPageForResponse,
|
|
875
|
+
hasMore
|
|
876
|
+
};
|
|
1026
877
|
} catch (error) {
|
|
1027
|
-
|
|
878
|
+
const mastraError = new MastraError(
|
|
1028
879
|
{
|
|
1029
|
-
id: "
|
|
880
|
+
id: createStorageErrorId("LANCE", "LIST_MESSAGES", "FAILED"),
|
|
1030
881
|
domain: ErrorDomain.STORAGE,
|
|
1031
882
|
category: ErrorCategory.THIRD_PARTY,
|
|
1032
|
-
details: {
|
|
883
|
+
details: {
|
|
884
|
+
threadId: Array.isArray(threadId) ? threadId.join(",") : threadId,
|
|
885
|
+
resourceId: resourceId ?? ""
|
|
886
|
+
}
|
|
1033
887
|
},
|
|
1034
888
|
error
|
|
1035
889
|
);
|
|
890
|
+
this.logger?.error?.(mastraError.toString());
|
|
891
|
+
this.logger?.trackException?.(mastraError);
|
|
892
|
+
return {
|
|
893
|
+
messages: [],
|
|
894
|
+
total: 0,
|
|
895
|
+
page,
|
|
896
|
+
perPage: perPageForResponse,
|
|
897
|
+
hasMore: false
|
|
898
|
+
};
|
|
1036
899
|
}
|
|
1037
900
|
}
|
|
1038
|
-
async
|
|
901
|
+
async saveMessages(args) {
|
|
1039
902
|
try {
|
|
1040
|
-
|
|
1041
|
-
|
|
903
|
+
const { messages } = args;
|
|
904
|
+
if (messages.length === 0) {
|
|
905
|
+
return { messages: [] };
|
|
1042
906
|
}
|
|
1043
|
-
|
|
1044
|
-
|
|
907
|
+
const threadId = messages[0]?.threadId;
|
|
908
|
+
if (!threadId) {
|
|
909
|
+
throw new Error("Thread ID is required");
|
|
1045
910
|
}
|
|
1046
|
-
|
|
911
|
+
for (const message of messages) {
|
|
912
|
+
if (!message.id) {
|
|
913
|
+
throw new Error("Message ID is required");
|
|
914
|
+
}
|
|
915
|
+
if (!message.threadId) {
|
|
916
|
+
throw new Error("Thread ID is required for all messages");
|
|
917
|
+
}
|
|
918
|
+
if (message.resourceId === null || message.resourceId === void 0) {
|
|
919
|
+
throw new Error("Resource ID cannot be null or undefined");
|
|
920
|
+
}
|
|
921
|
+
if (!message.content) {
|
|
922
|
+
throw new Error("Message content is required");
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
const transformedMessages = messages.map((message) => {
|
|
926
|
+
const { threadId: threadId2, type, ...rest } = message;
|
|
927
|
+
return {
|
|
928
|
+
...rest,
|
|
929
|
+
thread_id: threadId2,
|
|
930
|
+
type: type ?? "v2",
|
|
931
|
+
content: JSON.stringify(message.content)
|
|
932
|
+
};
|
|
933
|
+
});
|
|
934
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
935
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(transformedMessages);
|
|
936
|
+
const threadsTable = await this.client.openTable(TABLE_THREADS);
|
|
937
|
+
const currentTime = (/* @__PURE__ */ new Date()).getTime();
|
|
938
|
+
const updateRecord = { id: threadId, updatedAt: currentTime };
|
|
939
|
+
await threadsTable.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([updateRecord]);
|
|
940
|
+
const list = new MessageList().add(messages, "memory");
|
|
941
|
+
return { messages: list.get.all.db() };
|
|
942
|
+
} catch (error) {
|
|
1047
943
|
throw new MastraError(
|
|
1048
944
|
{
|
|
1049
|
-
id: "
|
|
945
|
+
id: createStorageErrorId("LANCE", "SAVE_MESSAGES", "FAILED"),
|
|
1050
946
|
domain: ErrorDomain.STORAGE,
|
|
1051
|
-
category: ErrorCategory.
|
|
1052
|
-
text: validationError.message,
|
|
1053
|
-
details: { tableName }
|
|
947
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1054
948
|
},
|
|
1055
|
-
|
|
949
|
+
error
|
|
1056
950
|
);
|
|
1057
951
|
}
|
|
952
|
+
}
|
|
953
|
+
async listThreadsByResourceId(args) {
|
|
1058
954
|
try {
|
|
1059
|
-
const
|
|
1060
|
-
|
|
955
|
+
const { resourceId, page = 0, perPage: perPageInput, orderBy } = args;
|
|
956
|
+
const perPage = normalizePerPage(perPageInput, 100);
|
|
957
|
+
if (page < 0) {
|
|
958
|
+
throw new MastraError(
|
|
959
|
+
{
|
|
960
|
+
id: createStorageErrorId("LANCE", "LIST_THREADS_BY_RESOURCE_ID", "INVALID_PAGE"),
|
|
961
|
+
domain: ErrorDomain.STORAGE,
|
|
962
|
+
category: ErrorCategory.USER,
|
|
963
|
+
details: { page }
|
|
964
|
+
},
|
|
965
|
+
new Error("page must be >= 0")
|
|
966
|
+
);
|
|
967
|
+
}
|
|
968
|
+
const { offset, perPage: perPageForResponse } = calculatePagination(page, perPageInput, perPage);
|
|
969
|
+
const { field, direction } = this.parseOrderBy(orderBy);
|
|
970
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
971
|
+
const total = await table.countRows(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
972
|
+
const query = table.query().where(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
973
|
+
const records = await query.toArray();
|
|
974
|
+
records.sort((a, b) => {
|
|
975
|
+
const aValue = ["createdAt", "updatedAt"].includes(field) ? new Date(a[field]).getTime() : a[field];
|
|
976
|
+
const bValue = ["createdAt", "updatedAt"].includes(field) ? new Date(b[field]).getTime() : b[field];
|
|
977
|
+
if (aValue == null && bValue == null) return 0;
|
|
978
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
979
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
980
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
981
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
982
|
+
}
|
|
983
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
984
|
+
});
|
|
985
|
+
const paginatedRecords = records.slice(offset, offset + perPage);
|
|
986
|
+
const schema = await getTableSchema({ tableName: TABLE_THREADS, client: this.client });
|
|
987
|
+
const threads = paginatedRecords.map(
|
|
988
|
+
(record) => processResultWithTypeConversion(record, schema)
|
|
989
|
+
);
|
|
990
|
+
return {
|
|
991
|
+
threads,
|
|
992
|
+
total,
|
|
993
|
+
page,
|
|
994
|
+
perPage: perPageForResponse,
|
|
995
|
+
hasMore: offset + perPage < total
|
|
996
|
+
};
|
|
1061
997
|
} catch (error) {
|
|
1062
998
|
throw new MastraError(
|
|
1063
999
|
{
|
|
1064
|
-
id: "
|
|
1000
|
+
id: createStorageErrorId("LANCE", "LIST_THREADS_BY_RESOURCE_ID", "FAILED"),
|
|
1065
1001
|
domain: ErrorDomain.STORAGE,
|
|
1066
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1067
|
-
details: { tableName }
|
|
1002
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1068
1003
|
},
|
|
1069
1004
|
error
|
|
1070
1005
|
);
|
|
1071
1006
|
}
|
|
1072
1007
|
}
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1008
|
+
/**
|
|
1009
|
+
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
1010
|
+
* @param records - The sorted array of records to process
|
|
1011
|
+
* @param include - The array of include specifications with context parameters
|
|
1012
|
+
* @returns The processed array with context messages included
|
|
1013
|
+
*/
|
|
1014
|
+
processMessagesWithContext(records, include) {
|
|
1015
|
+
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
1016
|
+
if (messagesWithContext.length === 0) {
|
|
1017
|
+
return records;
|
|
1018
|
+
}
|
|
1019
|
+
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
1020
|
+
records.forEach((message, index) => {
|
|
1021
|
+
messageIndexMap.set(message.id, index);
|
|
1022
|
+
});
|
|
1023
|
+
const additionalIndices = /* @__PURE__ */ new Set();
|
|
1024
|
+
for (const item of messagesWithContext) {
|
|
1025
|
+
const messageIndex = messageIndexMap.get(item.id);
|
|
1026
|
+
if (messageIndex !== void 0) {
|
|
1027
|
+
if (item.withPreviousMessages) {
|
|
1028
|
+
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
1029
|
+
for (let i = startIdx; i < messageIndex; i++) {
|
|
1030
|
+
additionalIndices.add(i);
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
if (item.withNextMessages) {
|
|
1034
|
+
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
1035
|
+
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
1036
|
+
additionalIndices.add(i);
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1080
1039
|
}
|
|
1081
|
-
|
|
1082
|
-
|
|
1040
|
+
}
|
|
1041
|
+
if (additionalIndices.size === 0) {
|
|
1042
|
+
return records;
|
|
1043
|
+
}
|
|
1044
|
+
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
1045
|
+
const allIndices = /* @__PURE__ */ new Set();
|
|
1046
|
+
records.forEach((record, index) => {
|
|
1047
|
+
if (originalMatchIds.has(record.id)) {
|
|
1048
|
+
allIndices.add(index);
|
|
1083
1049
|
}
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1050
|
+
});
|
|
1051
|
+
additionalIndices.forEach((index) => {
|
|
1052
|
+
allIndices.add(index);
|
|
1053
|
+
});
|
|
1054
|
+
return Array.from(allIndices).sort((a, b) => a - b).map((index) => records[index]);
|
|
1055
|
+
}
|
|
1056
|
+
/**
|
|
1057
|
+
* Parse message data from LanceDB record format to MastraDBMessage format
|
|
1058
|
+
*/
|
|
1059
|
+
parseMessageData(data) {
|
|
1060
|
+
const { thread_id, ...rest } = data;
|
|
1061
|
+
return {
|
|
1062
|
+
...rest,
|
|
1063
|
+
threadId: thread_id,
|
|
1064
|
+
content: typeof data.content === "string" ? (() => {
|
|
1065
|
+
try {
|
|
1066
|
+
return JSON.parse(data.content);
|
|
1067
|
+
} catch {
|
|
1068
|
+
return data.content;
|
|
1069
|
+
}
|
|
1070
|
+
})() : data.content,
|
|
1071
|
+
createdAt: new Date(data.createdAt),
|
|
1072
|
+
updatedAt: new Date(data.updatedAt)
|
|
1073
|
+
};
|
|
1074
|
+
}
|
|
1075
|
+
async updateMessages(args) {
|
|
1076
|
+
const { messages } = args;
|
|
1077
|
+
this.logger.debug("Updating messages", { count: messages.length });
|
|
1078
|
+
if (!messages.length) {
|
|
1079
|
+
return [];
|
|
1095
1080
|
}
|
|
1081
|
+
const updatedMessages = [];
|
|
1082
|
+
const affectedThreadIds = /* @__PURE__ */ new Set();
|
|
1096
1083
|
try {
|
|
1097
|
-
const
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1084
|
+
for (const updateData of messages) {
|
|
1085
|
+
const { id, ...updates } = updateData;
|
|
1086
|
+
const existingMessage = await this.#db.load({ tableName: TABLE_MESSAGES, keys: { id } });
|
|
1087
|
+
if (!existingMessage) {
|
|
1088
|
+
this.logger.warn("Message not found for update", { id });
|
|
1089
|
+
continue;
|
|
1090
|
+
}
|
|
1091
|
+
const existingMsg = this.parseMessageData(existingMessage);
|
|
1092
|
+
const originalThreadId = existingMsg.threadId;
|
|
1093
|
+
affectedThreadIds.add(originalThreadId);
|
|
1094
|
+
const updatePayload = {};
|
|
1095
|
+
if ("role" in updates && updates.role !== void 0) updatePayload.role = updates.role;
|
|
1096
|
+
if ("type" in updates && updates.type !== void 0) updatePayload.type = updates.type;
|
|
1097
|
+
if ("resourceId" in updates && updates.resourceId !== void 0) updatePayload.resourceId = updates.resourceId;
|
|
1098
|
+
if ("threadId" in updates && updates.threadId !== void 0 && updates.threadId !== null) {
|
|
1099
|
+
updatePayload.thread_id = updates.threadId;
|
|
1100
|
+
affectedThreadIds.add(updates.threadId);
|
|
1104
1101
|
}
|
|
1102
|
+
if (updates.content) {
|
|
1103
|
+
const existingContent = existingMsg.content;
|
|
1104
|
+
let newContent = { ...existingContent };
|
|
1105
|
+
if (updates.content.metadata !== void 0) {
|
|
1106
|
+
newContent.metadata = {
|
|
1107
|
+
...existingContent.metadata || {},
|
|
1108
|
+
...updates.content.metadata || {}
|
|
1109
|
+
};
|
|
1110
|
+
}
|
|
1111
|
+
if (updates.content.content !== void 0) {
|
|
1112
|
+
newContent.content = updates.content.content;
|
|
1113
|
+
}
|
|
1114
|
+
if ("parts" in updates.content && updates.content.parts !== void 0) {
|
|
1115
|
+
newContent.parts = updates.content.parts;
|
|
1116
|
+
}
|
|
1117
|
+
updatePayload.content = JSON.stringify(newContent);
|
|
1118
|
+
}
|
|
1119
|
+
await this.#db.insert({ tableName: TABLE_MESSAGES, record: { id, ...updatePayload } });
|
|
1120
|
+
const updatedMessage = await this.#db.load({ tableName: TABLE_MESSAGES, keys: { id } });
|
|
1121
|
+
if (updatedMessage) {
|
|
1122
|
+
updatedMessages.push(this.parseMessageData(updatedMessage));
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
for (const threadId of affectedThreadIds) {
|
|
1126
|
+
await this.#db.insert({
|
|
1127
|
+
tableName: TABLE_THREADS,
|
|
1128
|
+
record: { id: threadId, updatedAt: Date.now() }
|
|
1129
|
+
});
|
|
1105
1130
|
}
|
|
1106
|
-
|
|
1107
|
-
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([processedRecord]);
|
|
1131
|
+
return updatedMessages;
|
|
1108
1132
|
} catch (error) {
|
|
1109
1133
|
throw new MastraError(
|
|
1110
1134
|
{
|
|
1111
|
-
id: "
|
|
1135
|
+
id: createStorageErrorId("LANCE", "UPDATE_MESSAGES", "FAILED"),
|
|
1112
1136
|
domain: ErrorDomain.STORAGE,
|
|
1113
1137
|
category: ErrorCategory.THIRD_PARTY,
|
|
1114
|
-
details: {
|
|
1138
|
+
details: { count: messages.length }
|
|
1115
1139
|
},
|
|
1116
1140
|
error
|
|
1117
1141
|
);
|
|
1118
1142
|
}
|
|
1119
1143
|
}
|
|
1120
|
-
async
|
|
1144
|
+
async getResourceById({ resourceId }) {
|
|
1121
1145
|
try {
|
|
1122
|
-
|
|
1123
|
-
|
|
1146
|
+
const resource = await this.#db.load({ tableName: TABLE_RESOURCES, keys: { id: resourceId } });
|
|
1147
|
+
if (!resource) {
|
|
1148
|
+
return null;
|
|
1124
1149
|
}
|
|
1125
|
-
|
|
1126
|
-
|
|
1150
|
+
let createdAt;
|
|
1151
|
+
let updatedAt;
|
|
1152
|
+
try {
|
|
1153
|
+
if (resource.createdAt instanceof Date) {
|
|
1154
|
+
createdAt = resource.createdAt;
|
|
1155
|
+
} else if (typeof resource.createdAt === "string") {
|
|
1156
|
+
createdAt = new Date(resource.createdAt);
|
|
1157
|
+
} else if (typeof resource.createdAt === "number") {
|
|
1158
|
+
createdAt = new Date(resource.createdAt);
|
|
1159
|
+
} else {
|
|
1160
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1161
|
+
}
|
|
1162
|
+
if (isNaN(createdAt.getTime())) {
|
|
1163
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1164
|
+
}
|
|
1165
|
+
} catch {
|
|
1166
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1127
1167
|
}
|
|
1128
|
-
|
|
1129
|
-
|
|
1168
|
+
try {
|
|
1169
|
+
if (resource.updatedAt instanceof Date) {
|
|
1170
|
+
updatedAt = resource.updatedAt;
|
|
1171
|
+
} else if (typeof resource.updatedAt === "string") {
|
|
1172
|
+
updatedAt = new Date(resource.updatedAt);
|
|
1173
|
+
} else if (typeof resource.updatedAt === "number") {
|
|
1174
|
+
updatedAt = new Date(resource.updatedAt);
|
|
1175
|
+
} else {
|
|
1176
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1177
|
+
}
|
|
1178
|
+
if (isNaN(updatedAt.getTime())) {
|
|
1179
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1180
|
+
}
|
|
1181
|
+
} catch {
|
|
1182
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1130
1183
|
}
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
const processedRecord = { ...record };
|
|
1148
|
-
for (const key in processedRecord) {
|
|
1149
|
-
if (processedRecord[key] == null) continue;
|
|
1150
|
-
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
1151
|
-
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
1152
|
-
}
|
|
1184
|
+
let workingMemory = resource.workingMemory;
|
|
1185
|
+
if (workingMemory === null || workingMemory === void 0) {
|
|
1186
|
+
workingMemory = void 0;
|
|
1187
|
+
} else if (workingMemory === "") {
|
|
1188
|
+
workingMemory = "";
|
|
1189
|
+
} else if (typeof workingMemory === "object") {
|
|
1190
|
+
workingMemory = JSON.stringify(workingMemory);
|
|
1191
|
+
}
|
|
1192
|
+
let metadata = resource.metadata;
|
|
1193
|
+
if (metadata === "" || metadata === null || metadata === void 0) {
|
|
1194
|
+
metadata = void 0;
|
|
1195
|
+
} else if (typeof metadata === "string") {
|
|
1196
|
+
try {
|
|
1197
|
+
metadata = JSON.parse(metadata);
|
|
1198
|
+
} catch {
|
|
1199
|
+
metadata = metadata;
|
|
1153
1200
|
}
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1201
|
+
}
|
|
1202
|
+
return {
|
|
1203
|
+
...resource,
|
|
1204
|
+
createdAt,
|
|
1205
|
+
updatedAt,
|
|
1206
|
+
workingMemory,
|
|
1207
|
+
metadata
|
|
1208
|
+
};
|
|
1157
1209
|
} catch (error) {
|
|
1158
1210
|
throw new MastraError(
|
|
1159
1211
|
{
|
|
1160
|
-
id: "
|
|
1212
|
+
id: createStorageErrorId("LANCE", "GET_RESOURCE_BY_ID", "FAILED"),
|
|
1161
1213
|
domain: ErrorDomain.STORAGE,
|
|
1162
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1163
|
-
details: { tableName }
|
|
1214
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1164
1215
|
},
|
|
1165
1216
|
error
|
|
1166
1217
|
);
|
|
1167
1218
|
}
|
|
1168
1219
|
}
|
|
1169
|
-
async
|
|
1170
|
-
try {
|
|
1171
|
-
if (!this.client) {
|
|
1172
|
-
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1173
|
-
}
|
|
1174
|
-
if (!tableName) {
|
|
1175
|
-
throw new Error("tableName is required for load.");
|
|
1176
|
-
}
|
|
1177
|
-
if (!keys || Object.keys(keys).length === 0) {
|
|
1178
|
-
throw new Error("keys are required and cannot be empty for load.");
|
|
1179
|
-
}
|
|
1180
|
-
} catch (validationError) {
|
|
1181
|
-
throw new MastraError(
|
|
1182
|
-
{
|
|
1183
|
-
id: "STORAGE_LANCE_STORAGE_LOAD_INVALID_ARGS",
|
|
1184
|
-
domain: ErrorDomain.STORAGE,
|
|
1185
|
-
category: ErrorCategory.USER,
|
|
1186
|
-
text: validationError.message,
|
|
1187
|
-
details: { tableName }
|
|
1188
|
-
},
|
|
1189
|
-
validationError
|
|
1190
|
-
);
|
|
1191
|
-
}
|
|
1220
|
+
async saveResource({ resource }) {
|
|
1192
1221
|
try {
|
|
1193
|
-
const
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
return `${quotedKey} IS NULL`;
|
|
1205
|
-
} else {
|
|
1206
|
-
return `${quotedKey} = ${value}`;
|
|
1207
|
-
}
|
|
1208
|
-
}).join(" AND ");
|
|
1209
|
-
this.logger.debug("where clause generated: " + filterConditions);
|
|
1210
|
-
query.where(filterConditions);
|
|
1211
|
-
}
|
|
1212
|
-
const result = await query.limit(1).toArray();
|
|
1213
|
-
if (result.length === 0) {
|
|
1214
|
-
this.logger.debug("No record found");
|
|
1215
|
-
return null;
|
|
1216
|
-
}
|
|
1217
|
-
return processResultWithTypeConversion(result[0], tableSchema);
|
|
1222
|
+
const record = {
|
|
1223
|
+
...resource,
|
|
1224
|
+
metadata: resource.metadata ? JSON.stringify(resource.metadata) : "",
|
|
1225
|
+
createdAt: resource.createdAt.getTime(),
|
|
1226
|
+
// Store as timestamp (milliseconds)
|
|
1227
|
+
updatedAt: resource.updatedAt.getTime()
|
|
1228
|
+
// Store as timestamp (milliseconds)
|
|
1229
|
+
};
|
|
1230
|
+
const table = await this.client.openTable(TABLE_RESOURCES);
|
|
1231
|
+
await table.add([record], { mode: "append" });
|
|
1232
|
+
return resource;
|
|
1218
1233
|
} catch (error) {
|
|
1219
|
-
if (error instanceof MastraError) throw error;
|
|
1220
1234
|
throw new MastraError(
|
|
1221
1235
|
{
|
|
1222
|
-
id: "
|
|
1236
|
+
id: createStorageErrorId("LANCE", "SAVE_RESOURCE", "FAILED"),
|
|
1223
1237
|
domain: ErrorDomain.STORAGE,
|
|
1224
|
-
category: ErrorCategory.THIRD_PARTY
|
|
1225
|
-
details: { tableName, keyCount: Object.keys(keys).length, firstKey: Object.keys(keys)[0] ?? "" }
|
|
1238
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1226
1239
|
},
|
|
1227
1240
|
error
|
|
1228
1241
|
);
|
|
1229
1242
|
}
|
|
1230
1243
|
}
|
|
1244
|
+
async updateResource({
|
|
1245
|
+
resourceId,
|
|
1246
|
+
workingMemory,
|
|
1247
|
+
metadata
|
|
1248
|
+
}) {
|
|
1249
|
+
const maxRetries = 3;
|
|
1250
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
1251
|
+
try {
|
|
1252
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1253
|
+
if (!existingResource) {
|
|
1254
|
+
const newResource = {
|
|
1255
|
+
id: resourceId,
|
|
1256
|
+
workingMemory,
|
|
1257
|
+
metadata: metadata || {},
|
|
1258
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1259
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1260
|
+
};
|
|
1261
|
+
return this.saveResource({ resource: newResource });
|
|
1262
|
+
}
|
|
1263
|
+
const updatedResource = {
|
|
1264
|
+
...existingResource,
|
|
1265
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1266
|
+
metadata: {
|
|
1267
|
+
...existingResource.metadata,
|
|
1268
|
+
...metadata
|
|
1269
|
+
},
|
|
1270
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1271
|
+
};
|
|
1272
|
+
const record = {
|
|
1273
|
+
id: resourceId,
|
|
1274
|
+
workingMemory: updatedResource.workingMemory || "",
|
|
1275
|
+
metadata: updatedResource.metadata ? JSON.stringify(updatedResource.metadata) : "",
|
|
1276
|
+
updatedAt: updatedResource.updatedAt.getTime()
|
|
1277
|
+
// Store as timestamp (milliseconds)
|
|
1278
|
+
};
|
|
1279
|
+
const table = await this.client.openTable(TABLE_RESOURCES);
|
|
1280
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
1281
|
+
return updatedResource;
|
|
1282
|
+
} catch (error) {
|
|
1283
|
+
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
1284
|
+
const delay = Math.pow(2, attempt) * 10;
|
|
1285
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1286
|
+
continue;
|
|
1287
|
+
}
|
|
1288
|
+
throw new MastraError(
|
|
1289
|
+
{
|
|
1290
|
+
id: createStorageErrorId("LANCE", "UPDATE_RESOURCE", "FAILED"),
|
|
1291
|
+
domain: ErrorDomain.STORAGE,
|
|
1292
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1293
|
+
},
|
|
1294
|
+
error
|
|
1295
|
+
);
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
throw new Error("Unexpected end of retry loop");
|
|
1299
|
+
}
|
|
1231
1300
|
};
|
|
1232
1301
|
var StoreScoresLance = class extends ScoresStorage {
|
|
1233
1302
|
client;
|
|
1234
|
-
|
|
1303
|
+
#db;
|
|
1304
|
+
constructor(config) {
|
|
1235
1305
|
super();
|
|
1306
|
+
const client = resolveLanceConfig(config);
|
|
1236
1307
|
this.client = client;
|
|
1308
|
+
this.#db = new LanceDB({ client });
|
|
1309
|
+
}
|
|
1310
|
+
async init() {
|
|
1311
|
+
await this.#db.createTable({ tableName: TABLE_SCORERS, schema: SCORERS_SCHEMA });
|
|
1312
|
+
await this.#db.alterTable({
|
|
1313
|
+
tableName: TABLE_SCORERS,
|
|
1314
|
+
schema: SCORERS_SCHEMA,
|
|
1315
|
+
ifNotExists: ["spanId", "requestContext"]
|
|
1316
|
+
});
|
|
1317
|
+
}
|
|
1318
|
+
async dangerouslyClearAll() {
|
|
1319
|
+
await this.#db.clearTable({ tableName: TABLE_SCORERS });
|
|
1237
1320
|
}
|
|
1238
1321
|
async saveScore(score) {
|
|
1239
1322
|
let validatedScore;
|
|
@@ -1242,37 +1325,47 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1242
1325
|
} catch (error) {
|
|
1243
1326
|
throw new MastraError(
|
|
1244
1327
|
{
|
|
1245
|
-
id: "
|
|
1328
|
+
id: createStorageErrorId("LANCE", "SAVE_SCORE", "VALIDATION_FAILED"),
|
|
1246
1329
|
text: "Failed to save score in LanceStorage",
|
|
1247
1330
|
domain: ErrorDomain.STORAGE,
|
|
1248
|
-
category: ErrorCategory.
|
|
1331
|
+
category: ErrorCategory.USER,
|
|
1332
|
+
details: {
|
|
1333
|
+
scorer: typeof score.scorer?.id === "string" ? score.scorer.id : String(score.scorer?.id ?? "unknown"),
|
|
1334
|
+
entityId: score.entityId ?? "unknown",
|
|
1335
|
+
entityType: score.entityType ?? "unknown",
|
|
1336
|
+
traceId: score.traceId ?? "",
|
|
1337
|
+
spanId: score.spanId ?? ""
|
|
1338
|
+
}
|
|
1249
1339
|
},
|
|
1250
1340
|
error
|
|
1251
1341
|
);
|
|
1252
1342
|
}
|
|
1343
|
+
const id = crypto.randomUUID();
|
|
1344
|
+
const now = /* @__PURE__ */ new Date();
|
|
1253
1345
|
try {
|
|
1254
|
-
const id = crypto.randomUUID();
|
|
1255
1346
|
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1256
1347
|
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1257
1348
|
const allowedFields = new Set(schema.fields.map((f) => f.name));
|
|
1258
1349
|
const filteredScore = {};
|
|
1259
|
-
Object.keys(validatedScore)
|
|
1350
|
+
for (const key of Object.keys(validatedScore)) {
|
|
1260
1351
|
if (allowedFields.has(key)) {
|
|
1261
|
-
filteredScore[key] =
|
|
1352
|
+
filteredScore[key] = validatedScore[key];
|
|
1262
1353
|
}
|
|
1263
|
-
}
|
|
1354
|
+
}
|
|
1264
1355
|
for (const key in filteredScore) {
|
|
1265
1356
|
if (filteredScore[key] !== null && typeof filteredScore[key] === "object" && !(filteredScore[key] instanceof Date)) {
|
|
1266
1357
|
filteredScore[key] = JSON.stringify(filteredScore[key]);
|
|
1267
1358
|
}
|
|
1268
1359
|
}
|
|
1269
1360
|
filteredScore.id = id;
|
|
1361
|
+
filteredScore.createdAt = now;
|
|
1362
|
+
filteredScore.updatedAt = now;
|
|
1270
1363
|
await table.add([filteredScore], { mode: "append" });
|
|
1271
|
-
return { score };
|
|
1364
|
+
return { score: { ...validatedScore, id, createdAt: now, updatedAt: now } };
|
|
1272
1365
|
} catch (error) {
|
|
1273
1366
|
throw new MastraError(
|
|
1274
1367
|
{
|
|
1275
|
-
id: "
|
|
1368
|
+
id: createStorageErrorId("LANCE", "SAVE_SCORE", "FAILED"),
|
|
1276
1369
|
text: "Failed to save score in LanceStorage",
|
|
1277
1370
|
domain: ErrorDomain.STORAGE,
|
|
1278
1371
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1288,12 +1381,11 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1288
1381
|
const query = table.query().where(`id = '${id}'`).limit(1);
|
|
1289
1382
|
const records = await query.toArray();
|
|
1290
1383
|
if (records.length === 0) return null;
|
|
1291
|
-
|
|
1292
|
-
return processResultWithTypeConversion(records[0], schema);
|
|
1384
|
+
return await this.transformScoreRow(records[0]);
|
|
1293
1385
|
} catch (error) {
|
|
1294
1386
|
throw new MastraError(
|
|
1295
1387
|
{
|
|
1296
|
-
id: "
|
|
1388
|
+
id: createStorageErrorId("LANCE", "GET_SCORE_BY_ID", "FAILED"),
|
|
1297
1389
|
text: "Failed to get score by id in LanceStorage",
|
|
1298
1390
|
domain: ErrorDomain.STORAGE,
|
|
1299
1391
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1303,6 +1395,22 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1303
1395
|
);
|
|
1304
1396
|
}
|
|
1305
1397
|
}
|
|
1398
|
+
/**
|
|
1399
|
+
* LanceDB-specific score row transformation.
|
|
1400
|
+
*
|
|
1401
|
+
* Note: This implementation does NOT use coreTransformScoreRow because:
|
|
1402
|
+
* 1. LanceDB stores schema information in the table itself (requires async fetch)
|
|
1403
|
+
* 2. Uses processResultWithTypeConversion utility for LanceDB-specific type handling
|
|
1404
|
+
*/
|
|
1405
|
+
async transformScoreRow(row) {
|
|
1406
|
+
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1407
|
+
const transformed = processResultWithTypeConversion(row, schema);
|
|
1408
|
+
return {
|
|
1409
|
+
...transformed,
|
|
1410
|
+
createdAt: row.createdAt,
|
|
1411
|
+
updatedAt: row.updatedAt
|
|
1412
|
+
};
|
|
1413
|
+
}
|
|
1306
1414
|
async listScoresByScorerId({
|
|
1307
1415
|
scorerId,
|
|
1308
1416
|
pagination,
|
|
@@ -1343,8 +1451,7 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1343
1451
|
if (start > 0) query = query.offset(start);
|
|
1344
1452
|
}
|
|
1345
1453
|
const records = await query.toArray();
|
|
1346
|
-
const
|
|
1347
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1454
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1348
1455
|
return {
|
|
1349
1456
|
pagination: {
|
|
1350
1457
|
page,
|
|
@@ -1357,7 +1464,7 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1357
1464
|
} catch (error) {
|
|
1358
1465
|
throw new MastraError(
|
|
1359
1466
|
{
|
|
1360
|
-
id: "
|
|
1467
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_SCORER_ID", "FAILED"),
|
|
1361
1468
|
text: "Failed to get scores by scorerId in LanceStorage",
|
|
1362
1469
|
domain: ErrorDomain.STORAGE,
|
|
1363
1470
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1385,8 +1492,7 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1385
1492
|
if (start > 0) query = query.offset(start);
|
|
1386
1493
|
}
|
|
1387
1494
|
const records = await query.toArray();
|
|
1388
|
-
const
|
|
1389
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1495
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1390
1496
|
return {
|
|
1391
1497
|
pagination: {
|
|
1392
1498
|
page,
|
|
@@ -1399,7 +1505,7 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1399
1505
|
} catch (error) {
|
|
1400
1506
|
throw new MastraError(
|
|
1401
1507
|
{
|
|
1402
|
-
id: "
|
|
1508
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_RUN_ID", "FAILED"),
|
|
1403
1509
|
text: "Failed to get scores by runId in LanceStorage",
|
|
1404
1510
|
domain: ErrorDomain.STORAGE,
|
|
1405
1511
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1428,8 +1534,7 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1428
1534
|
if (start > 0) query = query.offset(start);
|
|
1429
1535
|
}
|
|
1430
1536
|
const records = await query.toArray();
|
|
1431
|
-
const
|
|
1432
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1537
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1433
1538
|
return {
|
|
1434
1539
|
pagination: {
|
|
1435
1540
|
page,
|
|
@@ -1442,7 +1547,7 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1442
1547
|
} catch (error) {
|
|
1443
1548
|
throw new MastraError(
|
|
1444
1549
|
{
|
|
1445
|
-
id: "
|
|
1550
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_ENTITY_ID", "FAILED"),
|
|
1446
1551
|
text: "Failed to get scores by entityId and entityType in LanceStorage",
|
|
1447
1552
|
domain: ErrorDomain.STORAGE,
|
|
1448
1553
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1471,8 +1576,7 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1471
1576
|
if (start > 0) query = query.offset(start);
|
|
1472
1577
|
}
|
|
1473
1578
|
const records = await query.toArray();
|
|
1474
|
-
const
|
|
1475
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1579
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1476
1580
|
return {
|
|
1477
1581
|
pagination: {
|
|
1478
1582
|
page,
|
|
@@ -1485,7 +1589,7 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1485
1589
|
} catch (error) {
|
|
1486
1590
|
throw new MastraError(
|
|
1487
1591
|
{
|
|
1488
|
-
id: "
|
|
1592
|
+
id: createStorageErrorId("LANCE", "LIST_SCORES_BY_SPAN", "FAILED"),
|
|
1489
1593
|
text: "Failed to get scores by traceId and spanId in LanceStorage",
|
|
1490
1594
|
domain: ErrorDomain.STORAGE,
|
|
1491
1595
|
category: ErrorCategory.THIRD_PARTY,
|
|
@@ -1496,62 +1600,111 @@ var StoreScoresLance = class extends ScoresStorage {
|
|
|
1496
1600
|
}
|
|
1497
1601
|
}
|
|
1498
1602
|
};
|
|
1499
|
-
function
|
|
1500
|
-
|
|
1501
|
-
if (typeof parsedSnapshot === "string") {
|
|
1502
|
-
try {
|
|
1503
|
-
parsedSnapshot = JSON.parse(row.snapshot);
|
|
1504
|
-
} catch (e) {
|
|
1505
|
-
console.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
|
|
1506
|
-
}
|
|
1507
|
-
}
|
|
1508
|
-
return {
|
|
1509
|
-
workflowName: row.workflow_name,
|
|
1510
|
-
runId: row.run_id,
|
|
1511
|
-
snapshot: parsedSnapshot,
|
|
1512
|
-
createdAt: ensureDate(row.createdAt),
|
|
1513
|
-
updatedAt: ensureDate(row.updatedAt),
|
|
1514
|
-
resourceId: row.resourceId
|
|
1515
|
-
};
|
|
1603
|
+
function escapeSql(str) {
|
|
1604
|
+
return str.replace(/'/g, "''");
|
|
1516
1605
|
}
|
|
1517
1606
|
var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
1518
1607
|
client;
|
|
1519
|
-
|
|
1608
|
+
#db;
|
|
1609
|
+
constructor(config) {
|
|
1520
1610
|
super();
|
|
1611
|
+
const client = resolveLanceConfig(config);
|
|
1521
1612
|
this.client = client;
|
|
1613
|
+
this.#db = new LanceDB({ client });
|
|
1522
1614
|
}
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1615
|
+
parseWorkflowRun(row) {
|
|
1616
|
+
let parsedSnapshot = row.snapshot;
|
|
1617
|
+
if (typeof parsedSnapshot === "string") {
|
|
1618
|
+
try {
|
|
1619
|
+
parsedSnapshot = JSON.parse(row.snapshot);
|
|
1620
|
+
} catch (e) {
|
|
1621
|
+
this.logger.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
|
|
1622
|
+
}
|
|
1623
|
+
}
|
|
1624
|
+
return {
|
|
1625
|
+
workflowName: row.workflow_name,
|
|
1626
|
+
runId: row.run_id,
|
|
1627
|
+
snapshot: parsedSnapshot,
|
|
1628
|
+
createdAt: ensureDate(row.createdAt),
|
|
1629
|
+
updatedAt: ensureDate(row.updatedAt),
|
|
1630
|
+
resourceId: row.resourceId
|
|
1631
|
+
};
|
|
1632
|
+
}
|
|
1633
|
+
async init() {
|
|
1634
|
+
const schema = TABLE_SCHEMAS[TABLE_WORKFLOW_SNAPSHOT];
|
|
1635
|
+
await this.#db.createTable({ tableName: TABLE_WORKFLOW_SNAPSHOT, schema });
|
|
1636
|
+
await this.#db.alterTable({
|
|
1637
|
+
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
1638
|
+
schema,
|
|
1639
|
+
ifNotExists: ["resourceId"]
|
|
1640
|
+
});
|
|
1641
|
+
}
|
|
1642
|
+
async dangerouslyClearAll() {
|
|
1643
|
+
await this.#db.clearTable({ tableName: TABLE_WORKFLOW_SNAPSHOT });
|
|
1644
|
+
}
|
|
1645
|
+
async updateWorkflowResults({
|
|
1646
|
+
workflowName,
|
|
1647
|
+
runId,
|
|
1648
|
+
stepId,
|
|
1649
|
+
result,
|
|
1650
|
+
requestContext
|
|
1529
1651
|
}) {
|
|
1530
|
-
|
|
1652
|
+
let snapshot = await this.loadWorkflowSnapshot({ workflowName, runId });
|
|
1653
|
+
if (!snapshot) {
|
|
1654
|
+
snapshot = {
|
|
1655
|
+
context: {},
|
|
1656
|
+
activePaths: [],
|
|
1657
|
+
timestamp: Date.now(),
|
|
1658
|
+
suspendedPaths: {},
|
|
1659
|
+
activeStepsPath: {},
|
|
1660
|
+
resumeLabels: {},
|
|
1661
|
+
serializedStepGraph: [],
|
|
1662
|
+
status: "pending",
|
|
1663
|
+
value: {},
|
|
1664
|
+
waitingPaths: {},
|
|
1665
|
+
runId,
|
|
1666
|
+
requestContext: {}
|
|
1667
|
+
};
|
|
1668
|
+
}
|
|
1669
|
+
snapshot.context[stepId] = result;
|
|
1670
|
+
snapshot.requestContext = { ...snapshot.requestContext, ...requestContext };
|
|
1671
|
+
await this.persistWorkflowSnapshot({ workflowName, runId, snapshot });
|
|
1672
|
+
return snapshot.context;
|
|
1531
1673
|
}
|
|
1532
|
-
updateWorkflowState({
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1674
|
+
async updateWorkflowState({
|
|
1675
|
+
workflowName,
|
|
1676
|
+
runId,
|
|
1677
|
+
opts
|
|
1536
1678
|
}) {
|
|
1537
|
-
|
|
1679
|
+
const snapshot = await this.loadWorkflowSnapshot({ workflowName, runId });
|
|
1680
|
+
if (!snapshot) {
|
|
1681
|
+
return void 0;
|
|
1682
|
+
}
|
|
1683
|
+
if (!snapshot.context) {
|
|
1684
|
+
throw new Error(`Snapshot not found for runId ${runId}`);
|
|
1685
|
+
}
|
|
1686
|
+
const updatedSnapshot = { ...snapshot, ...opts };
|
|
1687
|
+
await this.persistWorkflowSnapshot({ workflowName, runId, snapshot: updatedSnapshot });
|
|
1688
|
+
return updatedSnapshot;
|
|
1538
1689
|
}
|
|
1539
1690
|
async persistWorkflowSnapshot({
|
|
1540
1691
|
workflowName,
|
|
1541
1692
|
runId,
|
|
1542
1693
|
resourceId,
|
|
1543
|
-
snapshot
|
|
1694
|
+
snapshot,
|
|
1695
|
+
createdAt,
|
|
1696
|
+
updatedAt
|
|
1544
1697
|
}) {
|
|
1545
1698
|
try {
|
|
1546
1699
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1547
|
-
const query = table.query().where(`workflow_name = '${workflowName}' AND run_id = '${runId}'`);
|
|
1700
|
+
const query = table.query().where(`workflow_name = '${escapeSql(workflowName)}' AND run_id = '${escapeSql(runId)}'`);
|
|
1548
1701
|
const records = await query.toArray();
|
|
1549
|
-
let
|
|
1550
|
-
const now = Date.now();
|
|
1702
|
+
let createdAtValue;
|
|
1703
|
+
const now = createdAt?.getTime() ?? Date.now();
|
|
1551
1704
|
if (records.length > 0) {
|
|
1552
|
-
|
|
1705
|
+
createdAtValue = records[0].createdAt ?? now;
|
|
1553
1706
|
} else {
|
|
1554
|
-
|
|
1707
|
+
createdAtValue = now;
|
|
1555
1708
|
}
|
|
1556
1709
|
const { status, value, ...rest } = snapshot;
|
|
1557
1710
|
const record = {
|
|
@@ -1560,14 +1713,14 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1560
1713
|
resourceId,
|
|
1561
1714
|
snapshot: JSON.stringify({ status, value, ...rest }),
|
|
1562
1715
|
// this is to ensure status is always just before value, for when querying the db by status
|
|
1563
|
-
createdAt,
|
|
1564
|
-
updatedAt: now
|
|
1716
|
+
createdAt: createdAtValue,
|
|
1717
|
+
updatedAt: updatedAt ?? now
|
|
1565
1718
|
};
|
|
1566
1719
|
await table.mergeInsert(["workflow_name", "run_id"]).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
1567
1720
|
} catch (error) {
|
|
1568
1721
|
throw new MastraError(
|
|
1569
1722
|
{
|
|
1570
|
-
id: "
|
|
1723
|
+
id: createStorageErrorId("LANCE", "PERSIST_WORKFLOW_SNAPSHOT", "FAILED"),
|
|
1571
1724
|
domain: ErrorDomain.STORAGE,
|
|
1572
1725
|
category: ErrorCategory.THIRD_PARTY,
|
|
1573
1726
|
details: { workflowName, runId }
|
|
@@ -1582,13 +1735,13 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1582
1735
|
}) {
|
|
1583
1736
|
try {
|
|
1584
1737
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1585
|
-
const query = table.query().where(`workflow_name = '${workflowName}' AND run_id = '${runId}'`);
|
|
1738
|
+
const query = table.query().where(`workflow_name = '${escapeSql(workflowName)}' AND run_id = '${escapeSql(runId)}'`);
|
|
1586
1739
|
const records = await query.toArray();
|
|
1587
1740
|
return records.length > 0 ? JSON.parse(records[0].snapshot) : null;
|
|
1588
1741
|
} catch (error) {
|
|
1589
1742
|
throw new MastraError(
|
|
1590
1743
|
{
|
|
1591
|
-
id: "
|
|
1744
|
+
id: createStorageErrorId("LANCE", "LOAD_WORKFLOW_SNAPSHOT", "FAILED"),
|
|
1592
1745
|
domain: ErrorDomain.STORAGE,
|
|
1593
1746
|
category: ErrorCategory.THIRD_PARTY,
|
|
1594
1747
|
details: { workflowName, runId }
|
|
@@ -1600,19 +1753,19 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1600
1753
|
async getWorkflowRunById(args) {
|
|
1601
1754
|
try {
|
|
1602
1755
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1603
|
-
let whereClause = `run_id = '${args.runId}'`;
|
|
1756
|
+
let whereClause = `run_id = '${escapeSql(args.runId)}'`;
|
|
1604
1757
|
if (args.workflowName) {
|
|
1605
|
-
whereClause += ` AND workflow_name = '${args.workflowName}'`;
|
|
1758
|
+
whereClause += ` AND workflow_name = '${escapeSql(args.workflowName)}'`;
|
|
1606
1759
|
}
|
|
1607
1760
|
const query = table.query().where(whereClause);
|
|
1608
1761
|
const records = await query.toArray();
|
|
1609
1762
|
if (records.length === 0) return null;
|
|
1610
1763
|
const record = records[0];
|
|
1611
|
-
return parseWorkflowRun(record);
|
|
1764
|
+
return this.parseWorkflowRun(record);
|
|
1612
1765
|
} catch (error) {
|
|
1613
1766
|
throw new MastraError(
|
|
1614
1767
|
{
|
|
1615
|
-
id: "
|
|
1768
|
+
id: createStorageErrorId("LANCE", "GET_WORKFLOW_RUN_BY_ID", "FAILED"),
|
|
1616
1769
|
domain: ErrorDomain.STORAGE,
|
|
1617
1770
|
category: ErrorCategory.THIRD_PARTY,
|
|
1618
1771
|
details: { runId: args.runId, workflowName: args.workflowName ?? "" }
|
|
@@ -1621,336 +1774,182 @@ var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
|
1621
1774
|
);
|
|
1622
1775
|
}
|
|
1623
1776
|
}
|
|
1777
|
+
async deleteWorkflowRunById({ runId, workflowName }) {
|
|
1778
|
+
try {
|
|
1779
|
+
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1780
|
+
const whereClause = `run_id = '${escapeSql(runId)}' AND workflow_name = '${escapeSql(workflowName)}'`;
|
|
1781
|
+
await table.delete(whereClause);
|
|
1782
|
+
} catch (error) {
|
|
1783
|
+
throw new MastraError(
|
|
1784
|
+
{
|
|
1785
|
+
id: createStorageErrorId("LANCE", "DELETE_WORKFLOW_RUN_BY_ID", "FAILED"),
|
|
1786
|
+
domain: ErrorDomain.STORAGE,
|
|
1787
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1788
|
+
details: { runId, workflowName }
|
|
1789
|
+
},
|
|
1790
|
+
error
|
|
1791
|
+
);
|
|
1792
|
+
}
|
|
1793
|
+
}
|
|
1624
1794
|
async listWorkflowRuns(args) {
|
|
1625
1795
|
try {
|
|
1626
1796
|
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1627
1797
|
let query = table.query();
|
|
1628
1798
|
const conditions = [];
|
|
1629
1799
|
if (args?.workflowName) {
|
|
1630
|
-
conditions.push(`workflow_name = '${args.workflowName
|
|
1800
|
+
conditions.push(`workflow_name = '${escapeSql(args.workflowName)}'`);
|
|
1631
1801
|
}
|
|
1632
1802
|
if (args?.status) {
|
|
1633
1803
|
const escapedStatus = args.status.replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/%/g, "\\%").replace(/_/g, "\\_");
|
|
1634
1804
|
conditions.push(`\`snapshot\` LIKE '%"status":"${escapedStatus}","value"%'`);
|
|
1635
1805
|
}
|
|
1636
1806
|
if (args?.resourceId) {
|
|
1637
|
-
conditions.push(`\`resourceId\` = '${args.resourceId}'`);
|
|
1807
|
+
conditions.push(`\`resourceId\` = '${escapeSql(args.resourceId)}'`);
|
|
1638
1808
|
}
|
|
1639
1809
|
if (args?.fromDate instanceof Date) {
|
|
1640
1810
|
conditions.push(`\`createdAt\` >= ${args.fromDate.getTime()}`);
|
|
1641
1811
|
}
|
|
1642
1812
|
if (args?.toDate instanceof Date) {
|
|
1643
|
-
conditions.push(`\`createdAt\` <= ${args.toDate.getTime()}`);
|
|
1644
|
-
}
|
|
1645
|
-
let total = 0;
|
|
1646
|
-
if (conditions.length > 0) {
|
|
1647
|
-
query = query.where(conditions.join(" AND "));
|
|
1648
|
-
total = await table.countRows(conditions.join(" AND "));
|
|
1649
|
-
} else {
|
|
1650
|
-
total = await table.countRows();
|
|
1651
|
-
}
|
|
1652
|
-
if (args?.perPage !== void 0 && args?.page !== void 0) {
|
|
1653
|
-
const normalizedPerPage = normalizePerPage(args.perPage, Number.MAX_SAFE_INTEGER);
|
|
1654
|
-
if (args.page < 0 || !Number.isInteger(args.page)) {
|
|
1655
|
-
throw new MastraError(
|
|
1656
|
-
{
|
|
1657
|
-
id: "
|
|
1658
|
-
domain: ErrorDomain.STORAGE,
|
|
1659
|
-
category: ErrorCategory.USER,
|
|
1660
|
-
details: { page: args.page, perPage: args.perPage }
|
|
1661
|
-
},
|
|
1662
|
-
new Error(`Invalid pagination parameters: page=${args.page}, perPage=${args.perPage}`)
|
|
1663
|
-
);
|
|
1664
|
-
}
|
|
1665
|
-
const offset = args.page * normalizedPerPage;
|
|
1666
|
-
query.limit(normalizedPerPage);
|
|
1667
|
-
query.offset(offset);
|
|
1668
|
-
}
|
|
1669
|
-
const records = await query.toArray();
|
|
1670
|
-
return {
|
|
1671
|
-
runs: records.map((record) => parseWorkflowRun(record)),
|
|
1672
|
-
total: total || records.length
|
|
1673
|
-
};
|
|
1674
|
-
} catch (error) {
|
|
1675
|
-
throw new MastraError(
|
|
1676
|
-
{
|
|
1677
|
-
id: "
|
|
1678
|
-
domain: ErrorDomain.STORAGE,
|
|
1679
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
1680
|
-
details: { resourceId: args?.resourceId ?? "", workflowName: args?.workflowName ?? "" }
|
|
1681
|
-
},
|
|
1682
|
-
error
|
|
1683
|
-
);
|
|
1684
|
-
}
|
|
1685
|
-
}
|
|
1686
|
-
};
|
|
1687
|
-
|
|
1688
|
-
// src/storage/index.ts
|
|
1689
|
-
var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
1690
|
-
stores;
|
|
1691
|
-
lanceClient;
|
|
1692
|
-
/**
|
|
1693
|
-
* Creates a new instance of LanceStorage
|
|
1694
|
-
* @param id The unique identifier for this storage instance
|
|
1695
|
-
* @param name The name for this storage instance
|
|
1696
|
-
* @param uri The URI to connect to LanceDB
|
|
1697
|
-
* @param options connection options
|
|
1698
|
-
*
|
|
1699
|
-
* Usage:
|
|
1700
|
-
*
|
|
1701
|
-
* Connect to a local database
|
|
1702
|
-
* ```ts
|
|
1703
|
-
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', '/path/to/db');
|
|
1704
|
-
* ```
|
|
1705
|
-
*
|
|
1706
|
-
* Connect to a LanceDB cloud database
|
|
1707
|
-
* ```ts
|
|
1708
|
-
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 'db://host:port');
|
|
1709
|
-
* ```
|
|
1710
|
-
*
|
|
1711
|
-
* Connect to a cloud database
|
|
1712
|
-
* ```ts
|
|
1713
|
-
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 's3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
1714
|
-
* ```
|
|
1715
|
-
*/
|
|
1716
|
-
static async create(id, name, uri, options) {
|
|
1717
|
-
const instance = new _LanceStorage(id, name);
|
|
1718
|
-
try {
|
|
1719
|
-
instance.lanceClient = await connect(uri, options);
|
|
1720
|
-
const operations = new StoreOperationsLance({ client: instance.lanceClient });
|
|
1721
|
-
instance.stores = {
|
|
1722
|
-
operations: new StoreOperationsLance({ client: instance.lanceClient }),
|
|
1723
|
-
workflows: new StoreWorkflowsLance({ client: instance.lanceClient }),
|
|
1724
|
-
scores: new StoreScoresLance({ client: instance.lanceClient }),
|
|
1725
|
-
memory: new StoreMemoryLance({ client: instance.lanceClient, operations })
|
|
1726
|
-
};
|
|
1727
|
-
return instance;
|
|
1728
|
-
} catch (e) {
|
|
1729
|
-
throw new MastraError(
|
|
1730
|
-
{
|
|
1731
|
-
id: "STORAGE_LANCE_STORAGE_CONNECT_FAILED",
|
|
1732
|
-
domain: ErrorDomain.STORAGE,
|
|
1733
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
1734
|
-
text: `Failed to connect to LanceDB: ${e.message || e}`,
|
|
1735
|
-
details: { uri, optionsProvided: !!options }
|
|
1736
|
-
},
|
|
1737
|
-
e
|
|
1738
|
-
);
|
|
1739
|
-
}
|
|
1740
|
-
}
|
|
1741
|
-
/**
|
|
1742
|
-
* @internal
|
|
1743
|
-
* Private constructor to enforce using the create factory method
|
|
1744
|
-
*/
|
|
1745
|
-
constructor(id, name) {
|
|
1746
|
-
super({ id, name });
|
|
1747
|
-
const operations = new StoreOperationsLance({ client: this.lanceClient });
|
|
1748
|
-
this.stores = {
|
|
1749
|
-
operations: new StoreOperationsLance({ client: this.lanceClient }),
|
|
1750
|
-
workflows: new StoreWorkflowsLance({ client: this.lanceClient }),
|
|
1751
|
-
scores: new StoreScoresLance({ client: this.lanceClient }),
|
|
1752
|
-
memory: new StoreMemoryLance({ client: this.lanceClient, operations })
|
|
1753
|
-
};
|
|
1754
|
-
}
|
|
1755
|
-
async createTable({
|
|
1756
|
-
tableName,
|
|
1757
|
-
schema
|
|
1758
|
-
}) {
|
|
1759
|
-
return this.stores.operations.createTable({ tableName, schema });
|
|
1760
|
-
}
|
|
1761
|
-
async dropTable({ tableName }) {
|
|
1762
|
-
return this.stores.operations.dropTable({ tableName });
|
|
1763
|
-
}
|
|
1764
|
-
async alterTable({
|
|
1765
|
-
tableName,
|
|
1766
|
-
schema,
|
|
1767
|
-
ifNotExists
|
|
1768
|
-
}) {
|
|
1769
|
-
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
1770
|
-
}
|
|
1771
|
-
async clearTable({ tableName }) {
|
|
1772
|
-
return this.stores.operations.clearTable({ tableName });
|
|
1773
|
-
}
|
|
1774
|
-
async insert({ tableName, record }) {
|
|
1775
|
-
return this.stores.operations.insert({ tableName, record });
|
|
1776
|
-
}
|
|
1777
|
-
async batchInsert({ tableName, records }) {
|
|
1778
|
-
return this.stores.operations.batchInsert({ tableName, records });
|
|
1779
|
-
}
|
|
1780
|
-
async load({ tableName, keys }) {
|
|
1781
|
-
return this.stores.operations.load({ tableName, keys });
|
|
1782
|
-
}
|
|
1783
|
-
async getThreadById({ threadId }) {
|
|
1784
|
-
return this.stores.memory.getThreadById({ threadId });
|
|
1785
|
-
}
|
|
1786
|
-
/**
|
|
1787
|
-
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
1788
|
-
* @param thread - The thread to save
|
|
1789
|
-
* @returns The saved thread
|
|
1790
|
-
*/
|
|
1791
|
-
async saveThread({ thread }) {
|
|
1792
|
-
return this.stores.memory.saveThread({ thread });
|
|
1793
|
-
}
|
|
1794
|
-
async updateThread({
|
|
1795
|
-
id,
|
|
1796
|
-
title,
|
|
1797
|
-
metadata
|
|
1798
|
-
}) {
|
|
1799
|
-
return this.stores.memory.updateThread({ id, title, metadata });
|
|
1800
|
-
}
|
|
1801
|
-
async deleteThread({ threadId }) {
|
|
1802
|
-
return this.stores.memory.deleteThread({ threadId });
|
|
1803
|
-
}
|
|
1804
|
-
get supports() {
|
|
1805
|
-
return {
|
|
1806
|
-
selectByIncludeResourceScope: true,
|
|
1807
|
-
resourceWorkingMemory: true,
|
|
1808
|
-
hasColumn: true,
|
|
1809
|
-
createTable: true,
|
|
1810
|
-
deleteMessages: false,
|
|
1811
|
-
listScoresBySpan: true
|
|
1812
|
-
};
|
|
1813
|
-
}
|
|
1814
|
-
async getResourceById({ resourceId }) {
|
|
1815
|
-
return this.stores.memory.getResourceById({ resourceId });
|
|
1816
|
-
}
|
|
1817
|
-
async saveResource({ resource }) {
|
|
1818
|
-
return this.stores.memory.saveResource({ resource });
|
|
1819
|
-
}
|
|
1820
|
-
async updateResource({
|
|
1821
|
-
resourceId,
|
|
1822
|
-
workingMemory,
|
|
1823
|
-
metadata
|
|
1824
|
-
}) {
|
|
1825
|
-
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
1826
|
-
}
|
|
1827
|
-
/**
|
|
1828
|
-
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
1829
|
-
* @param records - The sorted array of records to process
|
|
1830
|
-
* @param include - The array of include specifications with context parameters
|
|
1831
|
-
* @returns The processed array with context messages included
|
|
1832
|
-
*/
|
|
1833
|
-
processMessagesWithContext(records, include) {
|
|
1834
|
-
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
1835
|
-
if (messagesWithContext.length === 0) {
|
|
1836
|
-
return records;
|
|
1837
|
-
}
|
|
1838
|
-
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
1839
|
-
records.forEach((message, index) => {
|
|
1840
|
-
messageIndexMap.set(message.id, index);
|
|
1841
|
-
});
|
|
1842
|
-
const additionalIndices = /* @__PURE__ */ new Set();
|
|
1843
|
-
for (const item of messagesWithContext) {
|
|
1844
|
-
const messageIndex = messageIndexMap.get(item.id);
|
|
1845
|
-
if (messageIndex !== void 0) {
|
|
1846
|
-
if (item.withPreviousMessages) {
|
|
1847
|
-
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
1848
|
-
for (let i = startIdx; i < messageIndex; i++) {
|
|
1849
|
-
additionalIndices.add(i);
|
|
1850
|
-
}
|
|
1851
|
-
}
|
|
1852
|
-
if (item.withNextMessages) {
|
|
1853
|
-
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
1854
|
-
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
1855
|
-
additionalIndices.add(i);
|
|
1856
|
-
}
|
|
1857
|
-
}
|
|
1858
|
-
}
|
|
1859
|
-
}
|
|
1860
|
-
if (additionalIndices.size === 0) {
|
|
1861
|
-
return records;
|
|
1862
|
-
}
|
|
1863
|
-
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
1864
|
-
const allIndices = /* @__PURE__ */ new Set();
|
|
1865
|
-
records.forEach((record, index) => {
|
|
1866
|
-
if (originalMatchIds.has(record.id)) {
|
|
1867
|
-
allIndices.add(index);
|
|
1868
|
-
}
|
|
1869
|
-
});
|
|
1870
|
-
additionalIndices.forEach((index) => {
|
|
1871
|
-
allIndices.add(index);
|
|
1872
|
-
});
|
|
1873
|
-
return Array.from(allIndices).sort((a, b) => a - b).map((index) => records[index]);
|
|
1874
|
-
}
|
|
1875
|
-
async listMessagesById({ messageIds }) {
|
|
1876
|
-
return this.stores.memory.listMessagesById({ messageIds });
|
|
1877
|
-
}
|
|
1878
|
-
async saveMessages(args) {
|
|
1879
|
-
return this.stores.memory.saveMessages(args);
|
|
1880
|
-
}
|
|
1881
|
-
async updateMessages(_args) {
|
|
1882
|
-
return this.stores.memory.updateMessages(_args);
|
|
1883
|
-
}
|
|
1884
|
-
async listWorkflowRuns(args) {
|
|
1885
|
-
return this.stores.workflows.listWorkflowRuns(args);
|
|
1886
|
-
}
|
|
1887
|
-
async getWorkflowRunById(args) {
|
|
1888
|
-
return this.stores.workflows.getWorkflowRunById(args);
|
|
1889
|
-
}
|
|
1890
|
-
async updateWorkflowResults({
|
|
1891
|
-
workflowName,
|
|
1892
|
-
runId,
|
|
1893
|
-
stepId,
|
|
1894
|
-
result,
|
|
1895
|
-
requestContext
|
|
1896
|
-
}) {
|
|
1897
|
-
return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, requestContext });
|
|
1898
|
-
}
|
|
1899
|
-
async updateWorkflowState({
|
|
1900
|
-
workflowName,
|
|
1901
|
-
runId,
|
|
1902
|
-
opts
|
|
1903
|
-
}) {
|
|
1904
|
-
return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
|
|
1905
|
-
}
|
|
1906
|
-
async persistWorkflowSnapshot({
|
|
1907
|
-
workflowName,
|
|
1908
|
-
runId,
|
|
1909
|
-
resourceId,
|
|
1910
|
-
snapshot
|
|
1911
|
-
}) {
|
|
1912
|
-
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot });
|
|
1913
|
-
}
|
|
1914
|
-
async loadWorkflowSnapshot({
|
|
1915
|
-
workflowName,
|
|
1916
|
-
runId
|
|
1917
|
-
}) {
|
|
1918
|
-
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
1919
|
-
}
|
|
1920
|
-
async getScoreById({ id: _id }) {
|
|
1921
|
-
return this.stores.scores.getScoreById({ id: _id });
|
|
1922
|
-
}
|
|
1923
|
-
async listScoresByScorerId({
|
|
1924
|
-
scorerId,
|
|
1925
|
-
source,
|
|
1926
|
-
entityId,
|
|
1927
|
-
entityType,
|
|
1928
|
-
pagination
|
|
1929
|
-
}) {
|
|
1930
|
-
return this.stores.scores.listScoresByScorerId({ scorerId, source, pagination, entityId, entityType });
|
|
1931
|
-
}
|
|
1932
|
-
async saveScore(_score) {
|
|
1933
|
-
return this.stores.scores.saveScore(_score);
|
|
1813
|
+
conditions.push(`\`createdAt\` <= ${args.toDate.getTime()}`);
|
|
1814
|
+
}
|
|
1815
|
+
let total = 0;
|
|
1816
|
+
if (conditions.length > 0) {
|
|
1817
|
+
query = query.where(conditions.join(" AND "));
|
|
1818
|
+
total = await table.countRows(conditions.join(" AND "));
|
|
1819
|
+
} else {
|
|
1820
|
+
total = await table.countRows();
|
|
1821
|
+
}
|
|
1822
|
+
if (args?.perPage !== void 0 && args?.page !== void 0) {
|
|
1823
|
+
const normalizedPerPage = normalizePerPage(args.perPage, Number.MAX_SAFE_INTEGER);
|
|
1824
|
+
if (args.page < 0 || !Number.isInteger(args.page)) {
|
|
1825
|
+
throw new MastraError(
|
|
1826
|
+
{
|
|
1827
|
+
id: createStorageErrorId("LANCE", "LIST_WORKFLOW_RUNS", "INVALID_PAGINATION"),
|
|
1828
|
+
domain: ErrorDomain.STORAGE,
|
|
1829
|
+
category: ErrorCategory.USER,
|
|
1830
|
+
details: { page: args.page, perPage: args.perPage }
|
|
1831
|
+
},
|
|
1832
|
+
new Error(`Invalid pagination parameters: page=${args.page}, perPage=${args.perPage}`)
|
|
1833
|
+
);
|
|
1834
|
+
}
|
|
1835
|
+
const offset = args.page * normalizedPerPage;
|
|
1836
|
+
query.limit(normalizedPerPage);
|
|
1837
|
+
query.offset(offset);
|
|
1838
|
+
}
|
|
1839
|
+
const records = await query.toArray();
|
|
1840
|
+
return {
|
|
1841
|
+
runs: records.map((record) => this.parseWorkflowRun(record)),
|
|
1842
|
+
total: total || records.length
|
|
1843
|
+
};
|
|
1844
|
+
} catch (error) {
|
|
1845
|
+
throw new MastraError(
|
|
1846
|
+
{
|
|
1847
|
+
id: createStorageErrorId("LANCE", "LIST_WORKFLOW_RUNS", "FAILED"),
|
|
1848
|
+
domain: ErrorDomain.STORAGE,
|
|
1849
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1850
|
+
details: { resourceId: args?.resourceId ?? "", workflowName: args?.workflowName ?? "" }
|
|
1851
|
+
},
|
|
1852
|
+
error
|
|
1853
|
+
);
|
|
1854
|
+
}
|
|
1934
1855
|
}
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1856
|
+
};
|
|
1857
|
+
|
|
1858
|
+
// src/storage/index.ts
|
|
1859
|
+
var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
1860
|
+
stores;
|
|
1861
|
+
lanceClient;
|
|
1862
|
+
/**
|
|
1863
|
+
* Creates a new instance of LanceStorage
|
|
1864
|
+
* @param id The unique identifier for this storage instance
|
|
1865
|
+
* @param name The name for this storage instance
|
|
1866
|
+
* @param uri The URI to connect to LanceDB
|
|
1867
|
+
* @param connectionOptions connection options for LanceDB
|
|
1868
|
+
* @param storageOptions storage options including disableInit
|
|
1869
|
+
*
|
|
1870
|
+
* Usage:
|
|
1871
|
+
*
|
|
1872
|
+
* Connect to a local database
|
|
1873
|
+
* ```ts
|
|
1874
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', '/path/to/db');
|
|
1875
|
+
* ```
|
|
1876
|
+
*
|
|
1877
|
+
* Connect to a LanceDB cloud database
|
|
1878
|
+
* ```ts
|
|
1879
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 'db://host:port');
|
|
1880
|
+
* ```
|
|
1881
|
+
*
|
|
1882
|
+
* Connect to a cloud database
|
|
1883
|
+
* ```ts
|
|
1884
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 's3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
1885
|
+
* ```
|
|
1886
|
+
*
|
|
1887
|
+
* Disable auto-init for runtime (after CI/CD has run migrations)
|
|
1888
|
+
* ```ts
|
|
1889
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', '/path/to/db', undefined, { disableInit: true });
|
|
1890
|
+
* ```
|
|
1891
|
+
*/
|
|
1892
|
+
static async create(id, name, uri, connectionOptions, storageOptions) {
|
|
1893
|
+
const instance = new _LanceStorage(id, name, storageOptions?.disableInit);
|
|
1894
|
+
try {
|
|
1895
|
+
instance.lanceClient = await connect(uri, connectionOptions);
|
|
1896
|
+
instance.stores = {
|
|
1897
|
+
workflows: new StoreWorkflowsLance({ client: instance.lanceClient }),
|
|
1898
|
+
scores: new StoreScoresLance({ client: instance.lanceClient }),
|
|
1899
|
+
memory: new StoreMemoryLance({ client: instance.lanceClient })
|
|
1900
|
+
};
|
|
1901
|
+
return instance;
|
|
1902
|
+
} catch (e) {
|
|
1903
|
+
throw new MastraError(
|
|
1904
|
+
{
|
|
1905
|
+
id: createStorageErrorId("LANCE", "CONNECT", "FAILED"),
|
|
1906
|
+
domain: ErrorDomain.STORAGE,
|
|
1907
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1908
|
+
text: `Failed to connect to LanceDB: ${e.message || e}`,
|
|
1909
|
+
details: { uri, optionsProvided: !!connectionOptions }
|
|
1910
|
+
},
|
|
1911
|
+
e
|
|
1912
|
+
);
|
|
1913
|
+
}
|
|
1940
1914
|
}
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1915
|
+
/**
|
|
1916
|
+
* Creates a new instance of LanceStorage from a pre-configured LanceDB connection.
|
|
1917
|
+
* Use this when you need to configure the connection before initialization.
|
|
1918
|
+
*
|
|
1919
|
+
* @param id The unique identifier for this storage instance
|
|
1920
|
+
* @param name The name for this storage instance
|
|
1921
|
+
* @param client Pre-configured LanceDB connection
|
|
1922
|
+
* @param options Storage options including disableInit
|
|
1923
|
+
*
|
|
1924
|
+
* @example
|
|
1925
|
+
* ```typescript
|
|
1926
|
+
* import { connect } from '@lancedb/lancedb';
|
|
1927
|
+
*
|
|
1928
|
+
* const client = await connect('/path/to/db', {
|
|
1929
|
+
* // Custom connection options
|
|
1930
|
+
* });
|
|
1931
|
+
*
|
|
1932
|
+
* const store = LanceStorage.fromClient('my-id', 'MyStorage', client);
|
|
1933
|
+
* ```
|
|
1934
|
+
*/
|
|
1935
|
+
static fromClient(id, name, client, options) {
|
|
1936
|
+
const instance = new _LanceStorage(id, name, options?.disableInit);
|
|
1937
|
+
instance.lanceClient = client;
|
|
1938
|
+
instance.stores = {
|
|
1939
|
+
workflows: new StoreWorkflowsLance({ client }),
|
|
1940
|
+
scores: new StoreScoresLance({ client }),
|
|
1941
|
+
memory: new StoreMemoryLance({ client })
|
|
1942
|
+
};
|
|
1943
|
+
return instance;
|
|
1947
1944
|
}
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1945
|
+
/**
|
|
1946
|
+
* @internal
|
|
1947
|
+
* Private constructor to enforce using the create factory method.
|
|
1948
|
+
* Note: stores is initialized in create() after the lanceClient is connected.
|
|
1949
|
+
*/
|
|
1950
|
+
constructor(id, name, disableInit) {
|
|
1951
|
+
super({ id, name, disableInit });
|
|
1952
|
+
this.stores = {};
|
|
1954
1953
|
}
|
|
1955
1954
|
};
|
|
1956
1955
|
var LanceFilterTranslator = class extends BaseFilterTranslator {
|
|
@@ -2306,7 +2305,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2306
2305
|
} catch (e) {
|
|
2307
2306
|
throw new MastraError(
|
|
2308
2307
|
{
|
|
2309
|
-
id: "
|
|
2308
|
+
id: createVectorErrorId("LANCE", "CONNECT", "FAILED"),
|
|
2310
2309
|
domain: ErrorDomain.STORAGE,
|
|
2311
2310
|
category: ErrorCategory.THIRD_PARTY,
|
|
2312
2311
|
details: { uri }
|
|
@@ -2329,6 +2328,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2329
2328
|
}
|
|
2330
2329
|
async query({
|
|
2331
2330
|
tableName,
|
|
2331
|
+
indexName,
|
|
2332
2332
|
queryVector,
|
|
2333
2333
|
filter,
|
|
2334
2334
|
includeVector = false,
|
|
@@ -2336,12 +2336,13 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2336
2336
|
columns = [],
|
|
2337
2337
|
includeAllColumns = false
|
|
2338
2338
|
}) {
|
|
2339
|
+
const resolvedTableName = tableName ?? indexName;
|
|
2339
2340
|
try {
|
|
2340
2341
|
if (!this.lanceClient) {
|
|
2341
2342
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2342
2343
|
}
|
|
2343
|
-
if (!
|
|
2344
|
-
throw new Error("tableName is required");
|
|
2344
|
+
if (!resolvedTableName) {
|
|
2345
|
+
throw new Error("tableName or indexName is required");
|
|
2345
2346
|
}
|
|
2346
2347
|
if (!queryVector) {
|
|
2347
2348
|
throw new Error("queryVector is required");
|
|
@@ -2349,28 +2350,33 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2349
2350
|
} catch (error) {
|
|
2350
2351
|
throw new MastraError(
|
|
2351
2352
|
{
|
|
2352
|
-
id: "
|
|
2353
|
+
id: createVectorErrorId("LANCE", "QUERY", "INVALID_ARGS"),
|
|
2353
2354
|
domain: ErrorDomain.STORAGE,
|
|
2354
2355
|
category: ErrorCategory.USER,
|
|
2355
|
-
text:
|
|
2356
|
-
details: { tableName }
|
|
2356
|
+
text: error instanceof Error ? error.message : "Invalid query arguments",
|
|
2357
|
+
details: { tableName: resolvedTableName }
|
|
2357
2358
|
},
|
|
2358
2359
|
error
|
|
2359
2360
|
);
|
|
2360
2361
|
}
|
|
2361
2362
|
try {
|
|
2362
|
-
const
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2363
|
+
const tables = await this.lanceClient.tableNames();
|
|
2364
|
+
if (!tables.includes(resolvedTableName)) {
|
|
2365
|
+
this.logger.debug(`Table ${resolvedTableName} does not exist. Returning empty results.`);
|
|
2366
|
+
return [];
|
|
2366
2367
|
}
|
|
2368
|
+
const table = await this.lanceClient.openTable(resolvedTableName);
|
|
2367
2369
|
let query = table.search(queryVector);
|
|
2368
2370
|
if (filter && Object.keys(filter).length > 0) {
|
|
2369
2371
|
const whereClause = this.filterTranslator(filter);
|
|
2370
2372
|
this.logger.debug(`Where clause generated: ${whereClause}`);
|
|
2371
2373
|
query = query.where(whereClause);
|
|
2372
2374
|
}
|
|
2373
|
-
if (!includeAllColumns &&
|
|
2375
|
+
if (!includeAllColumns && columns.length > 0) {
|
|
2376
|
+
const selectColumns = [...columns];
|
|
2377
|
+
if (!selectColumns.includes("id")) {
|
|
2378
|
+
selectColumns.push("id");
|
|
2379
|
+
}
|
|
2374
2380
|
query = query.select(selectColumns);
|
|
2375
2381
|
}
|
|
2376
2382
|
query = query.limit(topK);
|
|
@@ -2397,10 +2403,10 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2397
2403
|
} catch (error) {
|
|
2398
2404
|
throw new MastraError(
|
|
2399
2405
|
{
|
|
2400
|
-
id: "
|
|
2406
|
+
id: createVectorErrorId("LANCE", "QUERY", "FAILED"),
|
|
2401
2407
|
domain: ErrorDomain.STORAGE,
|
|
2402
2408
|
category: ErrorCategory.THIRD_PARTY,
|
|
2403
|
-
details: { tableName, includeVector, columnsCount: columns?.length, includeAllColumns }
|
|
2409
|
+
details: { tableName: resolvedTableName, includeVector, columnsCount: columns?.length, includeAllColumns }
|
|
2404
2410
|
},
|
|
2405
2411
|
error
|
|
2406
2412
|
);
|
|
@@ -2435,13 +2441,14 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2435
2441
|
const translator = new LanceFilterTranslator();
|
|
2436
2442
|
return translator.translate(prefixedFilter);
|
|
2437
2443
|
}
|
|
2438
|
-
async upsert({ tableName, vectors, metadata = [], ids = [] }) {
|
|
2444
|
+
async upsert({ tableName, indexName, vectors, metadata = [], ids = [] }) {
|
|
2445
|
+
const resolvedTableName = tableName ?? indexName;
|
|
2439
2446
|
try {
|
|
2440
2447
|
if (!this.lanceClient) {
|
|
2441
2448
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2442
2449
|
}
|
|
2443
|
-
if (!
|
|
2444
|
-
throw new Error("tableName is required");
|
|
2450
|
+
if (!resolvedTableName) {
|
|
2451
|
+
throw new Error("tableName or indexName is required");
|
|
2445
2452
|
}
|
|
2446
2453
|
if (!vectors || !Array.isArray(vectors) || vectors.length === 0) {
|
|
2447
2454
|
throw new Error("vectors array is required and must not be empty");
|
|
@@ -2449,21 +2456,24 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2449
2456
|
} catch (error) {
|
|
2450
2457
|
throw new MastraError(
|
|
2451
2458
|
{
|
|
2452
|
-
id: "
|
|
2459
|
+
id: createVectorErrorId("LANCE", "UPSERT", "INVALID_ARGS"),
|
|
2453
2460
|
domain: ErrorDomain.STORAGE,
|
|
2454
2461
|
category: ErrorCategory.USER,
|
|
2455
|
-
text:
|
|
2456
|
-
details: { tableName }
|
|
2462
|
+
text: error instanceof Error ? error.message : "Invalid upsert arguments",
|
|
2463
|
+
details: { tableName: resolvedTableName }
|
|
2457
2464
|
},
|
|
2458
2465
|
error
|
|
2459
2466
|
);
|
|
2460
2467
|
}
|
|
2461
2468
|
try {
|
|
2462
2469
|
const tables = await this.lanceClient.tableNames();
|
|
2463
|
-
|
|
2464
|
-
|
|
2470
|
+
const tableExists = tables.includes(resolvedTableName);
|
|
2471
|
+
let table = null;
|
|
2472
|
+
if (!tableExists) {
|
|
2473
|
+
this.logger.debug(`Table ${resolvedTableName} does not exist. Creating it with the first upsert data.`);
|
|
2474
|
+
} else {
|
|
2475
|
+
table = await this.lanceClient.openTable(resolvedTableName);
|
|
2465
2476
|
}
|
|
2466
|
-
const table = await this.lanceClient.openTable(tableName);
|
|
2467
2477
|
const vectorIds = ids.length === vectors.length ? ids : vectors.map((_, i) => ids[i] || crypto.randomUUID());
|
|
2468
2478
|
const data = vectors.map((vector, i) => {
|
|
2469
2479
|
const id = String(vectorIds[i]);
|
|
@@ -2480,15 +2490,55 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2480
2490
|
}
|
|
2481
2491
|
return rowData;
|
|
2482
2492
|
});
|
|
2483
|
-
|
|
2493
|
+
if (table !== null) {
|
|
2494
|
+
const rowCount = await table.countRows();
|
|
2495
|
+
const schema = await table.schema();
|
|
2496
|
+
const existingColumns = new Set(schema.fields.map((f) => f.name));
|
|
2497
|
+
const dataColumns = new Set(Object.keys(data[0] || {}));
|
|
2498
|
+
const extraColumns = [...dataColumns].filter((col) => !existingColumns.has(col));
|
|
2499
|
+
const missingSchemaColumns = [...existingColumns].filter((col) => !dataColumns.has(col));
|
|
2500
|
+
const hasSchemaMismatch = extraColumns.length > 0 || missingSchemaColumns.length > 0;
|
|
2501
|
+
if (rowCount === 0 && extraColumns.length > 0) {
|
|
2502
|
+
this.logger.warn(
|
|
2503
|
+
`Table ${resolvedTableName} is empty and data has extra columns ${extraColumns.join(", ")}. Recreating with new schema.`
|
|
2504
|
+
);
|
|
2505
|
+
await this.lanceClient.dropTable(resolvedTableName);
|
|
2506
|
+
await this.lanceClient.createTable(resolvedTableName, data);
|
|
2507
|
+
} else if (hasSchemaMismatch) {
|
|
2508
|
+
if (extraColumns.length > 0) {
|
|
2509
|
+
this.logger.warn(
|
|
2510
|
+
`Table ${resolvedTableName} has ${rowCount} rows. Columns ${extraColumns.join(", ")} will be dropped from upsert.`
|
|
2511
|
+
);
|
|
2512
|
+
}
|
|
2513
|
+
const schemaFieldNames = schema.fields.map((f) => f.name);
|
|
2514
|
+
const normalizedData = data.map((row) => {
|
|
2515
|
+
const normalized = {};
|
|
2516
|
+
for (const col of schemaFieldNames) {
|
|
2517
|
+
normalized[col] = col in row ? row[col] : null;
|
|
2518
|
+
}
|
|
2519
|
+
return normalized;
|
|
2520
|
+
});
|
|
2521
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(normalizedData);
|
|
2522
|
+
} else {
|
|
2523
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(data);
|
|
2524
|
+
}
|
|
2525
|
+
} else {
|
|
2526
|
+
this.logger.debug(`Creating table ${resolvedTableName} with initial data`);
|
|
2527
|
+
await this.lanceClient.createTable(resolvedTableName, data);
|
|
2528
|
+
}
|
|
2484
2529
|
return vectorIds;
|
|
2485
2530
|
} catch (error) {
|
|
2486
2531
|
throw new MastraError(
|
|
2487
2532
|
{
|
|
2488
|
-
id: "
|
|
2533
|
+
id: createVectorErrorId("LANCE", "UPSERT", "FAILED"),
|
|
2489
2534
|
domain: ErrorDomain.STORAGE,
|
|
2490
2535
|
category: ErrorCategory.THIRD_PARTY,
|
|
2491
|
-
details: {
|
|
2536
|
+
details: {
|
|
2537
|
+
tableName: resolvedTableName,
|
|
2538
|
+
vectorCount: vectors.length,
|
|
2539
|
+
metadataCount: metadata.length,
|
|
2540
|
+
idsCount: ids.length
|
|
2541
|
+
}
|
|
2492
2542
|
},
|
|
2493
2543
|
error
|
|
2494
2544
|
);
|
|
@@ -2512,7 +2562,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2512
2562
|
async createTable(tableName, data, options) {
|
|
2513
2563
|
if (!this.lanceClient) {
|
|
2514
2564
|
throw new MastraError({
|
|
2515
|
-
id: "
|
|
2565
|
+
id: createVectorErrorId("LANCE", "CREATE_TABLE", "INVALID_ARGS"),
|
|
2516
2566
|
domain: ErrorDomain.STORAGE,
|
|
2517
2567
|
category: ErrorCategory.USER,
|
|
2518
2568
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2527,7 +2577,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2527
2577
|
} catch (error) {
|
|
2528
2578
|
throw new MastraError(
|
|
2529
2579
|
{
|
|
2530
|
-
id: "
|
|
2580
|
+
id: createVectorErrorId("LANCE", "CREATE_TABLE", "FAILED"),
|
|
2531
2581
|
domain: ErrorDomain.STORAGE,
|
|
2532
2582
|
category: ErrorCategory.THIRD_PARTY,
|
|
2533
2583
|
details: { tableName }
|
|
@@ -2539,7 +2589,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2539
2589
|
async listTables() {
|
|
2540
2590
|
if (!this.lanceClient) {
|
|
2541
2591
|
throw new MastraError({
|
|
2542
|
-
id: "
|
|
2592
|
+
id: createVectorErrorId("LANCE", "LIST_TABLES", "INVALID_ARGS"),
|
|
2543
2593
|
domain: ErrorDomain.STORAGE,
|
|
2544
2594
|
category: ErrorCategory.USER,
|
|
2545
2595
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2551,7 +2601,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2551
2601
|
} catch (error) {
|
|
2552
2602
|
throw new MastraError(
|
|
2553
2603
|
{
|
|
2554
|
-
id: "
|
|
2604
|
+
id: createVectorErrorId("LANCE", "LIST_TABLES", "FAILED"),
|
|
2555
2605
|
domain: ErrorDomain.STORAGE,
|
|
2556
2606
|
category: ErrorCategory.THIRD_PARTY
|
|
2557
2607
|
},
|
|
@@ -2562,7 +2612,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2562
2612
|
async getTableSchema(tableName) {
|
|
2563
2613
|
if (!this.lanceClient) {
|
|
2564
2614
|
throw new MastraError({
|
|
2565
|
-
id: "
|
|
2615
|
+
id: createVectorErrorId("LANCE", "GET_TABLE_SCHEMA", "INVALID_ARGS"),
|
|
2566
2616
|
domain: ErrorDomain.STORAGE,
|
|
2567
2617
|
category: ErrorCategory.USER,
|
|
2568
2618
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2575,7 +2625,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2575
2625
|
} catch (error) {
|
|
2576
2626
|
throw new MastraError(
|
|
2577
2627
|
{
|
|
2578
|
-
id: "
|
|
2628
|
+
id: createVectorErrorId("LANCE", "GET_TABLE_SCHEMA", "FAILED"),
|
|
2579
2629
|
domain: ErrorDomain.STORAGE,
|
|
2580
2630
|
category: ErrorCategory.THIRD_PARTY,
|
|
2581
2631
|
details: { tableName }
|
|
@@ -2585,7 +2635,17 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2585
2635
|
}
|
|
2586
2636
|
}
|
|
2587
2637
|
/**
|
|
2588
|
-
*
|
|
2638
|
+
* Creates a vector index on a table.
|
|
2639
|
+
*
|
|
2640
|
+
* The behavior of `indexName` depends on whether `tableName` is provided:
|
|
2641
|
+
* - With `tableName`: `indexName` is the column to index (advanced use case)
|
|
2642
|
+
* - Without `tableName`: `indexName` becomes the table name, and 'vector' is used as the column (Memory compatibility)
|
|
2643
|
+
*
|
|
2644
|
+
* @param tableName - Optional table name. If not provided, defaults to indexName.
|
|
2645
|
+
* @param indexName - The index/column name, or table name if tableName is not provided.
|
|
2646
|
+
* @param dimension - Vector dimension size.
|
|
2647
|
+
* @param metric - Distance metric: 'cosine', 'euclidean', or 'dotproduct'.
|
|
2648
|
+
* @param indexConfig - Optional index configuration.
|
|
2589
2649
|
*/
|
|
2590
2650
|
async createIndex({
|
|
2591
2651
|
tableName,
|
|
@@ -2594,13 +2654,12 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2594
2654
|
metric = "cosine",
|
|
2595
2655
|
indexConfig = {}
|
|
2596
2656
|
}) {
|
|
2657
|
+
const resolvedTableName = tableName ?? indexName;
|
|
2658
|
+
const columnToIndex = tableName ? indexName : "vector";
|
|
2597
2659
|
try {
|
|
2598
2660
|
if (!this.lanceClient) {
|
|
2599
2661
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2600
2662
|
}
|
|
2601
|
-
if (!tableName) {
|
|
2602
|
-
throw new Error("tableName is required");
|
|
2603
|
-
}
|
|
2604
2663
|
if (!indexName) {
|
|
2605
2664
|
throw new Error("indexName is required");
|
|
2606
2665
|
}
|
|
@@ -2610,22 +2669,36 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2610
2669
|
} catch (err) {
|
|
2611
2670
|
throw new MastraError(
|
|
2612
2671
|
{
|
|
2613
|
-
id: "
|
|
2672
|
+
id: createVectorErrorId("LANCE", "CREATE_INDEX", "INVALID_ARGS"),
|
|
2614
2673
|
domain: ErrorDomain.STORAGE,
|
|
2615
2674
|
category: ErrorCategory.USER,
|
|
2616
|
-
details: { tableName:
|
|
2675
|
+
details: { tableName: resolvedTableName, indexName, dimension, metric }
|
|
2617
2676
|
},
|
|
2618
2677
|
err
|
|
2619
2678
|
);
|
|
2620
2679
|
}
|
|
2621
2680
|
try {
|
|
2622
2681
|
const tables = await this.lanceClient.tableNames();
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2682
|
+
let table;
|
|
2683
|
+
if (!tables.includes(resolvedTableName)) {
|
|
2684
|
+
this.logger.debug(
|
|
2685
|
+
`Table ${resolvedTableName} does not exist. Creating empty table with dimension ${dimension}.`
|
|
2626
2686
|
);
|
|
2687
|
+
const initVector = new Array(dimension).fill(0);
|
|
2688
|
+
table = await this.lanceClient.createTable(resolvedTableName, [{ id: "__init__", vector: initVector }]);
|
|
2689
|
+
try {
|
|
2690
|
+
await table.delete("id = '__init__'");
|
|
2691
|
+
} catch (deleteError) {
|
|
2692
|
+
this.logger.warn(
|
|
2693
|
+
`Failed to delete initialization row from ${resolvedTableName}. Subsequent queries may include '__init__' row.`,
|
|
2694
|
+
deleteError
|
|
2695
|
+
);
|
|
2696
|
+
}
|
|
2697
|
+
this.logger.debug(`Table ${resolvedTableName} created. Index creation deferred until data is available.`);
|
|
2698
|
+
return;
|
|
2699
|
+
} else {
|
|
2700
|
+
table = await this.lanceClient.openTable(resolvedTableName);
|
|
2627
2701
|
}
|
|
2628
|
-
const table = await this.lanceClient.openTable(tableName);
|
|
2629
2702
|
let metricType;
|
|
2630
2703
|
if (metric === "euclidean") {
|
|
2631
2704
|
metricType = "l2";
|
|
@@ -2634,8 +2707,15 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2634
2707
|
} else if (metric === "cosine") {
|
|
2635
2708
|
metricType = "cosine";
|
|
2636
2709
|
}
|
|
2710
|
+
const rowCount = await table.countRows();
|
|
2711
|
+
if (rowCount < 256) {
|
|
2712
|
+
this.logger.warn(
|
|
2713
|
+
`Table ${resolvedTableName} has ${rowCount} rows, which is below the 256 row minimum for index creation. Skipping index creation.`
|
|
2714
|
+
);
|
|
2715
|
+
return;
|
|
2716
|
+
}
|
|
2637
2717
|
if (indexConfig.type === "ivfflat") {
|
|
2638
|
-
await table.createIndex(
|
|
2718
|
+
await table.createIndex(columnToIndex, {
|
|
2639
2719
|
config: Index.ivfPq({
|
|
2640
2720
|
numPartitions: indexConfig.numPartitions || 128,
|
|
2641
2721
|
numSubVectors: indexConfig.numSubVectors || 16,
|
|
@@ -2644,7 +2724,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2644
2724
|
});
|
|
2645
2725
|
} else {
|
|
2646
2726
|
this.logger.debug("Creating HNSW PQ index with config:", indexConfig);
|
|
2647
|
-
await table.createIndex(
|
|
2727
|
+
await table.createIndex(columnToIndex, {
|
|
2648
2728
|
config: Index.hnswPq({
|
|
2649
2729
|
m: indexConfig?.hnsw?.m || 16,
|
|
2650
2730
|
efConstruction: indexConfig?.hnsw?.efConstruction || 100,
|
|
@@ -2655,10 +2735,10 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2655
2735
|
} catch (error) {
|
|
2656
2736
|
throw new MastraError(
|
|
2657
2737
|
{
|
|
2658
|
-
id: "
|
|
2738
|
+
id: createVectorErrorId("LANCE", "CREATE_INDEX", "FAILED"),
|
|
2659
2739
|
domain: ErrorDomain.STORAGE,
|
|
2660
2740
|
category: ErrorCategory.THIRD_PARTY,
|
|
2661
|
-
details: { tableName:
|
|
2741
|
+
details: { tableName: resolvedTableName, indexName, dimension }
|
|
2662
2742
|
},
|
|
2663
2743
|
error
|
|
2664
2744
|
);
|
|
@@ -2667,7 +2747,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2667
2747
|
async listIndexes() {
|
|
2668
2748
|
if (!this.lanceClient) {
|
|
2669
2749
|
throw new MastraError({
|
|
2670
|
-
id: "
|
|
2750
|
+
id: createVectorErrorId("LANCE", "LIST_INDEXES", "INVALID_ARGS"),
|
|
2671
2751
|
domain: ErrorDomain.STORAGE,
|
|
2672
2752
|
category: ErrorCategory.USER,
|
|
2673
2753
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2686,7 +2766,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2686
2766
|
} catch (error) {
|
|
2687
2767
|
throw new MastraError(
|
|
2688
2768
|
{
|
|
2689
|
-
id: "
|
|
2769
|
+
id: createVectorErrorId("LANCE", "LIST_INDEXES", "FAILED"),
|
|
2690
2770
|
domain: ErrorDomain.STORAGE,
|
|
2691
2771
|
category: ErrorCategory.THIRD_PARTY
|
|
2692
2772
|
},
|
|
@@ -2705,7 +2785,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2705
2785
|
} catch (err) {
|
|
2706
2786
|
throw new MastraError(
|
|
2707
2787
|
{
|
|
2708
|
-
id: "
|
|
2788
|
+
id: createVectorErrorId("LANCE", "DESCRIBE_INDEX", "INVALID_ARGS"),
|
|
2709
2789
|
domain: ErrorDomain.STORAGE,
|
|
2710
2790
|
category: ErrorCategory.USER,
|
|
2711
2791
|
details: { indexName }
|
|
@@ -2740,7 +2820,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2740
2820
|
} catch (error) {
|
|
2741
2821
|
throw new MastraError(
|
|
2742
2822
|
{
|
|
2743
|
-
id: "
|
|
2823
|
+
id: createVectorErrorId("LANCE", "DESCRIBE_INDEX", "FAILED"),
|
|
2744
2824
|
domain: ErrorDomain.STORAGE,
|
|
2745
2825
|
category: ErrorCategory.THIRD_PARTY,
|
|
2746
2826
|
details: { indexName }
|
|
@@ -2760,7 +2840,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2760
2840
|
} catch (err) {
|
|
2761
2841
|
throw new MastraError(
|
|
2762
2842
|
{
|
|
2763
|
-
id: "
|
|
2843
|
+
id: createVectorErrorId("LANCE", "DELETE_INDEX", "INVALID_ARGS"),
|
|
2764
2844
|
domain: ErrorDomain.STORAGE,
|
|
2765
2845
|
category: ErrorCategory.USER,
|
|
2766
2846
|
details: { indexName }
|
|
@@ -2783,7 +2863,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2783
2863
|
} catch (error) {
|
|
2784
2864
|
throw new MastraError(
|
|
2785
2865
|
{
|
|
2786
|
-
id: "
|
|
2866
|
+
id: createVectorErrorId("LANCE", "DELETE_INDEX", "FAILED"),
|
|
2787
2867
|
domain: ErrorDomain.STORAGE,
|
|
2788
2868
|
category: ErrorCategory.THIRD_PARTY,
|
|
2789
2869
|
details: { indexName }
|
|
@@ -2798,7 +2878,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2798
2878
|
async deleteAllTables() {
|
|
2799
2879
|
if (!this.lanceClient) {
|
|
2800
2880
|
throw new MastraError({
|
|
2801
|
-
id: "
|
|
2881
|
+
id: createVectorErrorId("LANCE", "DELETE_ALL_TABLES", "INVALID_ARGS"),
|
|
2802
2882
|
domain: ErrorDomain.STORAGE,
|
|
2803
2883
|
category: ErrorCategory.USER,
|
|
2804
2884
|
details: { methodName: "deleteAllTables" },
|
|
@@ -2810,7 +2890,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2810
2890
|
} catch (error) {
|
|
2811
2891
|
throw new MastraError(
|
|
2812
2892
|
{
|
|
2813
|
-
id: "
|
|
2893
|
+
id: createVectorErrorId("LANCE", "DELETE_ALL_TABLES", "FAILED"),
|
|
2814
2894
|
domain: ErrorDomain.STORAGE,
|
|
2815
2895
|
category: ErrorCategory.THIRD_PARTY,
|
|
2816
2896
|
details: { methodName: "deleteAllTables" }
|
|
@@ -2822,7 +2902,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2822
2902
|
async deleteTable(tableName) {
|
|
2823
2903
|
if (!this.lanceClient) {
|
|
2824
2904
|
throw new MastraError({
|
|
2825
|
-
id: "
|
|
2905
|
+
id: createVectorErrorId("LANCE", "DELETE_TABLE", "INVALID_ARGS"),
|
|
2826
2906
|
domain: ErrorDomain.STORAGE,
|
|
2827
2907
|
category: ErrorCategory.USER,
|
|
2828
2908
|
details: { tableName },
|
|
@@ -2834,7 +2914,7 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2834
2914
|
} catch (error) {
|
|
2835
2915
|
throw new MastraError(
|
|
2836
2916
|
{
|
|
2837
|
-
id: "
|
|
2917
|
+
id: createVectorErrorId("LANCE", "DELETE_TABLE", "FAILED"),
|
|
2838
2918
|
domain: ErrorDomain.STORAGE,
|
|
2839
2919
|
category: ErrorCategory.THIRD_PARTY,
|
|
2840
2920
|
details: { tableName }
|
|
@@ -2843,7 +2923,44 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2843
2923
|
);
|
|
2844
2924
|
}
|
|
2845
2925
|
}
|
|
2846
|
-
async updateVector(
|
|
2926
|
+
async updateVector(params) {
|
|
2927
|
+
const { indexName, update } = params;
|
|
2928
|
+
if ("id" in params && "filter" in params && params.id && params.filter) {
|
|
2929
|
+
throw new MastraError({
|
|
2930
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "MUTUALLY_EXCLUSIVE"),
|
|
2931
|
+
domain: ErrorDomain.STORAGE,
|
|
2932
|
+
category: ErrorCategory.USER,
|
|
2933
|
+
text: "id and filter are mutually exclusive",
|
|
2934
|
+
details: { indexName }
|
|
2935
|
+
});
|
|
2936
|
+
}
|
|
2937
|
+
if (!("id" in params || "filter" in params) || !params.id && !params.filter) {
|
|
2938
|
+
throw new MastraError({
|
|
2939
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "NO_TARGET"),
|
|
2940
|
+
domain: ErrorDomain.STORAGE,
|
|
2941
|
+
category: ErrorCategory.USER,
|
|
2942
|
+
text: "Either id or filter must be provided",
|
|
2943
|
+
details: { indexName }
|
|
2944
|
+
});
|
|
2945
|
+
}
|
|
2946
|
+
if ("filter" in params && params.filter && Object.keys(params.filter).length === 0) {
|
|
2947
|
+
throw new MastraError({
|
|
2948
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "EMPTY_FILTER"),
|
|
2949
|
+
domain: ErrorDomain.STORAGE,
|
|
2950
|
+
category: ErrorCategory.USER,
|
|
2951
|
+
text: "Cannot update with empty filter",
|
|
2952
|
+
details: { indexName }
|
|
2953
|
+
});
|
|
2954
|
+
}
|
|
2955
|
+
if (!update.vector && !update.metadata) {
|
|
2956
|
+
throw new MastraError({
|
|
2957
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "NO_PAYLOAD"),
|
|
2958
|
+
domain: ErrorDomain.STORAGE,
|
|
2959
|
+
category: ErrorCategory.USER,
|
|
2960
|
+
text: "No updates provided",
|
|
2961
|
+
details: { indexName }
|
|
2962
|
+
});
|
|
2963
|
+
}
|
|
2847
2964
|
try {
|
|
2848
2965
|
if (!this.lanceClient) {
|
|
2849
2966
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
@@ -2851,21 +2968,6 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2851
2968
|
if (!indexName) {
|
|
2852
2969
|
throw new Error("indexName is required");
|
|
2853
2970
|
}
|
|
2854
|
-
if (!id) {
|
|
2855
|
-
throw new Error("id is required");
|
|
2856
|
-
}
|
|
2857
|
-
} catch (err) {
|
|
2858
|
-
throw new MastraError(
|
|
2859
|
-
{
|
|
2860
|
-
id: "STORAGE_LANCE_VECTOR_UPDATE_VECTOR_FAILED_INVALID_ARGS",
|
|
2861
|
-
domain: ErrorDomain.STORAGE,
|
|
2862
|
-
category: ErrorCategory.USER,
|
|
2863
|
-
details: { indexName, id }
|
|
2864
|
-
},
|
|
2865
|
-
err
|
|
2866
|
-
);
|
|
2867
|
-
}
|
|
2868
|
-
try {
|
|
2869
2971
|
const tables = await this.lanceClient.tableNames();
|
|
2870
2972
|
for (const tableName of tables) {
|
|
2871
2973
|
this.logger.debug("Checking table:" + tableName);
|
|
@@ -2875,39 +2977,66 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2875
2977
|
const hasColumn = schema.fields.some((field) => field.name === indexName);
|
|
2876
2978
|
if (hasColumn) {
|
|
2877
2979
|
this.logger.debug(`Found column ${indexName} in table ${tableName}`);
|
|
2878
|
-
|
|
2879
|
-
if (
|
|
2880
|
-
|
|
2980
|
+
let whereClause;
|
|
2981
|
+
if ("id" in params && params.id) {
|
|
2982
|
+
whereClause = `id = '${params.id}'`;
|
|
2983
|
+
} else if ("filter" in params && params.filter) {
|
|
2984
|
+
const translator = new LanceFilterTranslator();
|
|
2985
|
+
const processFilterKeys = (filter) => {
|
|
2986
|
+
const processedFilter = {};
|
|
2987
|
+
Object.entries(filter).forEach(([key, value]) => {
|
|
2988
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
2989
|
+
Object.entries(value).forEach(([nestedKey, nestedValue]) => {
|
|
2990
|
+
processedFilter[`metadata_${key}_${nestedKey}`] = nestedValue;
|
|
2991
|
+
});
|
|
2992
|
+
} else {
|
|
2993
|
+
processedFilter[`metadata_${key}`] = value;
|
|
2994
|
+
}
|
|
2995
|
+
});
|
|
2996
|
+
return processedFilter;
|
|
2997
|
+
};
|
|
2998
|
+
const prefixedFilter = processFilterKeys(params.filter);
|
|
2999
|
+
whereClause = translator.translate(prefixedFilter) || "";
|
|
3000
|
+
if (!whereClause) {
|
|
3001
|
+
throw new Error("Failed to translate filter to SQL");
|
|
3002
|
+
}
|
|
3003
|
+
} else {
|
|
3004
|
+
throw new Error("Either id or filter must be provided");
|
|
2881
3005
|
}
|
|
2882
|
-
const
|
|
2883
|
-
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
|
|
3006
|
+
const existingRecords = await table.query().where(whereClause).select(schema.fields.map((field) => field.name)).toArray();
|
|
3007
|
+
if (existingRecords.length === 0) {
|
|
3008
|
+
this.logger.info(`No records found matching criteria in table ${tableName}`);
|
|
3009
|
+
return;
|
|
3010
|
+
}
|
|
3011
|
+
const updatedRecords = existingRecords.map((record) => {
|
|
3012
|
+
const rowData = {};
|
|
3013
|
+
Object.entries(record).forEach(([key, value]) => {
|
|
3014
|
+
if (key !== "_distance") {
|
|
3015
|
+
if (key === indexName) {
|
|
3016
|
+
if (update.vector) {
|
|
3017
|
+
rowData[key] = update.vector;
|
|
2893
3018
|
} else {
|
|
2894
|
-
|
|
3019
|
+
if (Array.isArray(value)) {
|
|
3020
|
+
rowData[key] = [...value];
|
|
3021
|
+
} else if (typeof value === "object" && value !== null) {
|
|
3022
|
+
rowData[key] = Array.from(value);
|
|
3023
|
+
} else {
|
|
3024
|
+
rowData[key] = value;
|
|
3025
|
+
}
|
|
2895
3026
|
}
|
|
3027
|
+
} else {
|
|
3028
|
+
rowData[key] = value;
|
|
2896
3029
|
}
|
|
2897
|
-
} else {
|
|
2898
|
-
rowData[key] = value;
|
|
2899
3030
|
}
|
|
3031
|
+
});
|
|
3032
|
+
if (update.metadata) {
|
|
3033
|
+
Object.entries(update.metadata).forEach(([key, value]) => {
|
|
3034
|
+
rowData[`metadata_${key}`] = value;
|
|
3035
|
+
});
|
|
2900
3036
|
}
|
|
3037
|
+
return rowData;
|
|
2901
3038
|
});
|
|
2902
|
-
|
|
2903
|
-
rowData[indexName] = update.vector;
|
|
2904
|
-
}
|
|
2905
|
-
if (update.metadata) {
|
|
2906
|
-
Object.entries(update.metadata).forEach(([key, value]) => {
|
|
2907
|
-
rowData[`metadata_${key}`] = value;
|
|
2908
|
-
});
|
|
2909
|
-
}
|
|
2910
|
-
await table.add([rowData], { mode: "overwrite" });
|
|
3039
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(updatedRecords);
|
|
2911
3040
|
return;
|
|
2912
3041
|
}
|
|
2913
3042
|
} catch (err) {
|
|
@@ -2917,12 +3046,19 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2917
3046
|
}
|
|
2918
3047
|
throw new Error(`No table found with column/index '${indexName}'`);
|
|
2919
3048
|
} catch (error) {
|
|
3049
|
+
if (error instanceof MastraError) throw error;
|
|
2920
3050
|
throw new MastraError(
|
|
2921
3051
|
{
|
|
2922
|
-
id: "
|
|
3052
|
+
id: createVectorErrorId("LANCE", "UPDATE_VECTOR", "FAILED"),
|
|
2923
3053
|
domain: ErrorDomain.STORAGE,
|
|
2924
3054
|
category: ErrorCategory.THIRD_PARTY,
|
|
2925
|
-
details: {
|
|
3055
|
+
details: {
|
|
3056
|
+
indexName,
|
|
3057
|
+
..."id" in params && params.id && { id: params.id },
|
|
3058
|
+
..."filter" in params && params.filter && { filter: JSON.stringify(params.filter) },
|
|
3059
|
+
hasVector: !!update.vector,
|
|
3060
|
+
hasMetadata: !!update.metadata
|
|
3061
|
+
}
|
|
2926
3062
|
},
|
|
2927
3063
|
error
|
|
2928
3064
|
);
|
|
@@ -2942,10 +3078,13 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2942
3078
|
} catch (err) {
|
|
2943
3079
|
throw new MastraError(
|
|
2944
3080
|
{
|
|
2945
|
-
id: "
|
|
3081
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTOR", "INVALID_ARGS"),
|
|
2946
3082
|
domain: ErrorDomain.STORAGE,
|
|
2947
3083
|
category: ErrorCategory.USER,
|
|
2948
|
-
details: {
|
|
3084
|
+
details: {
|
|
3085
|
+
indexName,
|
|
3086
|
+
...id && { id }
|
|
3087
|
+
}
|
|
2949
3088
|
},
|
|
2950
3089
|
err
|
|
2951
3090
|
);
|
|
@@ -2972,10 +3111,13 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
2972
3111
|
} catch (error) {
|
|
2973
3112
|
throw new MastraError(
|
|
2974
3113
|
{
|
|
2975
|
-
id: "
|
|
3114
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTOR", "FAILED"),
|
|
2976
3115
|
domain: ErrorDomain.STORAGE,
|
|
2977
3116
|
category: ErrorCategory.THIRD_PARTY,
|
|
2978
|
-
details: {
|
|
3117
|
+
details: {
|
|
3118
|
+
indexName,
|
|
3119
|
+
...id && { id }
|
|
3120
|
+
}
|
|
2979
3121
|
},
|
|
2980
3122
|
error
|
|
2981
3123
|
);
|
|
@@ -3006,8 +3148,111 @@ var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
|
3006
3148
|
});
|
|
3007
3149
|
return result;
|
|
3008
3150
|
}
|
|
3151
|
+
async deleteVectors({ indexName, filter, ids }) {
|
|
3152
|
+
if (ids && filter) {
|
|
3153
|
+
throw new MastraError({
|
|
3154
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "MUTUALLY_EXCLUSIVE"),
|
|
3155
|
+
domain: ErrorDomain.STORAGE,
|
|
3156
|
+
category: ErrorCategory.USER,
|
|
3157
|
+
text: "ids and filter are mutually exclusive",
|
|
3158
|
+
details: { indexName }
|
|
3159
|
+
});
|
|
3160
|
+
}
|
|
3161
|
+
if (!ids && !filter) {
|
|
3162
|
+
throw new MastraError({
|
|
3163
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "NO_TARGET"),
|
|
3164
|
+
domain: ErrorDomain.STORAGE,
|
|
3165
|
+
category: ErrorCategory.USER,
|
|
3166
|
+
text: "Either filter or ids must be provided",
|
|
3167
|
+
details: { indexName }
|
|
3168
|
+
});
|
|
3169
|
+
}
|
|
3170
|
+
if (ids && ids.length === 0) {
|
|
3171
|
+
throw new MastraError({
|
|
3172
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "EMPTY_IDS"),
|
|
3173
|
+
domain: ErrorDomain.STORAGE,
|
|
3174
|
+
category: ErrorCategory.USER,
|
|
3175
|
+
text: "Cannot delete with empty ids array",
|
|
3176
|
+
details: { indexName }
|
|
3177
|
+
});
|
|
3178
|
+
}
|
|
3179
|
+
if (filter && Object.keys(filter).length === 0) {
|
|
3180
|
+
throw new MastraError({
|
|
3181
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "EMPTY_FILTER"),
|
|
3182
|
+
domain: ErrorDomain.STORAGE,
|
|
3183
|
+
category: ErrorCategory.USER,
|
|
3184
|
+
text: "Cannot delete with empty filter",
|
|
3185
|
+
details: { indexName }
|
|
3186
|
+
});
|
|
3187
|
+
}
|
|
3188
|
+
try {
|
|
3189
|
+
if (!this.lanceClient) {
|
|
3190
|
+
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
3191
|
+
}
|
|
3192
|
+
if (!indexName) {
|
|
3193
|
+
throw new Error("indexName is required");
|
|
3194
|
+
}
|
|
3195
|
+
const tables = await this.lanceClient.tableNames();
|
|
3196
|
+
for (const tableName of tables) {
|
|
3197
|
+
this.logger.debug("Checking table:" + tableName);
|
|
3198
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
3199
|
+
try {
|
|
3200
|
+
const schema = await table.schema();
|
|
3201
|
+
const hasColumn = schema.fields.some((field) => field.name === indexName);
|
|
3202
|
+
if (hasColumn) {
|
|
3203
|
+
this.logger.debug(`Found column ${indexName} in table ${tableName}`);
|
|
3204
|
+
if (ids) {
|
|
3205
|
+
const idsConditions = ids.map((id) => `id = '${id}'`).join(" OR ");
|
|
3206
|
+
await table.delete(idsConditions);
|
|
3207
|
+
} else if (filter) {
|
|
3208
|
+
const translator = new LanceFilterTranslator();
|
|
3209
|
+
const processFilterKeys = (filter2) => {
|
|
3210
|
+
const processedFilter = {};
|
|
3211
|
+
Object.entries(filter2).forEach(([key, value]) => {
|
|
3212
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
3213
|
+
Object.entries(value).forEach(([nestedKey, nestedValue]) => {
|
|
3214
|
+
processedFilter[`metadata_${key}_${nestedKey}`] = nestedValue;
|
|
3215
|
+
});
|
|
3216
|
+
} else {
|
|
3217
|
+
processedFilter[`metadata_${key}`] = value;
|
|
3218
|
+
}
|
|
3219
|
+
});
|
|
3220
|
+
return processedFilter;
|
|
3221
|
+
};
|
|
3222
|
+
const prefixedFilter = processFilterKeys(filter);
|
|
3223
|
+
const whereClause = translator.translate(prefixedFilter);
|
|
3224
|
+
if (!whereClause) {
|
|
3225
|
+
throw new Error("Failed to translate filter to SQL");
|
|
3226
|
+
}
|
|
3227
|
+
await table.delete(whereClause);
|
|
3228
|
+
}
|
|
3229
|
+
return;
|
|
3230
|
+
}
|
|
3231
|
+
} catch (err) {
|
|
3232
|
+
this.logger.error(`Error checking schema for table ${tableName}:` + err);
|
|
3233
|
+
continue;
|
|
3234
|
+
}
|
|
3235
|
+
}
|
|
3236
|
+
throw new Error(`No table found with column/index '${indexName}'`);
|
|
3237
|
+
} catch (error) {
|
|
3238
|
+
if (error instanceof MastraError) throw error;
|
|
3239
|
+
throw new MastraError(
|
|
3240
|
+
{
|
|
3241
|
+
id: createVectorErrorId("LANCE", "DELETE_VECTORS", "FAILED"),
|
|
3242
|
+
domain: ErrorDomain.STORAGE,
|
|
3243
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
3244
|
+
details: {
|
|
3245
|
+
indexName,
|
|
3246
|
+
...filter && { filter: JSON.stringify(filter) },
|
|
3247
|
+
...ids && { idsCount: ids.length }
|
|
3248
|
+
}
|
|
3249
|
+
},
|
|
3250
|
+
error
|
|
3251
|
+
);
|
|
3252
|
+
}
|
|
3253
|
+
}
|
|
3009
3254
|
};
|
|
3010
3255
|
|
|
3011
|
-
export { LanceStorage, LanceVectorStore };
|
|
3256
|
+
export { LanceStorage, LanceVectorStore, StoreMemoryLance, StoreScoresLance, StoreWorkflowsLance };
|
|
3012
3257
|
//# sourceMappingURL=index.js.map
|
|
3013
3258
|
//# sourceMappingURL=index.js.map
|