@mastra/lance 1.0.0-beta.1 → 1.0.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +498 -0
- package/dist/docs/README.md +33 -0
- package/dist/docs/SKILL.md +34 -0
- package/dist/docs/SOURCE_MAP.json +6 -0
- package/dist/docs/rag/01-vector-databases.md +643 -0
- package/dist/docs/storage/01-reference.md +113 -0
- package/dist/docs/vectors/01-reference.md +149 -0
- package/dist/index.cjs +1594 -1346
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1593 -1348
- package/dist/index.js.map +1 -1
- package/dist/storage/{domains/operations → db}/index.d.ts +21 -2
- package/dist/storage/db/index.d.ts.map +1 -0
- package/dist/storage/db/utils.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +6 -7
- package/dist/storage/domains/memory/index.d.ts.map +1 -1
- package/dist/storage/domains/scores/index.d.ts +20 -23
- package/dist/storage/domains/scores/index.d.ts.map +1 -1
- package/dist/storage/domains/workflows/index.d.ts +17 -14
- package/dist/storage/domains/workflows/index.d.ts.map +1 -1
- package/dist/storage/index.d.ts +96 -178
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/vector/filter.d.ts +5 -5
- package/dist/vector/index.d.ts +18 -7
- package/dist/vector/index.d.ts.map +1 -1
- package/package.json +9 -9
- package/dist/storage/domains/operations/index.d.ts.map +0 -1
- package/dist/storage/domains/utils.d.ts.map +0 -1
- /package/dist/storage/{domains → db}/utils.d.ts +0 -0
package/dist/index.cjs
CHANGED
|
@@ -4,6 +4,7 @@ var lancedb = require('@lancedb/lancedb');
|
|
|
4
4
|
var error = require('@mastra/core/error');
|
|
5
5
|
var storage = require('@mastra/core/storage');
|
|
6
6
|
var agent = require('@mastra/core/agent');
|
|
7
|
+
var base = require('@mastra/core/base');
|
|
7
8
|
var apacheArrow = require('apache-arrow');
|
|
8
9
|
var evals = require('@mastra/core/evals');
|
|
9
10
|
var vector = require('@mastra/core/vector');
|
|
@@ -91,7 +92,7 @@ async function getTableSchema({
|
|
|
91
92
|
} catch (validationError) {
|
|
92
93
|
throw new error.MastraError(
|
|
93
94
|
{
|
|
94
|
-
id: "
|
|
95
|
+
id: storage.createStorageErrorId("LANCE", "GET_TABLE_SCHEMA", "INVALID_ARGS"),
|
|
95
96
|
domain: error.ErrorDomain.STORAGE,
|
|
96
97
|
category: error.ErrorCategory.USER,
|
|
97
98
|
text: validationError.message,
|
|
@@ -114,7 +115,7 @@ async function getTableSchema({
|
|
|
114
115
|
} catch (error$1) {
|
|
115
116
|
throw new error.MastraError(
|
|
116
117
|
{
|
|
117
|
-
id: "
|
|
118
|
+
id: storage.createStorageErrorId("LANCE", "GET_TABLE_SCHEMA", "FAILED"),
|
|
118
119
|
domain: error.ErrorDomain.STORAGE,
|
|
119
120
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
120
121
|
details: { tableName }
|
|
@@ -124,1118 +125,1200 @@ async function getTableSchema({
|
|
|
124
125
|
}
|
|
125
126
|
}
|
|
126
127
|
|
|
127
|
-
// src/storage/
|
|
128
|
-
|
|
128
|
+
// src/storage/db/index.ts
|
|
129
|
+
function resolveLanceConfig(config) {
|
|
130
|
+
return config.client;
|
|
131
|
+
}
|
|
132
|
+
var LanceDB = class extends base.MastraBase {
|
|
129
133
|
client;
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
super();
|
|
134
|
+
constructor({ client }) {
|
|
135
|
+
super({ name: "lance-db" });
|
|
133
136
|
this.client = client;
|
|
134
|
-
this.operations = operations;
|
|
135
137
|
}
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
138
|
+
getDefaultValue(type) {
|
|
139
|
+
switch (type) {
|
|
140
|
+
case "text":
|
|
141
|
+
return "''";
|
|
142
|
+
case "timestamp":
|
|
143
|
+
return "CURRENT_TIMESTAMP";
|
|
144
|
+
case "integer":
|
|
145
|
+
case "bigint":
|
|
146
|
+
return "0";
|
|
147
|
+
case "jsonb":
|
|
148
|
+
return "'{}'";
|
|
149
|
+
case "uuid":
|
|
150
|
+
return "''";
|
|
151
|
+
default:
|
|
152
|
+
return storage.getDefaultValue(type);
|
|
153
|
+
}
|
|
139
154
|
}
|
|
140
|
-
async
|
|
155
|
+
async hasColumn(tableName, columnName) {
|
|
156
|
+
const table = await this.client.openTable(tableName);
|
|
157
|
+
const schema = await table.schema();
|
|
158
|
+
return schema.fields.some((field) => field.name === columnName);
|
|
159
|
+
}
|
|
160
|
+
translateSchema(schema) {
|
|
161
|
+
const fields = Object.entries(schema).map(([name, column]) => {
|
|
162
|
+
let arrowType;
|
|
163
|
+
switch (column.type.toLowerCase()) {
|
|
164
|
+
case "text":
|
|
165
|
+
case "uuid":
|
|
166
|
+
arrowType = new apacheArrow.Utf8();
|
|
167
|
+
break;
|
|
168
|
+
case "int":
|
|
169
|
+
case "integer":
|
|
170
|
+
arrowType = new apacheArrow.Int32();
|
|
171
|
+
break;
|
|
172
|
+
case "bigint":
|
|
173
|
+
arrowType = new apacheArrow.Float64();
|
|
174
|
+
break;
|
|
175
|
+
case "float":
|
|
176
|
+
arrowType = new apacheArrow.Float32();
|
|
177
|
+
break;
|
|
178
|
+
case "jsonb":
|
|
179
|
+
case "json":
|
|
180
|
+
arrowType = new apacheArrow.Utf8();
|
|
181
|
+
break;
|
|
182
|
+
case "binary":
|
|
183
|
+
arrowType = new apacheArrow.Binary();
|
|
184
|
+
break;
|
|
185
|
+
case "timestamp":
|
|
186
|
+
arrowType = new apacheArrow.Float64();
|
|
187
|
+
break;
|
|
188
|
+
default:
|
|
189
|
+
arrowType = new apacheArrow.Utf8();
|
|
190
|
+
}
|
|
191
|
+
return new apacheArrow.Field(name, arrowType, column.nullable ?? true);
|
|
192
|
+
});
|
|
193
|
+
return new apacheArrow.Schema(fields);
|
|
194
|
+
}
|
|
195
|
+
async createTable({
|
|
196
|
+
tableName,
|
|
197
|
+
schema
|
|
198
|
+
}) {
|
|
141
199
|
try {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
200
|
+
if (!this.client) {
|
|
201
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
202
|
+
}
|
|
203
|
+
if (!tableName) {
|
|
204
|
+
throw new Error("tableName is required for createTable.");
|
|
205
|
+
}
|
|
206
|
+
if (!schema) {
|
|
207
|
+
throw new Error("schema is required for createTable.");
|
|
145
208
|
}
|
|
146
|
-
return {
|
|
147
|
-
...thread,
|
|
148
|
-
createdAt: new Date(thread.createdAt),
|
|
149
|
-
updatedAt: new Date(thread.updatedAt)
|
|
150
|
-
};
|
|
151
209
|
} catch (error$1) {
|
|
152
210
|
throw new error.MastraError(
|
|
153
211
|
{
|
|
154
|
-
id: "
|
|
212
|
+
id: storage.createStorageErrorId("LANCE", "CREATE_TABLE", "INVALID_ARGS"),
|
|
155
213
|
domain: error.ErrorDomain.STORAGE,
|
|
156
|
-
category: error.ErrorCategory.
|
|
214
|
+
category: error.ErrorCategory.USER,
|
|
215
|
+
details: { tableName }
|
|
157
216
|
},
|
|
158
217
|
error$1
|
|
159
218
|
);
|
|
160
219
|
}
|
|
161
|
-
}
|
|
162
|
-
/**
|
|
163
|
-
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
164
|
-
* @param thread - The thread to save
|
|
165
|
-
* @returns The saved thread
|
|
166
|
-
*/
|
|
167
|
-
async saveThread({ thread }) {
|
|
168
220
|
try {
|
|
169
|
-
const
|
|
170
|
-
|
|
171
|
-
await table.add([record], { mode: "append" });
|
|
172
|
-
return thread;
|
|
221
|
+
const arrowSchema = this.translateSchema(schema);
|
|
222
|
+
await this.client.createEmptyTable(tableName, arrowSchema);
|
|
173
223
|
} catch (error$1) {
|
|
224
|
+
if (error$1.message?.includes("already exists")) {
|
|
225
|
+
this.logger.debug(`Table '${tableName}' already exists, skipping create`);
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
174
228
|
throw new error.MastraError(
|
|
175
229
|
{
|
|
176
|
-
id: "
|
|
230
|
+
id: storage.createStorageErrorId("LANCE", "CREATE_TABLE", "FAILED"),
|
|
177
231
|
domain: error.ErrorDomain.STORAGE,
|
|
178
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
232
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
233
|
+
details: { tableName }
|
|
179
234
|
},
|
|
180
235
|
error$1
|
|
181
236
|
);
|
|
182
237
|
}
|
|
183
238
|
}
|
|
184
|
-
async
|
|
185
|
-
id,
|
|
186
|
-
title,
|
|
187
|
-
metadata
|
|
188
|
-
}) {
|
|
189
|
-
const maxRetries = 5;
|
|
190
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
191
|
-
try {
|
|
192
|
-
const current = await this.getThreadById({ threadId: id });
|
|
193
|
-
if (!current) {
|
|
194
|
-
throw new Error(`Thread with id ${id} not found`);
|
|
195
|
-
}
|
|
196
|
-
const mergedMetadata = { ...current.metadata, ...metadata };
|
|
197
|
-
const record = {
|
|
198
|
-
id,
|
|
199
|
-
title,
|
|
200
|
-
metadata: JSON.stringify(mergedMetadata),
|
|
201
|
-
updatedAt: (/* @__PURE__ */ new Date()).getTime()
|
|
202
|
-
};
|
|
203
|
-
const table = await this.client.openTable(storage.TABLE_THREADS);
|
|
204
|
-
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
205
|
-
const updatedThread = await this.getThreadById({ threadId: id });
|
|
206
|
-
if (!updatedThread) {
|
|
207
|
-
throw new Error(`Failed to retrieve updated thread ${id}`);
|
|
208
|
-
}
|
|
209
|
-
return updatedThread;
|
|
210
|
-
} catch (error$1) {
|
|
211
|
-
if (error$1.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
212
|
-
const delay = Math.pow(2, attempt) * 10;
|
|
213
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
214
|
-
continue;
|
|
215
|
-
}
|
|
216
|
-
throw new error.MastraError(
|
|
217
|
-
{
|
|
218
|
-
id: "LANCE_STORE_UPDATE_THREAD_FAILED",
|
|
219
|
-
domain: error.ErrorDomain.STORAGE,
|
|
220
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
221
|
-
},
|
|
222
|
-
error$1
|
|
223
|
-
);
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
throw new error.MastraError(
|
|
227
|
-
{
|
|
228
|
-
id: "LANCE_STORE_UPDATE_THREAD_FAILED",
|
|
229
|
-
domain: error.ErrorDomain.STORAGE,
|
|
230
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
231
|
-
},
|
|
232
|
-
new Error("All retries exhausted")
|
|
233
|
-
);
|
|
234
|
-
}
|
|
235
|
-
async deleteThread({ threadId }) {
|
|
239
|
+
async dropTable({ tableName }) {
|
|
236
240
|
try {
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
241
|
+
if (!this.client) {
|
|
242
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
243
|
+
}
|
|
244
|
+
if (!tableName) {
|
|
245
|
+
throw new Error("tableName is required for dropTable.");
|
|
246
|
+
}
|
|
247
|
+
} catch (validationError) {
|
|
242
248
|
throw new error.MastraError(
|
|
243
249
|
{
|
|
244
|
-
id: "
|
|
250
|
+
id: storage.createStorageErrorId("LANCE", "DROP_TABLE", "INVALID_ARGS"),
|
|
245
251
|
domain: error.ErrorDomain.STORAGE,
|
|
246
|
-
category: error.ErrorCategory.
|
|
252
|
+
category: error.ErrorCategory.USER,
|
|
253
|
+
text: validationError.message,
|
|
254
|
+
details: { tableName }
|
|
247
255
|
},
|
|
248
|
-
|
|
256
|
+
validationError
|
|
249
257
|
);
|
|
250
258
|
}
|
|
251
|
-
}
|
|
252
|
-
normalizeMessage(message) {
|
|
253
|
-
const { thread_id, ...rest } = message;
|
|
254
|
-
return {
|
|
255
|
-
...rest,
|
|
256
|
-
threadId: thread_id,
|
|
257
|
-
content: typeof message.content === "string" ? (() => {
|
|
258
|
-
try {
|
|
259
|
-
return JSON.parse(message.content);
|
|
260
|
-
} catch {
|
|
261
|
-
return message.content;
|
|
262
|
-
}
|
|
263
|
-
})() : message.content
|
|
264
|
-
};
|
|
265
|
-
}
|
|
266
|
-
async listMessagesById({ messageIds }) {
|
|
267
|
-
if (messageIds.length === 0) return { messages: [] };
|
|
268
259
|
try {
|
|
269
|
-
|
|
270
|
-
const quotedIds = messageIds.map((id) => `'${id}'`).join(", ");
|
|
271
|
-
const allRecords = await table.query().where(`id IN (${quotedIds})`).toArray();
|
|
272
|
-
const messages = processResultWithTypeConversion(
|
|
273
|
-
allRecords,
|
|
274
|
-
await getTableSchema({ tableName: storage.TABLE_MESSAGES, client: this.client })
|
|
275
|
-
);
|
|
276
|
-
const list = new agent.MessageList().add(
|
|
277
|
-
messages.map(this.normalizeMessage),
|
|
278
|
-
"memory"
|
|
279
|
-
);
|
|
280
|
-
return { messages: list.get.all.db() };
|
|
260
|
+
await this.client.dropTable(tableName);
|
|
281
261
|
} catch (error$1) {
|
|
262
|
+
if (error$1.toString().includes("was not found") || error$1.message?.includes("Table not found")) {
|
|
263
|
+
this.logger.debug(`Table '${tableName}' does not exist, skipping drop`);
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
282
266
|
throw new error.MastraError(
|
|
283
267
|
{
|
|
284
|
-
id: "
|
|
268
|
+
id: storage.createStorageErrorId("LANCE", "DROP_TABLE", "FAILED"),
|
|
285
269
|
domain: error.ErrorDomain.STORAGE,
|
|
286
270
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
287
|
-
details: {
|
|
288
|
-
messageIds: JSON.stringify(messageIds)
|
|
289
|
-
}
|
|
271
|
+
details: { tableName }
|
|
290
272
|
},
|
|
291
273
|
error$1
|
|
292
274
|
);
|
|
293
275
|
}
|
|
294
276
|
}
|
|
295
|
-
async
|
|
296
|
-
|
|
297
|
-
|
|
277
|
+
async alterTable({
|
|
278
|
+
tableName,
|
|
279
|
+
schema,
|
|
280
|
+
ifNotExists
|
|
281
|
+
}) {
|
|
282
|
+
try {
|
|
283
|
+
if (!this.client) {
|
|
284
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
285
|
+
}
|
|
286
|
+
if (!tableName) {
|
|
287
|
+
throw new Error("tableName is required for alterTable.");
|
|
288
|
+
}
|
|
289
|
+
if (!schema) {
|
|
290
|
+
throw new Error("schema is required for alterTable.");
|
|
291
|
+
}
|
|
292
|
+
if (!ifNotExists || ifNotExists.length === 0) {
|
|
293
|
+
this.logger.debug("No columns specified to add in alterTable, skipping.");
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
} catch (validationError) {
|
|
298
297
|
throw new error.MastraError(
|
|
299
298
|
{
|
|
300
|
-
id: "
|
|
299
|
+
id: storage.createStorageErrorId("LANCE", "ALTER_TABLE", "INVALID_ARGS"),
|
|
301
300
|
domain: error.ErrorDomain.STORAGE,
|
|
302
|
-
category: error.ErrorCategory.
|
|
303
|
-
|
|
301
|
+
category: error.ErrorCategory.USER,
|
|
302
|
+
text: validationError.message,
|
|
303
|
+
details: { tableName }
|
|
304
304
|
},
|
|
305
|
-
|
|
305
|
+
validationError
|
|
306
306
|
);
|
|
307
307
|
}
|
|
308
|
-
const perPage = storage.normalizePerPage(perPageInput, 40);
|
|
309
|
-
const { offset, perPage: perPageForResponse } = storage.calculatePagination(page, perPageInput, perPage);
|
|
310
308
|
try {
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
}
|
|
322
|
-
const
|
|
323
|
-
|
|
324
|
-
const conditions = [`thread_id = '${this.escapeSql(threadId)}'`];
|
|
325
|
-
if (resourceId) {
|
|
326
|
-
conditions.push(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
327
|
-
}
|
|
328
|
-
if (filter?.dateRange?.start) {
|
|
329
|
-
const startTime = filter.dateRange.start instanceof Date ? filter.dateRange.start.getTime() : new Date(filter.dateRange.start).getTime();
|
|
330
|
-
conditions.push(`\`createdAt\` >= ${startTime}`);
|
|
331
|
-
}
|
|
332
|
-
if (filter?.dateRange?.end) {
|
|
333
|
-
const endTime = filter.dateRange.end instanceof Date ? filter.dateRange.end.getTime() : new Date(filter.dateRange.end).getTime();
|
|
334
|
-
conditions.push(`\`createdAt\` <= ${endTime}`);
|
|
335
|
-
}
|
|
336
|
-
const whereClause = conditions.join(" AND ");
|
|
337
|
-
const total = await table.countRows(whereClause);
|
|
338
|
-
const query = table.query().where(whereClause);
|
|
339
|
-
let allRecords = await query.toArray();
|
|
340
|
-
allRecords.sort((a, b) => {
|
|
341
|
-
const aValue = field === "createdAt" ? a.createdAt : a[field];
|
|
342
|
-
const bValue = field === "createdAt" ? b.createdAt : b[field];
|
|
343
|
-
if (aValue == null && bValue == null) return 0;
|
|
344
|
-
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
345
|
-
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
346
|
-
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
347
|
-
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
348
|
-
}
|
|
349
|
-
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
350
|
-
});
|
|
351
|
-
const paginatedRecords = allRecords.slice(offset, offset + perPage);
|
|
352
|
-
const messages = paginatedRecords.map((row) => this.normalizeMessage(row));
|
|
353
|
-
if (total === 0 && messages.length === 0 && (!include || include.length === 0)) {
|
|
309
|
+
const table = await this.client.openTable(tableName);
|
|
310
|
+
const currentSchema = await table.schema();
|
|
311
|
+
const existingFields = new Set(currentSchema.fields.map((f) => f.name));
|
|
312
|
+
const typeMap = {
|
|
313
|
+
text: "string",
|
|
314
|
+
integer: "int",
|
|
315
|
+
bigint: "bigint",
|
|
316
|
+
timestamp: "timestamp",
|
|
317
|
+
jsonb: "string",
|
|
318
|
+
uuid: "string"
|
|
319
|
+
};
|
|
320
|
+
const columnsToAdd = ifNotExists.filter((col) => schema[col] && !existingFields.has(col)).map((col) => {
|
|
321
|
+
const colDef = schema[col];
|
|
354
322
|
return {
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
page,
|
|
358
|
-
perPage: perPageForResponse,
|
|
359
|
-
hasMore: false
|
|
323
|
+
name: col,
|
|
324
|
+
valueSql: colDef?.nullable ? `cast(NULL as ${typeMap[colDef.type ?? "text"]})` : `cast(${this.getDefaultValue(colDef?.type ?? "text")} as ${typeMap[colDef?.type ?? "text"]})`
|
|
360
325
|
};
|
|
361
|
-
}
|
|
362
|
-
const messageIds = new Set(messages.map((m) => m.id));
|
|
363
|
-
if (include && include.length > 0) {
|
|
364
|
-
const threadIds = [...new Set(include.map((item) => item.threadId || threadId))];
|
|
365
|
-
const allThreadMessages = [];
|
|
366
|
-
for (const tid of threadIds) {
|
|
367
|
-
const threadQuery = table.query().where(`thread_id = '${tid}'`);
|
|
368
|
-
let threadRecords = await threadQuery.toArray();
|
|
369
|
-
allThreadMessages.push(...threadRecords);
|
|
370
|
-
}
|
|
371
|
-
allThreadMessages.sort((a, b) => a.createdAt - b.createdAt);
|
|
372
|
-
const contextMessages = this.processMessagesWithContext(allThreadMessages, include);
|
|
373
|
-
const includedMessages = contextMessages.map((row) => this.normalizeMessage(row));
|
|
374
|
-
for (const includeMsg of includedMessages) {
|
|
375
|
-
if (!messageIds.has(includeMsg.id)) {
|
|
376
|
-
messages.push(includeMsg);
|
|
377
|
-
messageIds.add(includeMsg.id);
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
const list = new agent.MessageList().add(messages, "memory");
|
|
382
|
-
let finalMessages = list.get.all.db();
|
|
383
|
-
finalMessages = finalMessages.sort((a, b) => {
|
|
384
|
-
const aValue = field === "createdAt" ? new Date(a.createdAt).getTime() : a[field];
|
|
385
|
-
const bValue = field === "createdAt" ? new Date(b.createdAt).getTime() : b[field];
|
|
386
|
-
if (aValue == null && bValue == null) return 0;
|
|
387
|
-
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
388
|
-
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
389
|
-
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
390
|
-
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
391
|
-
}
|
|
392
|
-
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
393
326
|
});
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
return {
|
|
399
|
-
messages: finalMessages,
|
|
400
|
-
total,
|
|
401
|
-
page,
|
|
402
|
-
perPage: perPageForResponse,
|
|
403
|
-
hasMore
|
|
404
|
-
};
|
|
327
|
+
if (columnsToAdd.length > 0) {
|
|
328
|
+
await table.addColumns(columnsToAdd);
|
|
329
|
+
this.logger?.info?.(`Added columns [${columnsToAdd.map((c) => c.name).join(", ")}] to table ${tableName}`);
|
|
330
|
+
}
|
|
405
331
|
} catch (error$1) {
|
|
406
|
-
|
|
332
|
+
throw new error.MastraError(
|
|
407
333
|
{
|
|
408
|
-
id: "
|
|
334
|
+
id: storage.createStorageErrorId("LANCE", "ALTER_TABLE", "FAILED"),
|
|
409
335
|
domain: error.ErrorDomain.STORAGE,
|
|
410
336
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
411
|
-
details: {
|
|
412
|
-
threadId,
|
|
413
|
-
resourceId: resourceId ?? ""
|
|
414
|
-
}
|
|
337
|
+
details: { tableName }
|
|
415
338
|
},
|
|
416
339
|
error$1
|
|
417
340
|
);
|
|
418
|
-
this.logger?.error?.(mastraError.toString());
|
|
419
|
-
this.logger?.trackException?.(mastraError);
|
|
420
|
-
return {
|
|
421
|
-
messages: [],
|
|
422
|
-
total: 0,
|
|
423
|
-
page,
|
|
424
|
-
perPage: perPageForResponse,
|
|
425
|
-
hasMore: false
|
|
426
|
-
};
|
|
427
341
|
}
|
|
428
342
|
}
|
|
429
|
-
async
|
|
343
|
+
async clearTable({ tableName }) {
|
|
430
344
|
try {
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
return { messages: [] };
|
|
434
|
-
}
|
|
435
|
-
const threadId = messages[0]?.threadId;
|
|
436
|
-
if (!threadId) {
|
|
437
|
-
throw new Error("Thread ID is required");
|
|
345
|
+
if (!this.client) {
|
|
346
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
438
347
|
}
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
throw new Error("Message ID is required");
|
|
442
|
-
}
|
|
443
|
-
if (!message.threadId) {
|
|
444
|
-
throw new Error("Thread ID is required for all messages");
|
|
445
|
-
}
|
|
446
|
-
if (message.resourceId === null || message.resourceId === void 0) {
|
|
447
|
-
throw new Error("Resource ID cannot be null or undefined");
|
|
448
|
-
}
|
|
449
|
-
if (!message.content) {
|
|
450
|
-
throw new Error("Message content is required");
|
|
451
|
-
}
|
|
348
|
+
if (!tableName) {
|
|
349
|
+
throw new Error("tableName is required for clearTable.");
|
|
452
350
|
}
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
const
|
|
467
|
-
await
|
|
468
|
-
const list = new agent.MessageList().add(messages, "memory");
|
|
469
|
-
return { messages: list.get.all.db() };
|
|
351
|
+
} catch (validationError) {
|
|
352
|
+
throw new error.MastraError(
|
|
353
|
+
{
|
|
354
|
+
id: storage.createStorageErrorId("LANCE", "CLEAR_TABLE", "INVALID_ARGS"),
|
|
355
|
+
domain: error.ErrorDomain.STORAGE,
|
|
356
|
+
category: error.ErrorCategory.USER,
|
|
357
|
+
text: validationError.message,
|
|
358
|
+
details: { tableName }
|
|
359
|
+
},
|
|
360
|
+
validationError
|
|
361
|
+
);
|
|
362
|
+
}
|
|
363
|
+
try {
|
|
364
|
+
const table = await this.client.openTable(tableName);
|
|
365
|
+
await table.delete("1=1");
|
|
470
366
|
} catch (error$1) {
|
|
471
367
|
throw new error.MastraError(
|
|
472
368
|
{
|
|
473
|
-
id: "
|
|
369
|
+
id: storage.createStorageErrorId("LANCE", "CLEAR_TABLE", "FAILED"),
|
|
474
370
|
domain: error.ErrorDomain.STORAGE,
|
|
475
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
371
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
372
|
+
details: { tableName }
|
|
476
373
|
},
|
|
477
374
|
error$1
|
|
478
375
|
);
|
|
479
376
|
}
|
|
480
377
|
}
|
|
481
|
-
async
|
|
378
|
+
async insert({ tableName, record }) {
|
|
482
379
|
try {
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
if (page < 0) {
|
|
486
|
-
throw new error.MastraError(
|
|
487
|
-
{
|
|
488
|
-
id: "STORAGE_LANCE_LIST_THREADS_BY_RESOURCE_ID_INVALID_PAGE",
|
|
489
|
-
domain: error.ErrorDomain.STORAGE,
|
|
490
|
-
category: error.ErrorCategory.USER,
|
|
491
|
-
details: { page }
|
|
492
|
-
},
|
|
493
|
-
new Error("page must be >= 0")
|
|
494
|
-
);
|
|
380
|
+
if (!this.client) {
|
|
381
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
495
382
|
}
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
});
|
|
513
|
-
const paginatedRecords = records.slice(offset, offset + perPage);
|
|
514
|
-
const schema = await getTableSchema({ tableName: storage.TABLE_THREADS, client: this.client });
|
|
515
|
-
const threads = paginatedRecords.map(
|
|
516
|
-
(record) => processResultWithTypeConversion(record, schema)
|
|
383
|
+
if (!tableName) {
|
|
384
|
+
throw new Error("tableName is required for insert.");
|
|
385
|
+
}
|
|
386
|
+
if (!record || Object.keys(record).length === 0) {
|
|
387
|
+
throw new Error("record is required and cannot be empty for insert.");
|
|
388
|
+
}
|
|
389
|
+
} catch (validationError) {
|
|
390
|
+
throw new error.MastraError(
|
|
391
|
+
{
|
|
392
|
+
id: storage.createStorageErrorId("LANCE", "INSERT", "INVALID_ARGS"),
|
|
393
|
+
domain: error.ErrorDomain.STORAGE,
|
|
394
|
+
category: error.ErrorCategory.USER,
|
|
395
|
+
text: validationError.message,
|
|
396
|
+
details: { tableName }
|
|
397
|
+
},
|
|
398
|
+
validationError
|
|
517
399
|
);
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
400
|
+
}
|
|
401
|
+
try {
|
|
402
|
+
const table = await this.client.openTable(tableName);
|
|
403
|
+
const primaryId = getPrimaryKeys(tableName);
|
|
404
|
+
const processedRecord = { ...record };
|
|
405
|
+
for (const key in processedRecord) {
|
|
406
|
+
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
407
|
+
this.logger.debug("Converting object to JSON string: ", processedRecord[key]);
|
|
408
|
+
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([processedRecord]);
|
|
525
412
|
} catch (error$1) {
|
|
526
413
|
throw new error.MastraError(
|
|
527
414
|
{
|
|
528
|
-
id: "
|
|
415
|
+
id: storage.createStorageErrorId("LANCE", "INSERT", "FAILED"),
|
|
529
416
|
domain: error.ErrorDomain.STORAGE,
|
|
530
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
417
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
418
|
+
details: { tableName }
|
|
531
419
|
},
|
|
532
420
|
error$1
|
|
533
421
|
);
|
|
534
422
|
}
|
|
535
423
|
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
* @returns The processed array with context messages included
|
|
541
|
-
*/
|
|
542
|
-
processMessagesWithContext(records, include) {
|
|
543
|
-
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
544
|
-
if (messagesWithContext.length === 0) {
|
|
545
|
-
return records;
|
|
546
|
-
}
|
|
547
|
-
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
548
|
-
records.forEach((message, index) => {
|
|
549
|
-
messageIndexMap.set(message.id, index);
|
|
550
|
-
});
|
|
551
|
-
const additionalIndices = /* @__PURE__ */ new Set();
|
|
552
|
-
for (const item of messagesWithContext) {
|
|
553
|
-
const messageIndex = messageIndexMap.get(item.id);
|
|
554
|
-
if (messageIndex !== void 0) {
|
|
555
|
-
if (item.withPreviousMessages) {
|
|
556
|
-
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
557
|
-
for (let i = startIdx; i < messageIndex; i++) {
|
|
558
|
-
additionalIndices.add(i);
|
|
559
|
-
}
|
|
560
|
-
}
|
|
561
|
-
if (item.withNextMessages) {
|
|
562
|
-
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
563
|
-
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
564
|
-
additionalIndices.add(i);
|
|
565
|
-
}
|
|
566
|
-
}
|
|
424
|
+
async batchInsert({ tableName, records }) {
|
|
425
|
+
try {
|
|
426
|
+
if (!this.client) {
|
|
427
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
567
428
|
}
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
return records;
|
|
571
|
-
}
|
|
572
|
-
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
573
|
-
const allIndices = /* @__PURE__ */ new Set();
|
|
574
|
-
records.forEach((record, index) => {
|
|
575
|
-
if (originalMatchIds.has(record.id)) {
|
|
576
|
-
allIndices.add(index);
|
|
429
|
+
if (!tableName) {
|
|
430
|
+
throw new Error("tableName is required for batchInsert.");
|
|
577
431
|
}
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
})
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
content: typeof data.content === "string" ? (() => {
|
|
593
|
-
try {
|
|
594
|
-
return JSON.parse(data.content);
|
|
595
|
-
} catch {
|
|
596
|
-
return data.content;
|
|
597
|
-
}
|
|
598
|
-
})() : data.content,
|
|
599
|
-
createdAt: new Date(data.createdAt),
|
|
600
|
-
updatedAt: new Date(data.updatedAt)
|
|
601
|
-
};
|
|
602
|
-
}
|
|
603
|
-
async updateMessages(args) {
|
|
604
|
-
const { messages } = args;
|
|
605
|
-
this.logger.debug("Updating messages", { count: messages.length });
|
|
606
|
-
if (!messages.length) {
|
|
607
|
-
return [];
|
|
432
|
+
if (!records || records.length === 0) {
|
|
433
|
+
throw new Error("records array is required and cannot be empty for batchInsert.");
|
|
434
|
+
}
|
|
435
|
+
} catch (validationError) {
|
|
436
|
+
throw new error.MastraError(
|
|
437
|
+
{
|
|
438
|
+
id: storage.createStorageErrorId("LANCE", "BATCH_INSERT", "INVALID_ARGS"),
|
|
439
|
+
domain: error.ErrorDomain.STORAGE,
|
|
440
|
+
category: error.ErrorCategory.USER,
|
|
441
|
+
text: validationError.message,
|
|
442
|
+
details: { tableName }
|
|
443
|
+
},
|
|
444
|
+
validationError
|
|
445
|
+
);
|
|
608
446
|
}
|
|
609
|
-
const updatedMessages = [];
|
|
610
|
-
const affectedThreadIds = /* @__PURE__ */ new Set();
|
|
611
447
|
try {
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
continue;
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
const originalThreadId = existingMsg.threadId;
|
|
621
|
-
affectedThreadIds.add(originalThreadId);
|
|
622
|
-
const updatePayload = {};
|
|
623
|
-
if ("role" in updates && updates.role !== void 0) updatePayload.role = updates.role;
|
|
624
|
-
if ("type" in updates && updates.type !== void 0) updatePayload.type = updates.type;
|
|
625
|
-
if ("resourceId" in updates && updates.resourceId !== void 0) updatePayload.resourceId = updates.resourceId;
|
|
626
|
-
if ("threadId" in updates && updates.threadId !== void 0 && updates.threadId !== null) {
|
|
627
|
-
updatePayload.thread_id = updates.threadId;
|
|
628
|
-
affectedThreadIds.add(updates.threadId);
|
|
629
|
-
}
|
|
630
|
-
if (updates.content) {
|
|
631
|
-
const existingContent = existingMsg.content;
|
|
632
|
-
let newContent = { ...existingContent };
|
|
633
|
-
if (updates.content.metadata !== void 0) {
|
|
634
|
-
newContent.metadata = {
|
|
635
|
-
...existingContent.metadata || {},
|
|
636
|
-
...updates.content.metadata || {}
|
|
637
|
-
};
|
|
638
|
-
}
|
|
639
|
-
if (updates.content.content !== void 0) {
|
|
640
|
-
newContent.content = updates.content.content;
|
|
641
|
-
}
|
|
642
|
-
if ("parts" in updates.content && updates.content.parts !== void 0) {
|
|
643
|
-
newContent.parts = updates.content.parts;
|
|
448
|
+
const table = await this.client.openTable(tableName);
|
|
449
|
+
const primaryId = getPrimaryKeys(tableName);
|
|
450
|
+
const processedRecords = records.map((record) => {
|
|
451
|
+
const processedRecord = { ...record };
|
|
452
|
+
for (const key in processedRecord) {
|
|
453
|
+
if (processedRecord[key] == null) continue;
|
|
454
|
+
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
455
|
+
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
644
456
|
}
|
|
645
|
-
updatePayload.content = JSON.stringify(newContent);
|
|
646
|
-
}
|
|
647
|
-
await this.operations.insert({ tableName: storage.TABLE_MESSAGES, record: { id, ...updatePayload } });
|
|
648
|
-
const updatedMessage = await this.operations.load({ tableName: storage.TABLE_MESSAGES, keys: { id } });
|
|
649
|
-
if (updatedMessage) {
|
|
650
|
-
updatedMessages.push(this.parseMessageData(updatedMessage));
|
|
651
457
|
}
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
tableName: storage.TABLE_THREADS,
|
|
656
|
-
record: { id: threadId, updatedAt: Date.now() }
|
|
657
|
-
});
|
|
658
|
-
}
|
|
659
|
-
return updatedMessages;
|
|
458
|
+
return processedRecord;
|
|
459
|
+
});
|
|
460
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(processedRecords);
|
|
660
461
|
} catch (error$1) {
|
|
661
462
|
throw new error.MastraError(
|
|
662
463
|
{
|
|
663
|
-
id: "
|
|
464
|
+
id: storage.createStorageErrorId("LANCE", "BATCH_INSERT", "FAILED"),
|
|
664
465
|
domain: error.ErrorDomain.STORAGE,
|
|
665
466
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
666
|
-
details: {
|
|
467
|
+
details: { tableName }
|
|
667
468
|
},
|
|
668
469
|
error$1
|
|
669
470
|
);
|
|
670
471
|
}
|
|
671
472
|
}
|
|
672
|
-
async
|
|
473
|
+
async load({ tableName, keys }) {
|
|
673
474
|
try {
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
return null;
|
|
677
|
-
}
|
|
678
|
-
let createdAt;
|
|
679
|
-
let updatedAt;
|
|
680
|
-
try {
|
|
681
|
-
if (resource.createdAt instanceof Date) {
|
|
682
|
-
createdAt = resource.createdAt;
|
|
683
|
-
} else if (typeof resource.createdAt === "string") {
|
|
684
|
-
createdAt = new Date(resource.createdAt);
|
|
685
|
-
} else if (typeof resource.createdAt === "number") {
|
|
686
|
-
createdAt = new Date(resource.createdAt);
|
|
687
|
-
} else {
|
|
688
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
689
|
-
}
|
|
690
|
-
if (isNaN(createdAt.getTime())) {
|
|
691
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
692
|
-
}
|
|
693
|
-
} catch {
|
|
694
|
-
createdAt = /* @__PURE__ */ new Date();
|
|
695
|
-
}
|
|
696
|
-
try {
|
|
697
|
-
if (resource.updatedAt instanceof Date) {
|
|
698
|
-
updatedAt = resource.updatedAt;
|
|
699
|
-
} else if (typeof resource.updatedAt === "string") {
|
|
700
|
-
updatedAt = new Date(resource.updatedAt);
|
|
701
|
-
} else if (typeof resource.updatedAt === "number") {
|
|
702
|
-
updatedAt = new Date(resource.updatedAt);
|
|
703
|
-
} else {
|
|
704
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
705
|
-
}
|
|
706
|
-
if (isNaN(updatedAt.getTime())) {
|
|
707
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
708
|
-
}
|
|
709
|
-
} catch {
|
|
710
|
-
updatedAt = /* @__PURE__ */ new Date();
|
|
475
|
+
if (!this.client) {
|
|
476
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
711
477
|
}
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
workingMemory = void 0;
|
|
715
|
-
} else if (workingMemory === "") {
|
|
716
|
-
workingMemory = "";
|
|
717
|
-
} else if (typeof workingMemory === "object") {
|
|
718
|
-
workingMemory = JSON.stringify(workingMemory);
|
|
478
|
+
if (!tableName) {
|
|
479
|
+
throw new Error("tableName is required for load.");
|
|
719
480
|
}
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
metadata = void 0;
|
|
723
|
-
} else if (typeof metadata === "string") {
|
|
724
|
-
try {
|
|
725
|
-
metadata = JSON.parse(metadata);
|
|
726
|
-
} catch {
|
|
727
|
-
metadata = metadata;
|
|
728
|
-
}
|
|
481
|
+
if (!keys || Object.keys(keys).length === 0) {
|
|
482
|
+
throw new Error("keys are required and cannot be empty for load.");
|
|
729
483
|
}
|
|
730
|
-
|
|
731
|
-
...resource,
|
|
732
|
-
createdAt,
|
|
733
|
-
updatedAt,
|
|
734
|
-
workingMemory,
|
|
735
|
-
metadata
|
|
736
|
-
};
|
|
737
|
-
} catch (error$1) {
|
|
484
|
+
} catch (validationError) {
|
|
738
485
|
throw new error.MastraError(
|
|
739
486
|
{
|
|
740
|
-
id: "
|
|
487
|
+
id: storage.createStorageErrorId("LANCE", "LOAD", "INVALID_ARGS"),
|
|
741
488
|
domain: error.ErrorDomain.STORAGE,
|
|
742
|
-
category: error.ErrorCategory.
|
|
489
|
+
category: error.ErrorCategory.USER,
|
|
490
|
+
text: validationError.message,
|
|
491
|
+
details: { tableName }
|
|
743
492
|
},
|
|
744
|
-
|
|
493
|
+
validationError
|
|
745
494
|
);
|
|
746
495
|
}
|
|
747
|
-
}
|
|
748
|
-
async saveResource({ resource }) {
|
|
749
496
|
try {
|
|
750
|
-
const
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
497
|
+
const table = await this.client.openTable(tableName);
|
|
498
|
+
const tableSchema = await getTableSchema({ tableName, client: this.client });
|
|
499
|
+
const query = table.query();
|
|
500
|
+
if (Object.keys(keys).length > 0) {
|
|
501
|
+
validateKeyTypes(keys, tableSchema);
|
|
502
|
+
const filterConditions = Object.entries(keys).map(([key, value]) => {
|
|
503
|
+
const isCamelCase = /^[a-z][a-zA-Z]*$/.test(key) && /[A-Z]/.test(key);
|
|
504
|
+
const quotedKey = isCamelCase ? `\`${key}\`` : key;
|
|
505
|
+
if (typeof value === "string") {
|
|
506
|
+
return `${quotedKey} = '${value}'`;
|
|
507
|
+
} else if (value === null) {
|
|
508
|
+
return `${quotedKey} IS NULL`;
|
|
509
|
+
} else {
|
|
510
|
+
return `${quotedKey} = ${value}`;
|
|
511
|
+
}
|
|
512
|
+
}).join(" AND ");
|
|
513
|
+
this.logger.debug("where clause generated: " + filterConditions);
|
|
514
|
+
query.where(filterConditions);
|
|
515
|
+
}
|
|
516
|
+
const result = await query.limit(1).toArray();
|
|
517
|
+
if (result.length === 0) {
|
|
518
|
+
this.logger.debug("No record found");
|
|
519
|
+
return null;
|
|
520
|
+
}
|
|
521
|
+
return processResultWithTypeConversion(result[0], tableSchema);
|
|
761
522
|
} catch (error$1) {
|
|
523
|
+
if (error$1 instanceof error.MastraError) throw error$1;
|
|
762
524
|
throw new error.MastraError(
|
|
763
525
|
{
|
|
764
|
-
id: "
|
|
526
|
+
id: storage.createStorageErrorId("LANCE", "LOAD", "FAILED"),
|
|
765
527
|
domain: error.ErrorDomain.STORAGE,
|
|
766
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
528
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
529
|
+
details: { tableName, keyCount: Object.keys(keys).length, firstKey: Object.keys(keys)[0] ?? "" }
|
|
767
530
|
},
|
|
768
531
|
error$1
|
|
769
532
|
);
|
|
770
533
|
}
|
|
771
534
|
}
|
|
772
|
-
async updateResource({
|
|
773
|
-
resourceId,
|
|
774
|
-
workingMemory,
|
|
775
|
-
metadata
|
|
776
|
-
}) {
|
|
777
|
-
const maxRetries = 3;
|
|
778
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
779
|
-
try {
|
|
780
|
-
const existingResource = await this.getResourceById({ resourceId });
|
|
781
|
-
if (!existingResource) {
|
|
782
|
-
const newResource = {
|
|
783
|
-
id: resourceId,
|
|
784
|
-
workingMemory,
|
|
785
|
-
metadata: metadata || {},
|
|
786
|
-
createdAt: /* @__PURE__ */ new Date(),
|
|
787
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
788
|
-
};
|
|
789
|
-
return this.saveResource({ resource: newResource });
|
|
790
|
-
}
|
|
791
|
-
const updatedResource = {
|
|
792
|
-
...existingResource,
|
|
793
|
-
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
794
|
-
metadata: {
|
|
795
|
-
...existingResource.metadata,
|
|
796
|
-
...metadata
|
|
797
|
-
},
|
|
798
|
-
updatedAt: /* @__PURE__ */ new Date()
|
|
799
|
-
};
|
|
800
|
-
const record = {
|
|
801
|
-
id: resourceId,
|
|
802
|
-
workingMemory: updatedResource.workingMemory || "",
|
|
803
|
-
metadata: updatedResource.metadata ? JSON.stringify(updatedResource.metadata) : "",
|
|
804
|
-
updatedAt: updatedResource.updatedAt.getTime()
|
|
805
|
-
// Store as timestamp (milliseconds)
|
|
806
|
-
};
|
|
807
|
-
const table = await this.client.openTable(storage.TABLE_RESOURCES);
|
|
808
|
-
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
809
|
-
return updatedResource;
|
|
810
|
-
} catch (error$1) {
|
|
811
|
-
if (error$1.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
812
|
-
const delay = Math.pow(2, attempt) * 10;
|
|
813
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
814
|
-
continue;
|
|
815
|
-
}
|
|
816
|
-
throw new error.MastraError(
|
|
817
|
-
{
|
|
818
|
-
id: "LANCE_STORE_UPDATE_RESOURCE_FAILED",
|
|
819
|
-
domain: error.ErrorDomain.STORAGE,
|
|
820
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
821
|
-
},
|
|
822
|
-
error$1
|
|
823
|
-
);
|
|
824
|
-
}
|
|
825
|
-
}
|
|
826
|
-
throw new Error("Unexpected end of retry loop");
|
|
827
|
-
}
|
|
828
535
|
};
|
|
829
|
-
|
|
536
|
+
|
|
537
|
+
// src/storage/domains/memory/index.ts
|
|
538
|
+
var StoreMemoryLance = class extends storage.MemoryStorage {
|
|
830
539
|
client;
|
|
831
|
-
|
|
540
|
+
#db;
|
|
541
|
+
constructor(config) {
|
|
832
542
|
super();
|
|
543
|
+
const client = resolveLanceConfig(config);
|
|
833
544
|
this.client = client;
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
case "jsonb":
|
|
845
|
-
return "'{}'";
|
|
846
|
-
case "uuid":
|
|
847
|
-
return "''";
|
|
848
|
-
default:
|
|
849
|
-
return super.getDefaultValue(type);
|
|
850
|
-
}
|
|
851
|
-
}
|
|
852
|
-
async hasColumn(tableName, columnName) {
|
|
853
|
-
const table = await this.client.openTable(tableName);
|
|
854
|
-
const schema = await table.schema();
|
|
855
|
-
return schema.fields.some((field) => field.name === columnName);
|
|
856
|
-
}
|
|
857
|
-
translateSchema(schema) {
|
|
858
|
-
const fields = Object.entries(schema).map(([name, column]) => {
|
|
859
|
-
let arrowType;
|
|
860
|
-
switch (column.type.toLowerCase()) {
|
|
861
|
-
case "text":
|
|
862
|
-
case "uuid":
|
|
863
|
-
arrowType = new apacheArrow.Utf8();
|
|
864
|
-
break;
|
|
865
|
-
case "int":
|
|
866
|
-
case "integer":
|
|
867
|
-
arrowType = new apacheArrow.Int32();
|
|
868
|
-
break;
|
|
869
|
-
case "bigint":
|
|
870
|
-
arrowType = new apacheArrow.Float64();
|
|
871
|
-
break;
|
|
872
|
-
case "float":
|
|
873
|
-
arrowType = new apacheArrow.Float32();
|
|
874
|
-
break;
|
|
875
|
-
case "jsonb":
|
|
876
|
-
case "json":
|
|
877
|
-
arrowType = new apacheArrow.Utf8();
|
|
878
|
-
break;
|
|
879
|
-
case "binary":
|
|
880
|
-
arrowType = new apacheArrow.Binary();
|
|
881
|
-
break;
|
|
882
|
-
case "timestamp":
|
|
883
|
-
arrowType = new apacheArrow.Float64();
|
|
884
|
-
break;
|
|
885
|
-
default:
|
|
886
|
-
arrowType = new apacheArrow.Utf8();
|
|
887
|
-
}
|
|
888
|
-
return new apacheArrow.Field(name, arrowType, column.nullable ?? true);
|
|
545
|
+
this.#db = new LanceDB({ client });
|
|
546
|
+
}
|
|
547
|
+
async init() {
|
|
548
|
+
await this.#db.createTable({ tableName: storage.TABLE_THREADS, schema: storage.TABLE_SCHEMAS[storage.TABLE_THREADS] });
|
|
549
|
+
await this.#db.createTable({ tableName: storage.TABLE_MESSAGES, schema: storage.TABLE_SCHEMAS[storage.TABLE_MESSAGES] });
|
|
550
|
+
await this.#db.createTable({ tableName: storage.TABLE_RESOURCES, schema: storage.TABLE_SCHEMAS[storage.TABLE_RESOURCES] });
|
|
551
|
+
await this.#db.alterTable({
|
|
552
|
+
tableName: storage.TABLE_MESSAGES,
|
|
553
|
+
schema: storage.TABLE_SCHEMAS[storage.TABLE_MESSAGES],
|
|
554
|
+
ifNotExists: ["resourceId"]
|
|
889
555
|
});
|
|
890
|
-
return new apacheArrow.Schema(fields);
|
|
891
556
|
}
|
|
892
|
-
async
|
|
893
|
-
tableName
|
|
894
|
-
|
|
895
|
-
|
|
557
|
+
async dangerouslyClearAll() {
|
|
558
|
+
await this.#db.clearTable({ tableName: storage.TABLE_THREADS });
|
|
559
|
+
await this.#db.clearTable({ tableName: storage.TABLE_MESSAGES });
|
|
560
|
+
await this.#db.clearTable({ tableName: storage.TABLE_RESOURCES });
|
|
561
|
+
}
|
|
562
|
+
async deleteMessages(messageIds) {
|
|
563
|
+
if (!messageIds || messageIds.length === 0) {
|
|
564
|
+
return;
|
|
565
|
+
}
|
|
566
|
+
this.logger.debug("Deleting messages", { count: messageIds.length });
|
|
896
567
|
try {
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
568
|
+
const threadIds = /* @__PURE__ */ new Set();
|
|
569
|
+
for (const messageId of messageIds) {
|
|
570
|
+
const message = await this.#db.load({ tableName: storage.TABLE_MESSAGES, keys: { id: messageId } });
|
|
571
|
+
if (message?.thread_id) {
|
|
572
|
+
threadIds.add(message.thread_id);
|
|
573
|
+
}
|
|
902
574
|
}
|
|
903
|
-
|
|
904
|
-
|
|
575
|
+
const messagesTable = await this.client.openTable(storage.TABLE_MESSAGES);
|
|
576
|
+
const idConditions = messageIds.map((id) => `id = '${this.escapeSql(id)}'`).join(" OR ");
|
|
577
|
+
await messagesTable.delete(idConditions);
|
|
578
|
+
const now = (/* @__PURE__ */ new Date()).getTime();
|
|
579
|
+
const threadsTable = await this.client.openTable(storage.TABLE_THREADS);
|
|
580
|
+
for (const threadId of threadIds) {
|
|
581
|
+
const thread = await this.getThreadById({ threadId });
|
|
582
|
+
if (thread) {
|
|
583
|
+
const record = {
|
|
584
|
+
id: threadId,
|
|
585
|
+
resourceId: thread.resourceId,
|
|
586
|
+
title: thread.title,
|
|
587
|
+
metadata: JSON.stringify(thread.metadata),
|
|
588
|
+
createdAt: new Date(thread.createdAt).getTime(),
|
|
589
|
+
updatedAt: now
|
|
590
|
+
};
|
|
591
|
+
await threadsTable.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
592
|
+
}
|
|
905
593
|
}
|
|
906
594
|
} catch (error$1) {
|
|
907
595
|
throw new error.MastraError(
|
|
908
596
|
{
|
|
909
|
-
id: "
|
|
597
|
+
id: storage.createStorageErrorId("LANCE", "DELETE_MESSAGES", "FAILED"),
|
|
910
598
|
domain: error.ErrorDomain.STORAGE,
|
|
911
|
-
category: error.ErrorCategory.
|
|
912
|
-
details: {
|
|
599
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
600
|
+
details: { count: messageIds.length }
|
|
913
601
|
},
|
|
914
602
|
error$1
|
|
915
603
|
);
|
|
916
604
|
}
|
|
605
|
+
}
|
|
606
|
+
// Utility to escape single quotes in SQL strings
|
|
607
|
+
escapeSql(str) {
|
|
608
|
+
return str.replace(/'/g, "''");
|
|
609
|
+
}
|
|
610
|
+
async getThreadById({ threadId }) {
|
|
917
611
|
try {
|
|
918
|
-
const
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
if (error$1.message?.includes("already exists")) {
|
|
922
|
-
this.logger.debug(`Table '${tableName}' already exists, skipping create`);
|
|
923
|
-
return;
|
|
612
|
+
const thread = await this.#db.load({ tableName: storage.TABLE_THREADS, keys: { id: threadId } });
|
|
613
|
+
if (!thread) {
|
|
614
|
+
return null;
|
|
924
615
|
}
|
|
616
|
+
return {
|
|
617
|
+
...thread,
|
|
618
|
+
createdAt: new Date(thread.createdAt),
|
|
619
|
+
updatedAt: new Date(thread.updatedAt)
|
|
620
|
+
};
|
|
621
|
+
} catch (error$1) {
|
|
925
622
|
throw new error.MastraError(
|
|
926
623
|
{
|
|
927
|
-
id: "
|
|
624
|
+
id: storage.createStorageErrorId("LANCE", "GET_THREAD_BY_ID", "FAILED"),
|
|
928
625
|
domain: error.ErrorDomain.STORAGE,
|
|
929
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
930
|
-
details: { tableName }
|
|
626
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
931
627
|
},
|
|
932
628
|
error$1
|
|
933
629
|
);
|
|
934
630
|
}
|
|
935
631
|
}
|
|
936
|
-
|
|
632
|
+
/**
|
|
633
|
+
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
634
|
+
* @param thread - The thread to save
|
|
635
|
+
* @returns The saved thread
|
|
636
|
+
*/
|
|
637
|
+
async saveThread({ thread }) {
|
|
937
638
|
try {
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
}
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
}
|
|
944
|
-
} catch (validationError) {
|
|
639
|
+
const record = { ...thread, metadata: JSON.stringify(thread.metadata) };
|
|
640
|
+
const table = await this.client.openTable(storage.TABLE_THREADS);
|
|
641
|
+
await table.add([record], { mode: "append" });
|
|
642
|
+
return thread;
|
|
643
|
+
} catch (error$1) {
|
|
945
644
|
throw new error.MastraError(
|
|
946
645
|
{
|
|
947
|
-
id: "
|
|
646
|
+
id: storage.createStorageErrorId("LANCE", "SAVE_THREAD", "FAILED"),
|
|
948
647
|
domain: error.ErrorDomain.STORAGE,
|
|
949
|
-
category: error.ErrorCategory.
|
|
950
|
-
text: validationError.message,
|
|
951
|
-
details: { tableName }
|
|
648
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
952
649
|
},
|
|
953
|
-
|
|
650
|
+
error$1
|
|
954
651
|
);
|
|
955
652
|
}
|
|
653
|
+
}
|
|
654
|
+
async updateThread({
|
|
655
|
+
id,
|
|
656
|
+
title,
|
|
657
|
+
metadata
|
|
658
|
+
}) {
|
|
659
|
+
const maxRetries = 5;
|
|
660
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
661
|
+
try {
|
|
662
|
+
const current = await this.getThreadById({ threadId: id });
|
|
663
|
+
if (!current) {
|
|
664
|
+
throw new Error(`Thread with id ${id} not found`);
|
|
665
|
+
}
|
|
666
|
+
const mergedMetadata = { ...current.metadata, ...metadata };
|
|
667
|
+
const record = {
|
|
668
|
+
id,
|
|
669
|
+
title,
|
|
670
|
+
metadata: JSON.stringify(mergedMetadata),
|
|
671
|
+
updatedAt: (/* @__PURE__ */ new Date()).getTime()
|
|
672
|
+
};
|
|
673
|
+
const table = await this.client.openTable(storage.TABLE_THREADS);
|
|
674
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
675
|
+
const updatedThread = await this.getThreadById({ threadId: id });
|
|
676
|
+
if (!updatedThread) {
|
|
677
|
+
throw new Error(`Failed to retrieve updated thread ${id}`);
|
|
678
|
+
}
|
|
679
|
+
return updatedThread;
|
|
680
|
+
} catch (error$1) {
|
|
681
|
+
if (error$1.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
682
|
+
const delay = Math.pow(2, attempt) * 10;
|
|
683
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
684
|
+
continue;
|
|
685
|
+
}
|
|
686
|
+
throw new error.MastraError(
|
|
687
|
+
{
|
|
688
|
+
id: storage.createStorageErrorId("LANCE", "UPDATE_THREAD", "FAILED"),
|
|
689
|
+
domain: error.ErrorDomain.STORAGE,
|
|
690
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
691
|
+
},
|
|
692
|
+
error$1
|
|
693
|
+
);
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
throw new error.MastraError(
|
|
697
|
+
{
|
|
698
|
+
id: storage.createStorageErrorId("LANCE", "UPDATE_THREAD", "FAILED"),
|
|
699
|
+
domain: error.ErrorDomain.STORAGE,
|
|
700
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
701
|
+
},
|
|
702
|
+
new Error("All retries exhausted")
|
|
703
|
+
);
|
|
704
|
+
}
|
|
705
|
+
async deleteThread({ threadId }) {
|
|
956
706
|
try {
|
|
957
|
-
await this.client.
|
|
707
|
+
const table = await this.client.openTable(storage.TABLE_THREADS);
|
|
708
|
+
await table.delete(`id = '${threadId}'`);
|
|
709
|
+
const messagesTable = await this.client.openTable(storage.TABLE_MESSAGES);
|
|
710
|
+
await messagesTable.delete(`thread_id = '${threadId}'`);
|
|
958
711
|
} catch (error$1) {
|
|
959
|
-
if (error$1.toString().includes("was not found") || error$1.message?.includes("Table not found")) {
|
|
960
|
-
this.logger.debug(`Table '${tableName}' does not exist, skipping drop`);
|
|
961
|
-
return;
|
|
962
|
-
}
|
|
963
712
|
throw new error.MastraError(
|
|
964
713
|
{
|
|
965
|
-
id: "
|
|
714
|
+
id: storage.createStorageErrorId("LANCE", "DELETE_THREAD", "FAILED"),
|
|
966
715
|
domain: error.ErrorDomain.STORAGE,
|
|
967
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
968
|
-
details: { tableName }
|
|
716
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
969
717
|
},
|
|
970
718
|
error$1
|
|
971
719
|
);
|
|
972
720
|
}
|
|
973
721
|
}
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
722
|
+
normalizeMessage(message) {
|
|
723
|
+
const { thread_id, ...rest } = message;
|
|
724
|
+
return {
|
|
725
|
+
...rest,
|
|
726
|
+
threadId: thread_id,
|
|
727
|
+
content: typeof message.content === "string" ? (() => {
|
|
728
|
+
try {
|
|
729
|
+
return JSON.parse(message.content);
|
|
730
|
+
} catch {
|
|
731
|
+
return message.content;
|
|
732
|
+
}
|
|
733
|
+
})() : message.content
|
|
734
|
+
};
|
|
735
|
+
}
|
|
736
|
+
async listMessagesById({ messageIds }) {
|
|
737
|
+
if (messageIds.length === 0) return { messages: [] };
|
|
979
738
|
try {
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
}
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
} catch (validationError) {
|
|
739
|
+
const table = await this.client.openTable(storage.TABLE_MESSAGES);
|
|
740
|
+
const quotedIds = messageIds.map((id) => `'${id}'`).join(", ");
|
|
741
|
+
const allRecords = await table.query().where(`id IN (${quotedIds})`).toArray();
|
|
742
|
+
const messages = processResultWithTypeConversion(
|
|
743
|
+
allRecords,
|
|
744
|
+
await getTableSchema({ tableName: storage.TABLE_MESSAGES, client: this.client })
|
|
745
|
+
);
|
|
746
|
+
const list = new agent.MessageList().add(
|
|
747
|
+
messages.map(this.normalizeMessage),
|
|
748
|
+
"memory"
|
|
749
|
+
);
|
|
750
|
+
return { messages: list.get.all.db() };
|
|
751
|
+
} catch (error$1) {
|
|
994
752
|
throw new error.MastraError(
|
|
995
753
|
{
|
|
996
|
-
id: "
|
|
754
|
+
id: storage.createStorageErrorId("LANCE", "LIST_MESSAGES_BY_ID", "FAILED"),
|
|
997
755
|
domain: error.ErrorDomain.STORAGE,
|
|
998
|
-
category: error.ErrorCategory.
|
|
999
|
-
|
|
1000
|
-
|
|
756
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
757
|
+
details: {
|
|
758
|
+
messageIds: JSON.stringify(messageIds)
|
|
759
|
+
}
|
|
1001
760
|
},
|
|
1002
|
-
|
|
761
|
+
error$1
|
|
1003
762
|
);
|
|
1004
763
|
}
|
|
764
|
+
}
|
|
765
|
+
async listMessages(args) {
|
|
766
|
+
const { threadId, resourceId, include, filter, perPage: perPageInput, page = 0, orderBy } = args;
|
|
767
|
+
const threadIds = Array.isArray(threadId) ? threadId : [threadId];
|
|
768
|
+
if (threadIds.length === 0 || threadIds.some((id) => !id.trim())) {
|
|
769
|
+
throw new error.MastraError(
|
|
770
|
+
{
|
|
771
|
+
id: storage.createStorageErrorId("LANCE", "LIST_MESSAGES", "INVALID_THREAD_ID"),
|
|
772
|
+
domain: error.ErrorDomain.STORAGE,
|
|
773
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
774
|
+
details: { threadId: Array.isArray(threadId) ? threadId.join(",") : threadId }
|
|
775
|
+
},
|
|
776
|
+
new Error("threadId must be a non-empty string or array of non-empty strings")
|
|
777
|
+
);
|
|
778
|
+
}
|
|
779
|
+
const perPage = storage.normalizePerPage(perPageInput, 40);
|
|
780
|
+
const { offset, perPage: perPageForResponse } = storage.calculatePagination(page, perPageInput, perPage);
|
|
1005
781
|
try {
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
}
|
|
1017
|
-
const
|
|
1018
|
-
|
|
782
|
+
if (page < 0) {
|
|
783
|
+
throw new error.MastraError(
|
|
784
|
+
{
|
|
785
|
+
id: storage.createStorageErrorId("LANCE", "LIST_MESSAGES", "INVALID_PAGE"),
|
|
786
|
+
domain: error.ErrorDomain.STORAGE,
|
|
787
|
+
category: error.ErrorCategory.USER,
|
|
788
|
+
details: { page }
|
|
789
|
+
},
|
|
790
|
+
new Error("page must be >= 0")
|
|
791
|
+
);
|
|
792
|
+
}
|
|
793
|
+
const { field, direction } = this.parseOrderBy(orderBy, "ASC");
|
|
794
|
+
const table = await this.client.openTable(storage.TABLE_MESSAGES);
|
|
795
|
+
const threadCondition = threadIds.length === 1 ? `thread_id = '${this.escapeSql(threadIds[0])}'` : `thread_id IN (${threadIds.map((t) => `'${this.escapeSql(t)}'`).join(", ")})`;
|
|
796
|
+
const conditions = [threadCondition];
|
|
797
|
+
if (resourceId) {
|
|
798
|
+
conditions.push(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
799
|
+
}
|
|
800
|
+
if (filter?.dateRange?.start) {
|
|
801
|
+
const startTime = filter.dateRange.start instanceof Date ? filter.dateRange.start.getTime() : new Date(filter.dateRange.start).getTime();
|
|
802
|
+
const startOp = filter.dateRange.startExclusive ? ">" : ">=";
|
|
803
|
+
conditions.push(`\`createdAt\` ${startOp} ${startTime}`);
|
|
804
|
+
}
|
|
805
|
+
if (filter?.dateRange?.end) {
|
|
806
|
+
const endTime = filter.dateRange.end instanceof Date ? filter.dateRange.end.getTime() : new Date(filter.dateRange.end).getTime();
|
|
807
|
+
const endOp = filter.dateRange.endExclusive ? "<" : "<=";
|
|
808
|
+
conditions.push(`\`createdAt\` ${endOp} ${endTime}`);
|
|
809
|
+
}
|
|
810
|
+
const whereClause = conditions.join(" AND ");
|
|
811
|
+
const total = await table.countRows(whereClause);
|
|
812
|
+
const query = table.query().where(whereClause);
|
|
813
|
+
let allRecords = await query.toArray();
|
|
814
|
+
allRecords.sort((a, b) => {
|
|
815
|
+
const aValue = field === "createdAt" ? a.createdAt : a[field];
|
|
816
|
+
const bValue = field === "createdAt" ? b.createdAt : b[field];
|
|
817
|
+
if (aValue == null && bValue == null) return 0;
|
|
818
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
819
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
820
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
821
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
822
|
+
}
|
|
823
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
824
|
+
});
|
|
825
|
+
const paginatedRecords = allRecords.slice(offset, offset + perPage);
|
|
826
|
+
const messages = paginatedRecords.map((row) => this.normalizeMessage(row));
|
|
827
|
+
if (total === 0 && messages.length === 0 && (!include || include.length === 0)) {
|
|
1019
828
|
return {
|
|
1020
|
-
|
|
1021
|
-
|
|
829
|
+
messages: [],
|
|
830
|
+
total: 0,
|
|
831
|
+
page,
|
|
832
|
+
perPage: perPageForResponse,
|
|
833
|
+
hasMore: false
|
|
1022
834
|
};
|
|
1023
|
-
});
|
|
1024
|
-
if (columnsToAdd.length > 0) {
|
|
1025
|
-
await table.addColumns(columnsToAdd);
|
|
1026
|
-
this.logger?.info?.(`Added columns [${columnsToAdd.map((c) => c.name).join(", ")}] to table ${tableName}`);
|
|
1027
835
|
}
|
|
836
|
+
const messageIds = new Set(messages.map((m) => m.id));
|
|
837
|
+
if (include && include.length > 0) {
|
|
838
|
+
const threadIds2 = [...new Set(include.map((item) => item.threadId || threadId))];
|
|
839
|
+
const allThreadMessages = [];
|
|
840
|
+
for (const tid of threadIds2) {
|
|
841
|
+
const threadQuery = table.query().where(`thread_id = '${tid}'`);
|
|
842
|
+
let threadRecords = await threadQuery.toArray();
|
|
843
|
+
allThreadMessages.push(...threadRecords);
|
|
844
|
+
}
|
|
845
|
+
allThreadMessages.sort((a, b) => a.createdAt - b.createdAt);
|
|
846
|
+
const contextMessages = this.processMessagesWithContext(allThreadMessages, include);
|
|
847
|
+
const includedMessages = contextMessages.map((row) => this.normalizeMessage(row));
|
|
848
|
+
for (const includeMsg of includedMessages) {
|
|
849
|
+
if (!messageIds.has(includeMsg.id)) {
|
|
850
|
+
messages.push(includeMsg);
|
|
851
|
+
messageIds.add(includeMsg.id);
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
const list = new agent.MessageList().add(messages, "memory");
|
|
856
|
+
let finalMessages = list.get.all.db();
|
|
857
|
+
finalMessages = finalMessages.sort((a, b) => {
|
|
858
|
+
const aValue = field === "createdAt" ? new Date(a.createdAt).getTime() : a[field];
|
|
859
|
+
const bValue = field === "createdAt" ? new Date(b.createdAt).getTime() : b[field];
|
|
860
|
+
if (aValue == null && bValue == null) return 0;
|
|
861
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
862
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
863
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
864
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
865
|
+
}
|
|
866
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
867
|
+
});
|
|
868
|
+
const returnedThreadMessageIds = new Set(finalMessages.filter((m) => m.threadId === threadId).map((m) => m.id));
|
|
869
|
+
const allThreadMessagesReturned = returnedThreadMessageIds.size >= total;
|
|
870
|
+
const fetchedAll = perPageInput === false || allThreadMessagesReturned;
|
|
871
|
+
const hasMore = !fetchedAll && offset + perPage < total;
|
|
872
|
+
return {
|
|
873
|
+
messages: finalMessages,
|
|
874
|
+
total,
|
|
875
|
+
page,
|
|
876
|
+
perPage: perPageForResponse,
|
|
877
|
+
hasMore
|
|
878
|
+
};
|
|
1028
879
|
} catch (error$1) {
|
|
1029
|
-
|
|
880
|
+
const mastraError = new error.MastraError(
|
|
1030
881
|
{
|
|
1031
|
-
id: "
|
|
882
|
+
id: storage.createStorageErrorId("LANCE", "LIST_MESSAGES", "FAILED"),
|
|
1032
883
|
domain: error.ErrorDomain.STORAGE,
|
|
1033
884
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
1034
|
-
details: {
|
|
885
|
+
details: {
|
|
886
|
+
threadId: Array.isArray(threadId) ? threadId.join(",") : threadId,
|
|
887
|
+
resourceId: resourceId ?? ""
|
|
888
|
+
}
|
|
1035
889
|
},
|
|
1036
890
|
error$1
|
|
1037
891
|
);
|
|
892
|
+
this.logger?.error?.(mastraError.toString());
|
|
893
|
+
this.logger?.trackException?.(mastraError);
|
|
894
|
+
return {
|
|
895
|
+
messages: [],
|
|
896
|
+
total: 0,
|
|
897
|
+
page,
|
|
898
|
+
perPage: perPageForResponse,
|
|
899
|
+
hasMore: false
|
|
900
|
+
};
|
|
1038
901
|
}
|
|
1039
902
|
}
|
|
1040
|
-
async
|
|
903
|
+
async saveMessages(args) {
|
|
1041
904
|
try {
|
|
1042
|
-
|
|
1043
|
-
|
|
905
|
+
const { messages } = args;
|
|
906
|
+
if (messages.length === 0) {
|
|
907
|
+
return { messages: [] };
|
|
1044
908
|
}
|
|
1045
|
-
|
|
1046
|
-
|
|
909
|
+
const threadId = messages[0]?.threadId;
|
|
910
|
+
if (!threadId) {
|
|
911
|
+
throw new Error("Thread ID is required");
|
|
1047
912
|
}
|
|
1048
|
-
|
|
913
|
+
for (const message of messages) {
|
|
914
|
+
if (!message.id) {
|
|
915
|
+
throw new Error("Message ID is required");
|
|
916
|
+
}
|
|
917
|
+
if (!message.threadId) {
|
|
918
|
+
throw new Error("Thread ID is required for all messages");
|
|
919
|
+
}
|
|
920
|
+
if (message.resourceId === null || message.resourceId === void 0) {
|
|
921
|
+
throw new Error("Resource ID cannot be null or undefined");
|
|
922
|
+
}
|
|
923
|
+
if (!message.content) {
|
|
924
|
+
throw new Error("Message content is required");
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
const transformedMessages = messages.map((message) => {
|
|
928
|
+
const { threadId: threadId2, type, ...rest } = message;
|
|
929
|
+
return {
|
|
930
|
+
...rest,
|
|
931
|
+
thread_id: threadId2,
|
|
932
|
+
type: type ?? "v2",
|
|
933
|
+
content: JSON.stringify(message.content)
|
|
934
|
+
};
|
|
935
|
+
});
|
|
936
|
+
const table = await this.client.openTable(storage.TABLE_MESSAGES);
|
|
937
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(transformedMessages);
|
|
938
|
+
const threadsTable = await this.client.openTable(storage.TABLE_THREADS);
|
|
939
|
+
const currentTime = (/* @__PURE__ */ new Date()).getTime();
|
|
940
|
+
const updateRecord = { id: threadId, updatedAt: currentTime };
|
|
941
|
+
await threadsTable.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([updateRecord]);
|
|
942
|
+
const list = new agent.MessageList().add(messages, "memory");
|
|
943
|
+
return { messages: list.get.all.db() };
|
|
944
|
+
} catch (error$1) {
|
|
1049
945
|
throw new error.MastraError(
|
|
1050
946
|
{
|
|
1051
|
-
id: "
|
|
947
|
+
id: storage.createStorageErrorId("LANCE", "SAVE_MESSAGES", "FAILED"),
|
|
1052
948
|
domain: error.ErrorDomain.STORAGE,
|
|
1053
|
-
category: error.ErrorCategory.
|
|
1054
|
-
text: validationError.message,
|
|
1055
|
-
details: { tableName }
|
|
949
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
1056
950
|
},
|
|
1057
|
-
|
|
951
|
+
error$1
|
|
1058
952
|
);
|
|
1059
953
|
}
|
|
954
|
+
}
|
|
955
|
+
async listThreadsByResourceId(args) {
|
|
1060
956
|
try {
|
|
1061
|
-
const
|
|
1062
|
-
|
|
957
|
+
const { resourceId, page = 0, perPage: perPageInput, orderBy } = args;
|
|
958
|
+
const perPage = storage.normalizePerPage(perPageInput, 100);
|
|
959
|
+
if (page < 0) {
|
|
960
|
+
throw new error.MastraError(
|
|
961
|
+
{
|
|
962
|
+
id: storage.createStorageErrorId("LANCE", "LIST_THREADS_BY_RESOURCE_ID", "INVALID_PAGE"),
|
|
963
|
+
domain: error.ErrorDomain.STORAGE,
|
|
964
|
+
category: error.ErrorCategory.USER,
|
|
965
|
+
details: { page }
|
|
966
|
+
},
|
|
967
|
+
new Error("page must be >= 0")
|
|
968
|
+
);
|
|
969
|
+
}
|
|
970
|
+
const { offset, perPage: perPageForResponse } = storage.calculatePagination(page, perPageInput, perPage);
|
|
971
|
+
const { field, direction } = this.parseOrderBy(orderBy);
|
|
972
|
+
const table = await this.client.openTable(storage.TABLE_THREADS);
|
|
973
|
+
const total = await table.countRows(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
974
|
+
const query = table.query().where(`\`resourceId\` = '${this.escapeSql(resourceId)}'`);
|
|
975
|
+
const records = await query.toArray();
|
|
976
|
+
records.sort((a, b) => {
|
|
977
|
+
const aValue = ["createdAt", "updatedAt"].includes(field) ? new Date(a[field]).getTime() : a[field];
|
|
978
|
+
const bValue = ["createdAt", "updatedAt"].includes(field) ? new Date(b[field]).getTime() : b[field];
|
|
979
|
+
if (aValue == null && bValue == null) return 0;
|
|
980
|
+
if (aValue == null) return direction === "ASC" ? -1 : 1;
|
|
981
|
+
if (bValue == null) return direction === "ASC" ? 1 : -1;
|
|
982
|
+
if (typeof aValue === "string" && typeof bValue === "string") {
|
|
983
|
+
return direction === "ASC" ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
|
|
984
|
+
}
|
|
985
|
+
return direction === "ASC" ? aValue - bValue : bValue - aValue;
|
|
986
|
+
});
|
|
987
|
+
const paginatedRecords = records.slice(offset, offset + perPage);
|
|
988
|
+
const schema = await getTableSchema({ tableName: storage.TABLE_THREADS, client: this.client });
|
|
989
|
+
const threads = paginatedRecords.map(
|
|
990
|
+
(record) => processResultWithTypeConversion(record, schema)
|
|
991
|
+
);
|
|
992
|
+
return {
|
|
993
|
+
threads,
|
|
994
|
+
total,
|
|
995
|
+
page,
|
|
996
|
+
perPage: perPageForResponse,
|
|
997
|
+
hasMore: offset + perPage < total
|
|
998
|
+
};
|
|
1063
999
|
} catch (error$1) {
|
|
1064
1000
|
throw new error.MastraError(
|
|
1065
1001
|
{
|
|
1066
|
-
id: "
|
|
1002
|
+
id: storage.createStorageErrorId("LANCE", "LIST_THREADS_BY_RESOURCE_ID", "FAILED"),
|
|
1067
1003
|
domain: error.ErrorDomain.STORAGE,
|
|
1068
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
1069
|
-
details: { tableName }
|
|
1004
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
1070
1005
|
},
|
|
1071
1006
|
error$1
|
|
1072
1007
|
);
|
|
1073
1008
|
}
|
|
1074
1009
|
}
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1010
|
+
/**
|
|
1011
|
+
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
1012
|
+
* @param records - The sorted array of records to process
|
|
1013
|
+
* @param include - The array of include specifications with context parameters
|
|
1014
|
+
* @returns The processed array with context messages included
|
|
1015
|
+
*/
|
|
1016
|
+
processMessagesWithContext(records, include) {
|
|
1017
|
+
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
1018
|
+
if (messagesWithContext.length === 0) {
|
|
1019
|
+
return records;
|
|
1020
|
+
}
|
|
1021
|
+
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
1022
|
+
records.forEach((message, index) => {
|
|
1023
|
+
messageIndexMap.set(message.id, index);
|
|
1024
|
+
});
|
|
1025
|
+
const additionalIndices = /* @__PURE__ */ new Set();
|
|
1026
|
+
for (const item of messagesWithContext) {
|
|
1027
|
+
const messageIndex = messageIndexMap.get(item.id);
|
|
1028
|
+
if (messageIndex !== void 0) {
|
|
1029
|
+
if (item.withPreviousMessages) {
|
|
1030
|
+
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
1031
|
+
for (let i = startIdx; i < messageIndex; i++) {
|
|
1032
|
+
additionalIndices.add(i);
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
if (item.withNextMessages) {
|
|
1036
|
+
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
1037
|
+
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
1038
|
+
additionalIndices.add(i);
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1082
1041
|
}
|
|
1083
|
-
|
|
1084
|
-
|
|
1042
|
+
}
|
|
1043
|
+
if (additionalIndices.size === 0) {
|
|
1044
|
+
return records;
|
|
1045
|
+
}
|
|
1046
|
+
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
1047
|
+
const allIndices = /* @__PURE__ */ new Set();
|
|
1048
|
+
records.forEach((record, index) => {
|
|
1049
|
+
if (originalMatchIds.has(record.id)) {
|
|
1050
|
+
allIndices.add(index);
|
|
1085
1051
|
}
|
|
1086
|
-
}
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1052
|
+
});
|
|
1053
|
+
additionalIndices.forEach((index) => {
|
|
1054
|
+
allIndices.add(index);
|
|
1055
|
+
});
|
|
1056
|
+
return Array.from(allIndices).sort((a, b) => a - b).map((index) => records[index]);
|
|
1057
|
+
}
|
|
1058
|
+
/**
|
|
1059
|
+
* Parse message data from LanceDB record format to MastraDBMessage format
|
|
1060
|
+
*/
|
|
1061
|
+
parseMessageData(data) {
|
|
1062
|
+
const { thread_id, ...rest } = data;
|
|
1063
|
+
return {
|
|
1064
|
+
...rest,
|
|
1065
|
+
threadId: thread_id,
|
|
1066
|
+
content: typeof data.content === "string" ? (() => {
|
|
1067
|
+
try {
|
|
1068
|
+
return JSON.parse(data.content);
|
|
1069
|
+
} catch {
|
|
1070
|
+
return data.content;
|
|
1071
|
+
}
|
|
1072
|
+
})() : data.content,
|
|
1073
|
+
createdAt: new Date(data.createdAt),
|
|
1074
|
+
updatedAt: new Date(data.updatedAt)
|
|
1075
|
+
};
|
|
1076
|
+
}
|
|
1077
|
+
async updateMessages(args) {
|
|
1078
|
+
const { messages } = args;
|
|
1079
|
+
this.logger.debug("Updating messages", { count: messages.length });
|
|
1080
|
+
if (!messages.length) {
|
|
1081
|
+
return [];
|
|
1097
1082
|
}
|
|
1083
|
+
const updatedMessages = [];
|
|
1084
|
+
const affectedThreadIds = /* @__PURE__ */ new Set();
|
|
1098
1085
|
try {
|
|
1099
|
-
const
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1086
|
+
for (const updateData of messages) {
|
|
1087
|
+
const { id, ...updates } = updateData;
|
|
1088
|
+
const existingMessage = await this.#db.load({ tableName: storage.TABLE_MESSAGES, keys: { id } });
|
|
1089
|
+
if (!existingMessage) {
|
|
1090
|
+
this.logger.warn("Message not found for update", { id });
|
|
1091
|
+
continue;
|
|
1092
|
+
}
|
|
1093
|
+
const existingMsg = this.parseMessageData(existingMessage);
|
|
1094
|
+
const originalThreadId = existingMsg.threadId;
|
|
1095
|
+
affectedThreadIds.add(originalThreadId);
|
|
1096
|
+
const updatePayload = {};
|
|
1097
|
+
if ("role" in updates && updates.role !== void 0) updatePayload.role = updates.role;
|
|
1098
|
+
if ("type" in updates && updates.type !== void 0) updatePayload.type = updates.type;
|
|
1099
|
+
if ("resourceId" in updates && updates.resourceId !== void 0) updatePayload.resourceId = updates.resourceId;
|
|
1100
|
+
if ("threadId" in updates && updates.threadId !== void 0 && updates.threadId !== null) {
|
|
1101
|
+
updatePayload.thread_id = updates.threadId;
|
|
1102
|
+
affectedThreadIds.add(updates.threadId);
|
|
1106
1103
|
}
|
|
1104
|
+
if (updates.content) {
|
|
1105
|
+
const existingContent = existingMsg.content;
|
|
1106
|
+
let newContent = { ...existingContent };
|
|
1107
|
+
if (updates.content.metadata !== void 0) {
|
|
1108
|
+
newContent.metadata = {
|
|
1109
|
+
...existingContent.metadata || {},
|
|
1110
|
+
...updates.content.metadata || {}
|
|
1111
|
+
};
|
|
1112
|
+
}
|
|
1113
|
+
if (updates.content.content !== void 0) {
|
|
1114
|
+
newContent.content = updates.content.content;
|
|
1115
|
+
}
|
|
1116
|
+
if ("parts" in updates.content && updates.content.parts !== void 0) {
|
|
1117
|
+
newContent.parts = updates.content.parts;
|
|
1118
|
+
}
|
|
1119
|
+
updatePayload.content = JSON.stringify(newContent);
|
|
1120
|
+
}
|
|
1121
|
+
await this.#db.insert({ tableName: storage.TABLE_MESSAGES, record: { id, ...updatePayload } });
|
|
1122
|
+
const updatedMessage = await this.#db.load({ tableName: storage.TABLE_MESSAGES, keys: { id } });
|
|
1123
|
+
if (updatedMessage) {
|
|
1124
|
+
updatedMessages.push(this.parseMessageData(updatedMessage));
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
for (const threadId of affectedThreadIds) {
|
|
1128
|
+
await this.#db.insert({
|
|
1129
|
+
tableName: storage.TABLE_THREADS,
|
|
1130
|
+
record: { id: threadId, updatedAt: Date.now() }
|
|
1131
|
+
});
|
|
1107
1132
|
}
|
|
1108
|
-
|
|
1109
|
-
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([processedRecord]);
|
|
1133
|
+
return updatedMessages;
|
|
1110
1134
|
} catch (error$1) {
|
|
1111
1135
|
throw new error.MastraError(
|
|
1112
1136
|
{
|
|
1113
|
-
id: "
|
|
1137
|
+
id: storage.createStorageErrorId("LANCE", "UPDATE_MESSAGES", "FAILED"),
|
|
1114
1138
|
domain: error.ErrorDomain.STORAGE,
|
|
1115
1139
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
1116
|
-
details: {
|
|
1140
|
+
details: { count: messages.length }
|
|
1117
1141
|
},
|
|
1118
1142
|
error$1
|
|
1119
1143
|
);
|
|
1120
1144
|
}
|
|
1121
1145
|
}
|
|
1122
|
-
async
|
|
1146
|
+
async getResourceById({ resourceId }) {
|
|
1123
1147
|
try {
|
|
1124
|
-
|
|
1125
|
-
|
|
1148
|
+
const resource = await this.#db.load({ tableName: storage.TABLE_RESOURCES, keys: { id: resourceId } });
|
|
1149
|
+
if (!resource) {
|
|
1150
|
+
return null;
|
|
1126
1151
|
}
|
|
1127
|
-
|
|
1128
|
-
|
|
1152
|
+
let createdAt;
|
|
1153
|
+
let updatedAt;
|
|
1154
|
+
try {
|
|
1155
|
+
if (resource.createdAt instanceof Date) {
|
|
1156
|
+
createdAt = resource.createdAt;
|
|
1157
|
+
} else if (typeof resource.createdAt === "string") {
|
|
1158
|
+
createdAt = new Date(resource.createdAt);
|
|
1159
|
+
} else if (typeof resource.createdAt === "number") {
|
|
1160
|
+
createdAt = new Date(resource.createdAt);
|
|
1161
|
+
} else {
|
|
1162
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1163
|
+
}
|
|
1164
|
+
if (isNaN(createdAt.getTime())) {
|
|
1165
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1166
|
+
}
|
|
1167
|
+
} catch {
|
|
1168
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
1129
1169
|
}
|
|
1130
|
-
|
|
1131
|
-
|
|
1170
|
+
try {
|
|
1171
|
+
if (resource.updatedAt instanceof Date) {
|
|
1172
|
+
updatedAt = resource.updatedAt;
|
|
1173
|
+
} else if (typeof resource.updatedAt === "string") {
|
|
1174
|
+
updatedAt = new Date(resource.updatedAt);
|
|
1175
|
+
} else if (typeof resource.updatedAt === "number") {
|
|
1176
|
+
updatedAt = new Date(resource.updatedAt);
|
|
1177
|
+
} else {
|
|
1178
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1179
|
+
}
|
|
1180
|
+
if (isNaN(updatedAt.getTime())) {
|
|
1181
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1182
|
+
}
|
|
1183
|
+
} catch {
|
|
1184
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
1132
1185
|
}
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
const processedRecord = { ...record };
|
|
1150
|
-
for (const key in processedRecord) {
|
|
1151
|
-
if (processedRecord[key] == null) continue;
|
|
1152
|
-
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
1153
|
-
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
1154
|
-
}
|
|
1186
|
+
let workingMemory = resource.workingMemory;
|
|
1187
|
+
if (workingMemory === null || workingMemory === void 0) {
|
|
1188
|
+
workingMemory = void 0;
|
|
1189
|
+
} else if (workingMemory === "") {
|
|
1190
|
+
workingMemory = "";
|
|
1191
|
+
} else if (typeof workingMemory === "object") {
|
|
1192
|
+
workingMemory = JSON.stringify(workingMemory);
|
|
1193
|
+
}
|
|
1194
|
+
let metadata = resource.metadata;
|
|
1195
|
+
if (metadata === "" || metadata === null || metadata === void 0) {
|
|
1196
|
+
metadata = void 0;
|
|
1197
|
+
} else if (typeof metadata === "string") {
|
|
1198
|
+
try {
|
|
1199
|
+
metadata = JSON.parse(metadata);
|
|
1200
|
+
} catch {
|
|
1201
|
+
metadata = metadata;
|
|
1155
1202
|
}
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1203
|
+
}
|
|
1204
|
+
return {
|
|
1205
|
+
...resource,
|
|
1206
|
+
createdAt,
|
|
1207
|
+
updatedAt,
|
|
1208
|
+
workingMemory,
|
|
1209
|
+
metadata
|
|
1210
|
+
};
|
|
1159
1211
|
} catch (error$1) {
|
|
1160
1212
|
throw new error.MastraError(
|
|
1161
1213
|
{
|
|
1162
|
-
id: "
|
|
1214
|
+
id: storage.createStorageErrorId("LANCE", "GET_RESOURCE_BY_ID", "FAILED"),
|
|
1163
1215
|
domain: error.ErrorDomain.STORAGE,
|
|
1164
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
1165
|
-
details: { tableName }
|
|
1216
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
1166
1217
|
},
|
|
1167
1218
|
error$1
|
|
1168
1219
|
);
|
|
1169
1220
|
}
|
|
1170
1221
|
}
|
|
1171
|
-
async
|
|
1172
|
-
try {
|
|
1173
|
-
if (!this.client) {
|
|
1174
|
-
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1175
|
-
}
|
|
1176
|
-
if (!tableName) {
|
|
1177
|
-
throw new Error("tableName is required for load.");
|
|
1178
|
-
}
|
|
1179
|
-
if (!keys || Object.keys(keys).length === 0) {
|
|
1180
|
-
throw new Error("keys are required and cannot be empty for load.");
|
|
1181
|
-
}
|
|
1182
|
-
} catch (validationError) {
|
|
1183
|
-
throw new error.MastraError(
|
|
1184
|
-
{
|
|
1185
|
-
id: "STORAGE_LANCE_STORAGE_LOAD_INVALID_ARGS",
|
|
1186
|
-
domain: error.ErrorDomain.STORAGE,
|
|
1187
|
-
category: error.ErrorCategory.USER,
|
|
1188
|
-
text: validationError.message,
|
|
1189
|
-
details: { tableName }
|
|
1190
|
-
},
|
|
1191
|
-
validationError
|
|
1192
|
-
);
|
|
1193
|
-
}
|
|
1222
|
+
async saveResource({ resource }) {
|
|
1194
1223
|
try {
|
|
1195
|
-
const
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
return `${quotedKey} IS NULL`;
|
|
1207
|
-
} else {
|
|
1208
|
-
return `${quotedKey} = ${value}`;
|
|
1209
|
-
}
|
|
1210
|
-
}).join(" AND ");
|
|
1211
|
-
this.logger.debug("where clause generated: " + filterConditions);
|
|
1212
|
-
query.where(filterConditions);
|
|
1213
|
-
}
|
|
1214
|
-
const result = await query.limit(1).toArray();
|
|
1215
|
-
if (result.length === 0) {
|
|
1216
|
-
this.logger.debug("No record found");
|
|
1217
|
-
return null;
|
|
1218
|
-
}
|
|
1219
|
-
return processResultWithTypeConversion(result[0], tableSchema);
|
|
1224
|
+
const record = {
|
|
1225
|
+
...resource,
|
|
1226
|
+
metadata: resource.metadata ? JSON.stringify(resource.metadata) : "",
|
|
1227
|
+
createdAt: resource.createdAt.getTime(),
|
|
1228
|
+
// Store as timestamp (milliseconds)
|
|
1229
|
+
updatedAt: resource.updatedAt.getTime()
|
|
1230
|
+
// Store as timestamp (milliseconds)
|
|
1231
|
+
};
|
|
1232
|
+
const table = await this.client.openTable(storage.TABLE_RESOURCES);
|
|
1233
|
+
await table.add([record], { mode: "append" });
|
|
1234
|
+
return resource;
|
|
1220
1235
|
} catch (error$1) {
|
|
1221
|
-
if (error$1 instanceof error.MastraError) throw error$1;
|
|
1222
1236
|
throw new error.MastraError(
|
|
1223
1237
|
{
|
|
1224
|
-
id: "
|
|
1238
|
+
id: storage.createStorageErrorId("LANCE", "SAVE_RESOURCE", "FAILED"),
|
|
1225
1239
|
domain: error.ErrorDomain.STORAGE,
|
|
1226
|
-
category: error.ErrorCategory.THIRD_PARTY
|
|
1227
|
-
details: { tableName, keyCount: Object.keys(keys).length, firstKey: Object.keys(keys)[0] ?? "" }
|
|
1240
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
1228
1241
|
},
|
|
1229
1242
|
error$1
|
|
1230
1243
|
);
|
|
1231
1244
|
}
|
|
1232
1245
|
}
|
|
1246
|
+
async updateResource({
|
|
1247
|
+
resourceId,
|
|
1248
|
+
workingMemory,
|
|
1249
|
+
metadata
|
|
1250
|
+
}) {
|
|
1251
|
+
const maxRetries = 3;
|
|
1252
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
1253
|
+
try {
|
|
1254
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
1255
|
+
if (!existingResource) {
|
|
1256
|
+
const newResource = {
|
|
1257
|
+
id: resourceId,
|
|
1258
|
+
workingMemory,
|
|
1259
|
+
metadata: metadata || {},
|
|
1260
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1261
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1262
|
+
};
|
|
1263
|
+
return this.saveResource({ resource: newResource });
|
|
1264
|
+
}
|
|
1265
|
+
const updatedResource = {
|
|
1266
|
+
...existingResource,
|
|
1267
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
1268
|
+
metadata: {
|
|
1269
|
+
...existingResource.metadata,
|
|
1270
|
+
...metadata
|
|
1271
|
+
},
|
|
1272
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1273
|
+
};
|
|
1274
|
+
const record = {
|
|
1275
|
+
id: resourceId,
|
|
1276
|
+
workingMemory: updatedResource.workingMemory || "",
|
|
1277
|
+
metadata: updatedResource.metadata ? JSON.stringify(updatedResource.metadata) : "",
|
|
1278
|
+
updatedAt: updatedResource.updatedAt.getTime()
|
|
1279
|
+
// Store as timestamp (milliseconds)
|
|
1280
|
+
};
|
|
1281
|
+
const table = await this.client.openTable(storage.TABLE_RESOURCES);
|
|
1282
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
1283
|
+
return updatedResource;
|
|
1284
|
+
} catch (error$1) {
|
|
1285
|
+
if (error$1.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
1286
|
+
const delay = Math.pow(2, attempt) * 10;
|
|
1287
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1288
|
+
continue;
|
|
1289
|
+
}
|
|
1290
|
+
throw new error.MastraError(
|
|
1291
|
+
{
|
|
1292
|
+
id: storage.createStorageErrorId("LANCE", "UPDATE_RESOURCE", "FAILED"),
|
|
1293
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1294
|
+
category: error.ErrorCategory.THIRD_PARTY
|
|
1295
|
+
},
|
|
1296
|
+
error$1
|
|
1297
|
+
);
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
throw new Error("Unexpected end of retry loop");
|
|
1301
|
+
}
|
|
1233
1302
|
};
|
|
1234
1303
|
var StoreScoresLance = class extends storage.ScoresStorage {
|
|
1235
1304
|
client;
|
|
1236
|
-
|
|
1305
|
+
#db;
|
|
1306
|
+
constructor(config) {
|
|
1237
1307
|
super();
|
|
1308
|
+
const client = resolveLanceConfig(config);
|
|
1238
1309
|
this.client = client;
|
|
1310
|
+
this.#db = new LanceDB({ client });
|
|
1311
|
+
}
|
|
1312
|
+
async init() {
|
|
1313
|
+
await this.#db.createTable({ tableName: storage.TABLE_SCORERS, schema: storage.SCORERS_SCHEMA });
|
|
1314
|
+
await this.#db.alterTable({
|
|
1315
|
+
tableName: storage.TABLE_SCORERS,
|
|
1316
|
+
schema: storage.SCORERS_SCHEMA,
|
|
1317
|
+
ifNotExists: ["spanId", "requestContext"]
|
|
1318
|
+
});
|
|
1319
|
+
}
|
|
1320
|
+
async dangerouslyClearAll() {
|
|
1321
|
+
await this.#db.clearTable({ tableName: storage.TABLE_SCORERS });
|
|
1239
1322
|
}
|
|
1240
1323
|
async saveScore(score) {
|
|
1241
1324
|
let validatedScore;
|
|
@@ -1244,37 +1327,47 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1244
1327
|
} catch (error$1) {
|
|
1245
1328
|
throw new error.MastraError(
|
|
1246
1329
|
{
|
|
1247
|
-
id: "
|
|
1330
|
+
id: storage.createStorageErrorId("LANCE", "SAVE_SCORE", "VALIDATION_FAILED"),
|
|
1248
1331
|
text: "Failed to save score in LanceStorage",
|
|
1249
1332
|
domain: error.ErrorDomain.STORAGE,
|
|
1250
|
-
category: error.ErrorCategory.
|
|
1333
|
+
category: error.ErrorCategory.USER,
|
|
1334
|
+
details: {
|
|
1335
|
+
scorer: typeof score.scorer?.id === "string" ? score.scorer.id : String(score.scorer?.id ?? "unknown"),
|
|
1336
|
+
entityId: score.entityId ?? "unknown",
|
|
1337
|
+
entityType: score.entityType ?? "unknown",
|
|
1338
|
+
traceId: score.traceId ?? "",
|
|
1339
|
+
spanId: score.spanId ?? ""
|
|
1340
|
+
}
|
|
1251
1341
|
},
|
|
1252
1342
|
error$1
|
|
1253
1343
|
);
|
|
1254
1344
|
}
|
|
1345
|
+
const id = crypto.randomUUID();
|
|
1346
|
+
const now = /* @__PURE__ */ new Date();
|
|
1255
1347
|
try {
|
|
1256
|
-
const id = crypto.randomUUID();
|
|
1257
1348
|
const table = await this.client.openTable(storage.TABLE_SCORERS);
|
|
1258
1349
|
const schema = await getTableSchema({ tableName: storage.TABLE_SCORERS, client: this.client });
|
|
1259
1350
|
const allowedFields = new Set(schema.fields.map((f) => f.name));
|
|
1260
1351
|
const filteredScore = {};
|
|
1261
|
-
Object.keys(validatedScore)
|
|
1352
|
+
for (const key of Object.keys(validatedScore)) {
|
|
1262
1353
|
if (allowedFields.has(key)) {
|
|
1263
|
-
filteredScore[key] =
|
|
1354
|
+
filteredScore[key] = validatedScore[key];
|
|
1264
1355
|
}
|
|
1265
|
-
}
|
|
1356
|
+
}
|
|
1266
1357
|
for (const key in filteredScore) {
|
|
1267
1358
|
if (filteredScore[key] !== null && typeof filteredScore[key] === "object" && !(filteredScore[key] instanceof Date)) {
|
|
1268
1359
|
filteredScore[key] = JSON.stringify(filteredScore[key]);
|
|
1269
1360
|
}
|
|
1270
1361
|
}
|
|
1271
1362
|
filteredScore.id = id;
|
|
1363
|
+
filteredScore.createdAt = now;
|
|
1364
|
+
filteredScore.updatedAt = now;
|
|
1272
1365
|
await table.add([filteredScore], { mode: "append" });
|
|
1273
|
-
return { score };
|
|
1366
|
+
return { score: { ...validatedScore, id, createdAt: now, updatedAt: now } };
|
|
1274
1367
|
} catch (error$1) {
|
|
1275
1368
|
throw new error.MastraError(
|
|
1276
1369
|
{
|
|
1277
|
-
id: "
|
|
1370
|
+
id: storage.createStorageErrorId("LANCE", "SAVE_SCORE", "FAILED"),
|
|
1278
1371
|
text: "Failed to save score in LanceStorage",
|
|
1279
1372
|
domain: error.ErrorDomain.STORAGE,
|
|
1280
1373
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
@@ -1290,12 +1383,11 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1290
1383
|
const query = table.query().where(`id = '${id}'`).limit(1);
|
|
1291
1384
|
const records = await query.toArray();
|
|
1292
1385
|
if (records.length === 0) return null;
|
|
1293
|
-
|
|
1294
|
-
return processResultWithTypeConversion(records[0], schema);
|
|
1386
|
+
return await this.transformScoreRow(records[0]);
|
|
1295
1387
|
} catch (error$1) {
|
|
1296
1388
|
throw new error.MastraError(
|
|
1297
1389
|
{
|
|
1298
|
-
id: "
|
|
1390
|
+
id: storage.createStorageErrorId("LANCE", "GET_SCORE_BY_ID", "FAILED"),
|
|
1299
1391
|
text: "Failed to get score by id in LanceStorage",
|
|
1300
1392
|
domain: error.ErrorDomain.STORAGE,
|
|
1301
1393
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
@@ -1305,6 +1397,22 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1305
1397
|
);
|
|
1306
1398
|
}
|
|
1307
1399
|
}
|
|
1400
|
+
/**
|
|
1401
|
+
* LanceDB-specific score row transformation.
|
|
1402
|
+
*
|
|
1403
|
+
* Note: This implementation does NOT use coreTransformScoreRow because:
|
|
1404
|
+
* 1. LanceDB stores schema information in the table itself (requires async fetch)
|
|
1405
|
+
* 2. Uses processResultWithTypeConversion utility for LanceDB-specific type handling
|
|
1406
|
+
*/
|
|
1407
|
+
async transformScoreRow(row) {
|
|
1408
|
+
const schema = await getTableSchema({ tableName: storage.TABLE_SCORERS, client: this.client });
|
|
1409
|
+
const transformed = processResultWithTypeConversion(row, schema);
|
|
1410
|
+
return {
|
|
1411
|
+
...transformed,
|
|
1412
|
+
createdAt: row.createdAt,
|
|
1413
|
+
updatedAt: row.updatedAt
|
|
1414
|
+
};
|
|
1415
|
+
}
|
|
1308
1416
|
async listScoresByScorerId({
|
|
1309
1417
|
scorerId,
|
|
1310
1418
|
pagination,
|
|
@@ -1345,8 +1453,7 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1345
1453
|
if (start > 0) query = query.offset(start);
|
|
1346
1454
|
}
|
|
1347
1455
|
const records = await query.toArray();
|
|
1348
|
-
const
|
|
1349
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1456
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1350
1457
|
return {
|
|
1351
1458
|
pagination: {
|
|
1352
1459
|
page,
|
|
@@ -1359,7 +1466,7 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1359
1466
|
} catch (error$1) {
|
|
1360
1467
|
throw new error.MastraError(
|
|
1361
1468
|
{
|
|
1362
|
-
id: "
|
|
1469
|
+
id: storage.createStorageErrorId("LANCE", "LIST_SCORES_BY_SCORER_ID", "FAILED"),
|
|
1363
1470
|
text: "Failed to get scores by scorerId in LanceStorage",
|
|
1364
1471
|
domain: error.ErrorDomain.STORAGE,
|
|
1365
1472
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
@@ -1387,8 +1494,7 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1387
1494
|
if (start > 0) query = query.offset(start);
|
|
1388
1495
|
}
|
|
1389
1496
|
const records = await query.toArray();
|
|
1390
|
-
const
|
|
1391
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1497
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1392
1498
|
return {
|
|
1393
1499
|
pagination: {
|
|
1394
1500
|
page,
|
|
@@ -1401,7 +1507,7 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1401
1507
|
} catch (error$1) {
|
|
1402
1508
|
throw new error.MastraError(
|
|
1403
1509
|
{
|
|
1404
|
-
id: "
|
|
1510
|
+
id: storage.createStorageErrorId("LANCE", "LIST_SCORES_BY_RUN_ID", "FAILED"),
|
|
1405
1511
|
text: "Failed to get scores by runId in LanceStorage",
|
|
1406
1512
|
domain: error.ErrorDomain.STORAGE,
|
|
1407
1513
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
@@ -1430,8 +1536,7 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1430
1536
|
if (start > 0) query = query.offset(start);
|
|
1431
1537
|
}
|
|
1432
1538
|
const records = await query.toArray();
|
|
1433
|
-
const
|
|
1434
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1539
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1435
1540
|
return {
|
|
1436
1541
|
pagination: {
|
|
1437
1542
|
page,
|
|
@@ -1444,7 +1549,7 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1444
1549
|
} catch (error$1) {
|
|
1445
1550
|
throw new error.MastraError(
|
|
1446
1551
|
{
|
|
1447
|
-
id: "
|
|
1552
|
+
id: storage.createStorageErrorId("LANCE", "LIST_SCORES_BY_ENTITY_ID", "FAILED"),
|
|
1448
1553
|
text: "Failed to get scores by entityId and entityType in LanceStorage",
|
|
1449
1554
|
domain: error.ErrorDomain.STORAGE,
|
|
1450
1555
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
@@ -1473,8 +1578,7 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1473
1578
|
if (start > 0) query = query.offset(start);
|
|
1474
1579
|
}
|
|
1475
1580
|
const records = await query.toArray();
|
|
1476
|
-
const
|
|
1477
|
-
const scores = processResultWithTypeConversion(records, schema);
|
|
1581
|
+
const scores = await Promise.all(records.map(async (record) => await this.transformScoreRow(record)));
|
|
1478
1582
|
return {
|
|
1479
1583
|
pagination: {
|
|
1480
1584
|
page,
|
|
@@ -1487,7 +1591,7 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1487
1591
|
} catch (error$1) {
|
|
1488
1592
|
throw new error.MastraError(
|
|
1489
1593
|
{
|
|
1490
|
-
id: "
|
|
1594
|
+
id: storage.createStorageErrorId("LANCE", "LIST_SCORES_BY_SPAN", "FAILED"),
|
|
1491
1595
|
text: "Failed to get scores by traceId and spanId in LanceStorage",
|
|
1492
1596
|
domain: error.ErrorDomain.STORAGE,
|
|
1493
1597
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
@@ -1498,62 +1602,111 @@ var StoreScoresLance = class extends storage.ScoresStorage {
|
|
|
1498
1602
|
}
|
|
1499
1603
|
}
|
|
1500
1604
|
};
|
|
1501
|
-
function
|
|
1502
|
-
|
|
1503
|
-
if (typeof parsedSnapshot === "string") {
|
|
1504
|
-
try {
|
|
1505
|
-
parsedSnapshot = JSON.parse(row.snapshot);
|
|
1506
|
-
} catch (e) {
|
|
1507
|
-
console.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
|
|
1508
|
-
}
|
|
1509
|
-
}
|
|
1510
|
-
return {
|
|
1511
|
-
workflowName: row.workflow_name,
|
|
1512
|
-
runId: row.run_id,
|
|
1513
|
-
snapshot: parsedSnapshot,
|
|
1514
|
-
createdAt: storage.ensureDate(row.createdAt),
|
|
1515
|
-
updatedAt: storage.ensureDate(row.updatedAt),
|
|
1516
|
-
resourceId: row.resourceId
|
|
1517
|
-
};
|
|
1605
|
+
function escapeSql(str) {
|
|
1606
|
+
return str.replace(/'/g, "''");
|
|
1518
1607
|
}
|
|
1519
1608
|
var StoreWorkflowsLance = class extends storage.WorkflowsStorage {
|
|
1520
1609
|
client;
|
|
1521
|
-
|
|
1610
|
+
#db;
|
|
1611
|
+
constructor(config) {
|
|
1522
1612
|
super();
|
|
1613
|
+
const client = resolveLanceConfig(config);
|
|
1523
1614
|
this.client = client;
|
|
1615
|
+
this.#db = new LanceDB({ client });
|
|
1524
1616
|
}
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1617
|
+
parseWorkflowRun(row) {
|
|
1618
|
+
let parsedSnapshot = row.snapshot;
|
|
1619
|
+
if (typeof parsedSnapshot === "string") {
|
|
1620
|
+
try {
|
|
1621
|
+
parsedSnapshot = JSON.parse(row.snapshot);
|
|
1622
|
+
} catch (e) {
|
|
1623
|
+
this.logger.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
return {
|
|
1627
|
+
workflowName: row.workflow_name,
|
|
1628
|
+
runId: row.run_id,
|
|
1629
|
+
snapshot: parsedSnapshot,
|
|
1630
|
+
createdAt: storage.ensureDate(row.createdAt),
|
|
1631
|
+
updatedAt: storage.ensureDate(row.updatedAt),
|
|
1632
|
+
resourceId: row.resourceId
|
|
1633
|
+
};
|
|
1634
|
+
}
|
|
1635
|
+
async init() {
|
|
1636
|
+
const schema = storage.TABLE_SCHEMAS[storage.TABLE_WORKFLOW_SNAPSHOT];
|
|
1637
|
+
await this.#db.createTable({ tableName: storage.TABLE_WORKFLOW_SNAPSHOT, schema });
|
|
1638
|
+
await this.#db.alterTable({
|
|
1639
|
+
tableName: storage.TABLE_WORKFLOW_SNAPSHOT,
|
|
1640
|
+
schema,
|
|
1641
|
+
ifNotExists: ["resourceId"]
|
|
1642
|
+
});
|
|
1643
|
+
}
|
|
1644
|
+
async dangerouslyClearAll() {
|
|
1645
|
+
await this.#db.clearTable({ tableName: storage.TABLE_WORKFLOW_SNAPSHOT });
|
|
1646
|
+
}
|
|
1647
|
+
async updateWorkflowResults({
|
|
1648
|
+
workflowName,
|
|
1649
|
+
runId,
|
|
1650
|
+
stepId,
|
|
1651
|
+
result,
|
|
1652
|
+
requestContext
|
|
1531
1653
|
}) {
|
|
1532
|
-
|
|
1654
|
+
let snapshot = await this.loadWorkflowSnapshot({ workflowName, runId });
|
|
1655
|
+
if (!snapshot) {
|
|
1656
|
+
snapshot = {
|
|
1657
|
+
context: {},
|
|
1658
|
+
activePaths: [],
|
|
1659
|
+
timestamp: Date.now(),
|
|
1660
|
+
suspendedPaths: {},
|
|
1661
|
+
activeStepsPath: {},
|
|
1662
|
+
resumeLabels: {},
|
|
1663
|
+
serializedStepGraph: [],
|
|
1664
|
+
status: "pending",
|
|
1665
|
+
value: {},
|
|
1666
|
+
waitingPaths: {},
|
|
1667
|
+
runId,
|
|
1668
|
+
requestContext: {}
|
|
1669
|
+
};
|
|
1670
|
+
}
|
|
1671
|
+
snapshot.context[stepId] = result;
|
|
1672
|
+
snapshot.requestContext = { ...snapshot.requestContext, ...requestContext };
|
|
1673
|
+
await this.persistWorkflowSnapshot({ workflowName, runId, snapshot });
|
|
1674
|
+
return snapshot.context;
|
|
1533
1675
|
}
|
|
1534
|
-
updateWorkflowState({
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1676
|
+
async updateWorkflowState({
|
|
1677
|
+
workflowName,
|
|
1678
|
+
runId,
|
|
1679
|
+
opts
|
|
1538
1680
|
}) {
|
|
1539
|
-
|
|
1681
|
+
const snapshot = await this.loadWorkflowSnapshot({ workflowName, runId });
|
|
1682
|
+
if (!snapshot) {
|
|
1683
|
+
return void 0;
|
|
1684
|
+
}
|
|
1685
|
+
if (!snapshot.context) {
|
|
1686
|
+
throw new Error(`Snapshot not found for runId ${runId}`);
|
|
1687
|
+
}
|
|
1688
|
+
const updatedSnapshot = { ...snapshot, ...opts };
|
|
1689
|
+
await this.persistWorkflowSnapshot({ workflowName, runId, snapshot: updatedSnapshot });
|
|
1690
|
+
return updatedSnapshot;
|
|
1540
1691
|
}
|
|
1541
1692
|
async persistWorkflowSnapshot({
|
|
1542
1693
|
workflowName,
|
|
1543
1694
|
runId,
|
|
1544
1695
|
resourceId,
|
|
1545
|
-
snapshot
|
|
1696
|
+
snapshot,
|
|
1697
|
+
createdAt,
|
|
1698
|
+
updatedAt
|
|
1546
1699
|
}) {
|
|
1547
1700
|
try {
|
|
1548
1701
|
const table = await this.client.openTable(storage.TABLE_WORKFLOW_SNAPSHOT);
|
|
1549
|
-
const query = table.query().where(`workflow_name = '${workflowName}' AND run_id = '${runId}'`);
|
|
1702
|
+
const query = table.query().where(`workflow_name = '${escapeSql(workflowName)}' AND run_id = '${escapeSql(runId)}'`);
|
|
1550
1703
|
const records = await query.toArray();
|
|
1551
|
-
let
|
|
1552
|
-
const now = Date.now();
|
|
1704
|
+
let createdAtValue;
|
|
1705
|
+
const now = createdAt?.getTime() ?? Date.now();
|
|
1553
1706
|
if (records.length > 0) {
|
|
1554
|
-
|
|
1707
|
+
createdAtValue = records[0].createdAt ?? now;
|
|
1555
1708
|
} else {
|
|
1556
|
-
|
|
1709
|
+
createdAtValue = now;
|
|
1557
1710
|
}
|
|
1558
1711
|
const { status, value, ...rest } = snapshot;
|
|
1559
1712
|
const record = {
|
|
@@ -1562,14 +1715,14 @@ var StoreWorkflowsLance = class extends storage.WorkflowsStorage {
|
|
|
1562
1715
|
resourceId,
|
|
1563
1716
|
snapshot: JSON.stringify({ status, value, ...rest }),
|
|
1564
1717
|
// this is to ensure status is always just before value, for when querying the db by status
|
|
1565
|
-
createdAt,
|
|
1566
|
-
updatedAt: now
|
|
1718
|
+
createdAt: createdAtValue,
|
|
1719
|
+
updatedAt: updatedAt ?? now
|
|
1567
1720
|
};
|
|
1568
1721
|
await table.mergeInsert(["workflow_name", "run_id"]).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
1569
1722
|
} catch (error$1) {
|
|
1570
1723
|
throw new error.MastraError(
|
|
1571
1724
|
{
|
|
1572
|
-
id: "
|
|
1725
|
+
id: storage.createStorageErrorId("LANCE", "PERSIST_WORKFLOW_SNAPSHOT", "FAILED"),
|
|
1573
1726
|
domain: error.ErrorDomain.STORAGE,
|
|
1574
1727
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
1575
1728
|
details: { workflowName, runId }
|
|
@@ -1584,13 +1737,13 @@ var StoreWorkflowsLance = class extends storage.WorkflowsStorage {
|
|
|
1584
1737
|
}) {
|
|
1585
1738
|
try {
|
|
1586
1739
|
const table = await this.client.openTable(storage.TABLE_WORKFLOW_SNAPSHOT);
|
|
1587
|
-
const query = table.query().where(`workflow_name = '${workflowName}' AND run_id = '${runId}'`);
|
|
1740
|
+
const query = table.query().where(`workflow_name = '${escapeSql(workflowName)}' AND run_id = '${escapeSql(runId)}'`);
|
|
1588
1741
|
const records = await query.toArray();
|
|
1589
1742
|
return records.length > 0 ? JSON.parse(records[0].snapshot) : null;
|
|
1590
1743
|
} catch (error$1) {
|
|
1591
1744
|
throw new error.MastraError(
|
|
1592
1745
|
{
|
|
1593
|
-
id: "
|
|
1746
|
+
id: storage.createStorageErrorId("LANCE", "LOAD_WORKFLOW_SNAPSHOT", "FAILED"),
|
|
1594
1747
|
domain: error.ErrorDomain.STORAGE,
|
|
1595
1748
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
1596
1749
|
details: { workflowName, runId }
|
|
@@ -1602,19 +1755,19 @@ var StoreWorkflowsLance = class extends storage.WorkflowsStorage {
|
|
|
1602
1755
|
async getWorkflowRunById(args) {
|
|
1603
1756
|
try {
|
|
1604
1757
|
const table = await this.client.openTable(storage.TABLE_WORKFLOW_SNAPSHOT);
|
|
1605
|
-
let whereClause = `run_id = '${args.runId}'`;
|
|
1758
|
+
let whereClause = `run_id = '${escapeSql(args.runId)}'`;
|
|
1606
1759
|
if (args.workflowName) {
|
|
1607
|
-
whereClause += ` AND workflow_name = '${args.workflowName}'`;
|
|
1760
|
+
whereClause += ` AND workflow_name = '${escapeSql(args.workflowName)}'`;
|
|
1608
1761
|
}
|
|
1609
1762
|
const query = table.query().where(whereClause);
|
|
1610
1763
|
const records = await query.toArray();
|
|
1611
1764
|
if (records.length === 0) return null;
|
|
1612
1765
|
const record = records[0];
|
|
1613
|
-
return parseWorkflowRun(record);
|
|
1766
|
+
return this.parseWorkflowRun(record);
|
|
1614
1767
|
} catch (error$1) {
|
|
1615
1768
|
throw new error.MastraError(
|
|
1616
1769
|
{
|
|
1617
|
-
id: "
|
|
1770
|
+
id: storage.createStorageErrorId("LANCE", "GET_WORKFLOW_RUN_BY_ID", "FAILED"),
|
|
1618
1771
|
domain: error.ErrorDomain.STORAGE,
|
|
1619
1772
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
1620
1773
|
details: { runId: args.runId, workflowName: args.workflowName ?? "" }
|
|
@@ -1623,336 +1776,182 @@ var StoreWorkflowsLance = class extends storage.WorkflowsStorage {
|
|
|
1623
1776
|
);
|
|
1624
1777
|
}
|
|
1625
1778
|
}
|
|
1779
|
+
async deleteWorkflowRunById({ runId, workflowName }) {
|
|
1780
|
+
try {
|
|
1781
|
+
const table = await this.client.openTable(storage.TABLE_WORKFLOW_SNAPSHOT);
|
|
1782
|
+
const whereClause = `run_id = '${escapeSql(runId)}' AND workflow_name = '${escapeSql(workflowName)}'`;
|
|
1783
|
+
await table.delete(whereClause);
|
|
1784
|
+
} catch (error$1) {
|
|
1785
|
+
throw new error.MastraError(
|
|
1786
|
+
{
|
|
1787
|
+
id: storage.createStorageErrorId("LANCE", "DELETE_WORKFLOW_RUN_BY_ID", "FAILED"),
|
|
1788
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1789
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1790
|
+
details: { runId, workflowName }
|
|
1791
|
+
},
|
|
1792
|
+
error$1
|
|
1793
|
+
);
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1626
1796
|
async listWorkflowRuns(args) {
|
|
1627
1797
|
try {
|
|
1628
1798
|
const table = await this.client.openTable(storage.TABLE_WORKFLOW_SNAPSHOT);
|
|
1629
1799
|
let query = table.query();
|
|
1630
1800
|
const conditions = [];
|
|
1631
1801
|
if (args?.workflowName) {
|
|
1632
|
-
conditions.push(`workflow_name = '${args.workflowName
|
|
1802
|
+
conditions.push(`workflow_name = '${escapeSql(args.workflowName)}'`);
|
|
1633
1803
|
}
|
|
1634
1804
|
if (args?.status) {
|
|
1635
1805
|
const escapedStatus = args.status.replace(/\\/g, "\\\\").replace(/'/g, "''").replace(/%/g, "\\%").replace(/_/g, "\\_");
|
|
1636
1806
|
conditions.push(`\`snapshot\` LIKE '%"status":"${escapedStatus}","value"%'`);
|
|
1637
1807
|
}
|
|
1638
1808
|
if (args?.resourceId) {
|
|
1639
|
-
conditions.push(`\`resourceId\` = '${args.resourceId}'`);
|
|
1809
|
+
conditions.push(`\`resourceId\` = '${escapeSql(args.resourceId)}'`);
|
|
1640
1810
|
}
|
|
1641
1811
|
if (args?.fromDate instanceof Date) {
|
|
1642
1812
|
conditions.push(`\`createdAt\` >= ${args.fromDate.getTime()}`);
|
|
1643
1813
|
}
|
|
1644
1814
|
if (args?.toDate instanceof Date) {
|
|
1645
|
-
conditions.push(`\`createdAt\` <= ${args.toDate.getTime()}`);
|
|
1646
|
-
}
|
|
1647
|
-
let total = 0;
|
|
1648
|
-
if (conditions.length > 0) {
|
|
1649
|
-
query = query.where(conditions.join(" AND "));
|
|
1650
|
-
total = await table.countRows(conditions.join(" AND "));
|
|
1651
|
-
} else {
|
|
1652
|
-
total = await table.countRows();
|
|
1653
|
-
}
|
|
1654
|
-
if (args?.perPage !== void 0 && args?.page !== void 0) {
|
|
1655
|
-
const normalizedPerPage = storage.normalizePerPage(args.perPage, Number.MAX_SAFE_INTEGER);
|
|
1656
|
-
if (args.page < 0 || !Number.isInteger(args.page)) {
|
|
1657
|
-
throw new error.MastraError(
|
|
1658
|
-
{
|
|
1659
|
-
id: "
|
|
1660
|
-
domain: error.ErrorDomain.STORAGE,
|
|
1661
|
-
category: error.ErrorCategory.USER,
|
|
1662
|
-
details: { page: args.page, perPage: args.perPage }
|
|
1663
|
-
},
|
|
1664
|
-
new Error(`Invalid pagination parameters: page=${args.page}, perPage=${args.perPage}`)
|
|
1665
|
-
);
|
|
1666
|
-
}
|
|
1667
|
-
const offset = args.page * normalizedPerPage;
|
|
1668
|
-
query.limit(normalizedPerPage);
|
|
1669
|
-
query.offset(offset);
|
|
1670
|
-
}
|
|
1671
|
-
const records = await query.toArray();
|
|
1672
|
-
return {
|
|
1673
|
-
runs: records.map((record) => parseWorkflowRun(record)),
|
|
1674
|
-
total: total || records.length
|
|
1675
|
-
};
|
|
1676
|
-
} catch (error$1) {
|
|
1677
|
-
throw new error.MastraError(
|
|
1678
|
-
{
|
|
1679
|
-
id: "
|
|
1680
|
-
domain: error.ErrorDomain.STORAGE,
|
|
1681
|
-
category: error.ErrorCategory.THIRD_PARTY,
|
|
1682
|
-
details: { resourceId: args?.resourceId ?? "", workflowName: args?.workflowName ?? "" }
|
|
1683
|
-
},
|
|
1684
|
-
error$1
|
|
1685
|
-
);
|
|
1686
|
-
}
|
|
1687
|
-
}
|
|
1688
|
-
};
|
|
1689
|
-
|
|
1690
|
-
// src/storage/index.ts
|
|
1691
|
-
var LanceStorage = class _LanceStorage extends storage.MastraStorage {
|
|
1692
|
-
stores;
|
|
1693
|
-
lanceClient;
|
|
1694
|
-
/**
|
|
1695
|
-
* Creates a new instance of LanceStorage
|
|
1696
|
-
* @param id The unique identifier for this storage instance
|
|
1697
|
-
* @param name The name for this storage instance
|
|
1698
|
-
* @param uri The URI to connect to LanceDB
|
|
1699
|
-
* @param options connection options
|
|
1700
|
-
*
|
|
1701
|
-
* Usage:
|
|
1702
|
-
*
|
|
1703
|
-
* Connect to a local database
|
|
1704
|
-
* ```ts
|
|
1705
|
-
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', '/path/to/db');
|
|
1706
|
-
* ```
|
|
1707
|
-
*
|
|
1708
|
-
* Connect to a LanceDB cloud database
|
|
1709
|
-
* ```ts
|
|
1710
|
-
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 'db://host:port');
|
|
1711
|
-
* ```
|
|
1712
|
-
*
|
|
1713
|
-
* Connect to a cloud database
|
|
1714
|
-
* ```ts
|
|
1715
|
-
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 's3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
1716
|
-
* ```
|
|
1717
|
-
*/
|
|
1718
|
-
static async create(id, name, uri, options) {
|
|
1719
|
-
const instance = new _LanceStorage(id, name);
|
|
1720
|
-
try {
|
|
1721
|
-
instance.lanceClient = await lancedb.connect(uri, options);
|
|
1722
|
-
const operations = new StoreOperationsLance({ client: instance.lanceClient });
|
|
1723
|
-
instance.stores = {
|
|
1724
|
-
operations: new StoreOperationsLance({ client: instance.lanceClient }),
|
|
1725
|
-
workflows: new StoreWorkflowsLance({ client: instance.lanceClient }),
|
|
1726
|
-
scores: new StoreScoresLance({ client: instance.lanceClient }),
|
|
1727
|
-
memory: new StoreMemoryLance({ client: instance.lanceClient, operations })
|
|
1728
|
-
};
|
|
1729
|
-
return instance;
|
|
1730
|
-
} catch (e) {
|
|
1731
|
-
throw new error.MastraError(
|
|
1732
|
-
{
|
|
1733
|
-
id: "STORAGE_LANCE_STORAGE_CONNECT_FAILED",
|
|
1734
|
-
domain: error.ErrorDomain.STORAGE,
|
|
1735
|
-
category: error.ErrorCategory.THIRD_PARTY,
|
|
1736
|
-
text: `Failed to connect to LanceDB: ${e.message || e}`,
|
|
1737
|
-
details: { uri, optionsProvided: !!options }
|
|
1738
|
-
},
|
|
1739
|
-
e
|
|
1740
|
-
);
|
|
1741
|
-
}
|
|
1742
|
-
}
|
|
1743
|
-
/**
|
|
1744
|
-
* @internal
|
|
1745
|
-
* Private constructor to enforce using the create factory method
|
|
1746
|
-
*/
|
|
1747
|
-
constructor(id, name) {
|
|
1748
|
-
super({ id, name });
|
|
1749
|
-
const operations = new StoreOperationsLance({ client: this.lanceClient });
|
|
1750
|
-
this.stores = {
|
|
1751
|
-
operations: new StoreOperationsLance({ client: this.lanceClient }),
|
|
1752
|
-
workflows: new StoreWorkflowsLance({ client: this.lanceClient }),
|
|
1753
|
-
scores: new StoreScoresLance({ client: this.lanceClient }),
|
|
1754
|
-
memory: new StoreMemoryLance({ client: this.lanceClient, operations })
|
|
1755
|
-
};
|
|
1756
|
-
}
|
|
1757
|
-
async createTable({
|
|
1758
|
-
tableName,
|
|
1759
|
-
schema
|
|
1760
|
-
}) {
|
|
1761
|
-
return this.stores.operations.createTable({ tableName, schema });
|
|
1762
|
-
}
|
|
1763
|
-
async dropTable({ tableName }) {
|
|
1764
|
-
return this.stores.operations.dropTable({ tableName });
|
|
1765
|
-
}
|
|
1766
|
-
async alterTable({
|
|
1767
|
-
tableName,
|
|
1768
|
-
schema,
|
|
1769
|
-
ifNotExists
|
|
1770
|
-
}) {
|
|
1771
|
-
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
1772
|
-
}
|
|
1773
|
-
async clearTable({ tableName }) {
|
|
1774
|
-
return this.stores.operations.clearTable({ tableName });
|
|
1775
|
-
}
|
|
1776
|
-
async insert({ tableName, record }) {
|
|
1777
|
-
return this.stores.operations.insert({ tableName, record });
|
|
1778
|
-
}
|
|
1779
|
-
async batchInsert({ tableName, records }) {
|
|
1780
|
-
return this.stores.operations.batchInsert({ tableName, records });
|
|
1781
|
-
}
|
|
1782
|
-
async load({ tableName, keys }) {
|
|
1783
|
-
return this.stores.operations.load({ tableName, keys });
|
|
1784
|
-
}
|
|
1785
|
-
async getThreadById({ threadId }) {
|
|
1786
|
-
return this.stores.memory.getThreadById({ threadId });
|
|
1787
|
-
}
|
|
1788
|
-
/**
|
|
1789
|
-
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
1790
|
-
* @param thread - The thread to save
|
|
1791
|
-
* @returns The saved thread
|
|
1792
|
-
*/
|
|
1793
|
-
async saveThread({ thread }) {
|
|
1794
|
-
return this.stores.memory.saveThread({ thread });
|
|
1795
|
-
}
|
|
1796
|
-
async updateThread({
|
|
1797
|
-
id,
|
|
1798
|
-
title,
|
|
1799
|
-
metadata
|
|
1800
|
-
}) {
|
|
1801
|
-
return this.stores.memory.updateThread({ id, title, metadata });
|
|
1802
|
-
}
|
|
1803
|
-
async deleteThread({ threadId }) {
|
|
1804
|
-
return this.stores.memory.deleteThread({ threadId });
|
|
1805
|
-
}
|
|
1806
|
-
get supports() {
|
|
1807
|
-
return {
|
|
1808
|
-
selectByIncludeResourceScope: true,
|
|
1809
|
-
resourceWorkingMemory: true,
|
|
1810
|
-
hasColumn: true,
|
|
1811
|
-
createTable: true,
|
|
1812
|
-
deleteMessages: false,
|
|
1813
|
-
listScoresBySpan: true
|
|
1814
|
-
};
|
|
1815
|
-
}
|
|
1816
|
-
async getResourceById({ resourceId }) {
|
|
1817
|
-
return this.stores.memory.getResourceById({ resourceId });
|
|
1818
|
-
}
|
|
1819
|
-
async saveResource({ resource }) {
|
|
1820
|
-
return this.stores.memory.saveResource({ resource });
|
|
1821
|
-
}
|
|
1822
|
-
async updateResource({
|
|
1823
|
-
resourceId,
|
|
1824
|
-
workingMemory,
|
|
1825
|
-
metadata
|
|
1826
|
-
}) {
|
|
1827
|
-
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
1828
|
-
}
|
|
1829
|
-
/**
|
|
1830
|
-
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
1831
|
-
* @param records - The sorted array of records to process
|
|
1832
|
-
* @param include - The array of include specifications with context parameters
|
|
1833
|
-
* @returns The processed array with context messages included
|
|
1834
|
-
*/
|
|
1835
|
-
processMessagesWithContext(records, include) {
|
|
1836
|
-
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
1837
|
-
if (messagesWithContext.length === 0) {
|
|
1838
|
-
return records;
|
|
1839
|
-
}
|
|
1840
|
-
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
1841
|
-
records.forEach((message, index) => {
|
|
1842
|
-
messageIndexMap.set(message.id, index);
|
|
1843
|
-
});
|
|
1844
|
-
const additionalIndices = /* @__PURE__ */ new Set();
|
|
1845
|
-
for (const item of messagesWithContext) {
|
|
1846
|
-
const messageIndex = messageIndexMap.get(item.id);
|
|
1847
|
-
if (messageIndex !== void 0) {
|
|
1848
|
-
if (item.withPreviousMessages) {
|
|
1849
|
-
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
1850
|
-
for (let i = startIdx; i < messageIndex; i++) {
|
|
1851
|
-
additionalIndices.add(i);
|
|
1852
|
-
}
|
|
1853
|
-
}
|
|
1854
|
-
if (item.withNextMessages) {
|
|
1855
|
-
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
1856
|
-
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
1857
|
-
additionalIndices.add(i);
|
|
1858
|
-
}
|
|
1859
|
-
}
|
|
1860
|
-
}
|
|
1861
|
-
}
|
|
1862
|
-
if (additionalIndices.size === 0) {
|
|
1863
|
-
return records;
|
|
1864
|
-
}
|
|
1865
|
-
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
1866
|
-
const allIndices = /* @__PURE__ */ new Set();
|
|
1867
|
-
records.forEach((record, index) => {
|
|
1868
|
-
if (originalMatchIds.has(record.id)) {
|
|
1869
|
-
allIndices.add(index);
|
|
1870
|
-
}
|
|
1871
|
-
});
|
|
1872
|
-
additionalIndices.forEach((index) => {
|
|
1873
|
-
allIndices.add(index);
|
|
1874
|
-
});
|
|
1875
|
-
return Array.from(allIndices).sort((a, b) => a - b).map((index) => records[index]);
|
|
1876
|
-
}
|
|
1877
|
-
async listMessagesById({ messageIds }) {
|
|
1878
|
-
return this.stores.memory.listMessagesById({ messageIds });
|
|
1879
|
-
}
|
|
1880
|
-
async saveMessages(args) {
|
|
1881
|
-
return this.stores.memory.saveMessages(args);
|
|
1882
|
-
}
|
|
1883
|
-
async updateMessages(_args) {
|
|
1884
|
-
return this.stores.memory.updateMessages(_args);
|
|
1885
|
-
}
|
|
1886
|
-
async listWorkflowRuns(args) {
|
|
1887
|
-
return this.stores.workflows.listWorkflowRuns(args);
|
|
1888
|
-
}
|
|
1889
|
-
async getWorkflowRunById(args) {
|
|
1890
|
-
return this.stores.workflows.getWorkflowRunById(args);
|
|
1891
|
-
}
|
|
1892
|
-
async updateWorkflowResults({
|
|
1893
|
-
workflowName,
|
|
1894
|
-
runId,
|
|
1895
|
-
stepId,
|
|
1896
|
-
result,
|
|
1897
|
-
requestContext
|
|
1898
|
-
}) {
|
|
1899
|
-
return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, requestContext });
|
|
1900
|
-
}
|
|
1901
|
-
async updateWorkflowState({
|
|
1902
|
-
workflowName,
|
|
1903
|
-
runId,
|
|
1904
|
-
opts
|
|
1905
|
-
}) {
|
|
1906
|
-
return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
|
|
1907
|
-
}
|
|
1908
|
-
async persistWorkflowSnapshot({
|
|
1909
|
-
workflowName,
|
|
1910
|
-
runId,
|
|
1911
|
-
resourceId,
|
|
1912
|
-
snapshot
|
|
1913
|
-
}) {
|
|
1914
|
-
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot });
|
|
1915
|
-
}
|
|
1916
|
-
async loadWorkflowSnapshot({
|
|
1917
|
-
workflowName,
|
|
1918
|
-
runId
|
|
1919
|
-
}) {
|
|
1920
|
-
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
1921
|
-
}
|
|
1922
|
-
async getScoreById({ id: _id }) {
|
|
1923
|
-
return this.stores.scores.getScoreById({ id: _id });
|
|
1924
|
-
}
|
|
1925
|
-
async listScoresByScorerId({
|
|
1926
|
-
scorerId,
|
|
1927
|
-
source,
|
|
1928
|
-
entityId,
|
|
1929
|
-
entityType,
|
|
1930
|
-
pagination
|
|
1931
|
-
}) {
|
|
1932
|
-
return this.stores.scores.listScoresByScorerId({ scorerId, source, pagination, entityId, entityType });
|
|
1933
|
-
}
|
|
1934
|
-
async saveScore(_score) {
|
|
1935
|
-
return this.stores.scores.saveScore(_score);
|
|
1815
|
+
conditions.push(`\`createdAt\` <= ${args.toDate.getTime()}`);
|
|
1816
|
+
}
|
|
1817
|
+
let total = 0;
|
|
1818
|
+
if (conditions.length > 0) {
|
|
1819
|
+
query = query.where(conditions.join(" AND "));
|
|
1820
|
+
total = await table.countRows(conditions.join(" AND "));
|
|
1821
|
+
} else {
|
|
1822
|
+
total = await table.countRows();
|
|
1823
|
+
}
|
|
1824
|
+
if (args?.perPage !== void 0 && args?.page !== void 0) {
|
|
1825
|
+
const normalizedPerPage = storage.normalizePerPage(args.perPage, Number.MAX_SAFE_INTEGER);
|
|
1826
|
+
if (args.page < 0 || !Number.isInteger(args.page)) {
|
|
1827
|
+
throw new error.MastraError(
|
|
1828
|
+
{
|
|
1829
|
+
id: storage.createStorageErrorId("LANCE", "LIST_WORKFLOW_RUNS", "INVALID_PAGINATION"),
|
|
1830
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1831
|
+
category: error.ErrorCategory.USER,
|
|
1832
|
+
details: { page: args.page, perPage: args.perPage }
|
|
1833
|
+
},
|
|
1834
|
+
new Error(`Invalid pagination parameters: page=${args.page}, perPage=${args.perPage}`)
|
|
1835
|
+
);
|
|
1836
|
+
}
|
|
1837
|
+
const offset = args.page * normalizedPerPage;
|
|
1838
|
+
query.limit(normalizedPerPage);
|
|
1839
|
+
query.offset(offset);
|
|
1840
|
+
}
|
|
1841
|
+
const records = await query.toArray();
|
|
1842
|
+
return {
|
|
1843
|
+
runs: records.map((record) => this.parseWorkflowRun(record)),
|
|
1844
|
+
total: total || records.length
|
|
1845
|
+
};
|
|
1846
|
+
} catch (error$1) {
|
|
1847
|
+
throw new error.MastraError(
|
|
1848
|
+
{
|
|
1849
|
+
id: storage.createStorageErrorId("LANCE", "LIST_WORKFLOW_RUNS", "FAILED"),
|
|
1850
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1851
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1852
|
+
details: { resourceId: args?.resourceId ?? "", workflowName: args?.workflowName ?? "" }
|
|
1853
|
+
},
|
|
1854
|
+
error$1
|
|
1855
|
+
);
|
|
1856
|
+
}
|
|
1936
1857
|
}
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1858
|
+
};
|
|
1859
|
+
|
|
1860
|
+
// src/storage/index.ts
|
|
1861
|
+
var LanceStorage = class _LanceStorage extends storage.MastraStorage {
|
|
1862
|
+
stores;
|
|
1863
|
+
lanceClient;
|
|
1864
|
+
/**
|
|
1865
|
+
* Creates a new instance of LanceStorage
|
|
1866
|
+
* @param id The unique identifier for this storage instance
|
|
1867
|
+
* @param name The name for this storage instance
|
|
1868
|
+
* @param uri The URI to connect to LanceDB
|
|
1869
|
+
* @param connectionOptions connection options for LanceDB
|
|
1870
|
+
* @param storageOptions storage options including disableInit
|
|
1871
|
+
*
|
|
1872
|
+
* Usage:
|
|
1873
|
+
*
|
|
1874
|
+
* Connect to a local database
|
|
1875
|
+
* ```ts
|
|
1876
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', '/path/to/db');
|
|
1877
|
+
* ```
|
|
1878
|
+
*
|
|
1879
|
+
* Connect to a LanceDB cloud database
|
|
1880
|
+
* ```ts
|
|
1881
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 'db://host:port');
|
|
1882
|
+
* ```
|
|
1883
|
+
*
|
|
1884
|
+
* Connect to a cloud database
|
|
1885
|
+
* ```ts
|
|
1886
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', 's3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
1887
|
+
* ```
|
|
1888
|
+
*
|
|
1889
|
+
* Disable auto-init for runtime (after CI/CD has run migrations)
|
|
1890
|
+
* ```ts
|
|
1891
|
+
* const store = await LanceStorage.create('my-storage-id', 'MyStorage', '/path/to/db', undefined, { disableInit: true });
|
|
1892
|
+
* ```
|
|
1893
|
+
*/
|
|
1894
|
+
static async create(id, name, uri, connectionOptions, storageOptions) {
|
|
1895
|
+
const instance = new _LanceStorage(id, name, storageOptions?.disableInit);
|
|
1896
|
+
try {
|
|
1897
|
+
instance.lanceClient = await lancedb.connect(uri, connectionOptions);
|
|
1898
|
+
instance.stores = {
|
|
1899
|
+
workflows: new StoreWorkflowsLance({ client: instance.lanceClient }),
|
|
1900
|
+
scores: new StoreScoresLance({ client: instance.lanceClient }),
|
|
1901
|
+
memory: new StoreMemoryLance({ client: instance.lanceClient })
|
|
1902
|
+
};
|
|
1903
|
+
return instance;
|
|
1904
|
+
} catch (e) {
|
|
1905
|
+
throw new error.MastraError(
|
|
1906
|
+
{
|
|
1907
|
+
id: storage.createStorageErrorId("LANCE", "CONNECT", "FAILED"),
|
|
1908
|
+
domain: error.ErrorDomain.STORAGE,
|
|
1909
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
1910
|
+
text: `Failed to connect to LanceDB: ${e.message || e}`,
|
|
1911
|
+
details: { uri, optionsProvided: !!connectionOptions }
|
|
1912
|
+
},
|
|
1913
|
+
e
|
|
1914
|
+
);
|
|
1915
|
+
}
|
|
1942
1916
|
}
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1917
|
+
/**
|
|
1918
|
+
* Creates a new instance of LanceStorage from a pre-configured LanceDB connection.
|
|
1919
|
+
* Use this when you need to configure the connection before initialization.
|
|
1920
|
+
*
|
|
1921
|
+
* @param id The unique identifier for this storage instance
|
|
1922
|
+
* @param name The name for this storage instance
|
|
1923
|
+
* @param client Pre-configured LanceDB connection
|
|
1924
|
+
* @param options Storage options including disableInit
|
|
1925
|
+
*
|
|
1926
|
+
* @example
|
|
1927
|
+
* ```typescript
|
|
1928
|
+
* import { connect } from '@lancedb/lancedb';
|
|
1929
|
+
*
|
|
1930
|
+
* const client = await connect('/path/to/db', {
|
|
1931
|
+
* // Custom connection options
|
|
1932
|
+
* });
|
|
1933
|
+
*
|
|
1934
|
+
* const store = LanceStorage.fromClient('my-id', 'MyStorage', client);
|
|
1935
|
+
* ```
|
|
1936
|
+
*/
|
|
1937
|
+
static fromClient(id, name, client, options) {
|
|
1938
|
+
const instance = new _LanceStorage(id, name, options?.disableInit);
|
|
1939
|
+
instance.lanceClient = client;
|
|
1940
|
+
instance.stores = {
|
|
1941
|
+
workflows: new StoreWorkflowsLance({ client }),
|
|
1942
|
+
scores: new StoreScoresLance({ client }),
|
|
1943
|
+
memory: new StoreMemoryLance({ client })
|
|
1944
|
+
};
|
|
1945
|
+
return instance;
|
|
1949
1946
|
}
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1947
|
+
/**
|
|
1948
|
+
* @internal
|
|
1949
|
+
* Private constructor to enforce using the create factory method.
|
|
1950
|
+
* Note: stores is initialized in create() after the lanceClient is connected.
|
|
1951
|
+
*/
|
|
1952
|
+
constructor(id, name, disableInit) {
|
|
1953
|
+
super({ id, name, disableInit });
|
|
1954
|
+
this.stores = {};
|
|
1956
1955
|
}
|
|
1957
1956
|
};
|
|
1958
1957
|
var LanceFilterTranslator = class extends filter.BaseFilterTranslator {
|
|
@@ -2308,7 +2307,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2308
2307
|
} catch (e) {
|
|
2309
2308
|
throw new error.MastraError(
|
|
2310
2309
|
{
|
|
2311
|
-
id: "
|
|
2310
|
+
id: storage.createVectorErrorId("LANCE", "CONNECT", "FAILED"),
|
|
2312
2311
|
domain: error.ErrorDomain.STORAGE,
|
|
2313
2312
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2314
2313
|
details: { uri }
|
|
@@ -2331,6 +2330,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2331
2330
|
}
|
|
2332
2331
|
async query({
|
|
2333
2332
|
tableName,
|
|
2333
|
+
indexName,
|
|
2334
2334
|
queryVector,
|
|
2335
2335
|
filter,
|
|
2336
2336
|
includeVector = false,
|
|
@@ -2338,12 +2338,13 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2338
2338
|
columns = [],
|
|
2339
2339
|
includeAllColumns = false
|
|
2340
2340
|
}) {
|
|
2341
|
+
const resolvedTableName = tableName ?? indexName;
|
|
2341
2342
|
try {
|
|
2342
2343
|
if (!this.lanceClient) {
|
|
2343
2344
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2344
2345
|
}
|
|
2345
|
-
if (!
|
|
2346
|
-
throw new Error("tableName is required");
|
|
2346
|
+
if (!resolvedTableName) {
|
|
2347
|
+
throw new Error("tableName or indexName is required");
|
|
2347
2348
|
}
|
|
2348
2349
|
if (!queryVector) {
|
|
2349
2350
|
throw new Error("queryVector is required");
|
|
@@ -2351,28 +2352,33 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2351
2352
|
} catch (error$1) {
|
|
2352
2353
|
throw new error.MastraError(
|
|
2353
2354
|
{
|
|
2354
|
-
id: "
|
|
2355
|
+
id: storage.createVectorErrorId("LANCE", "QUERY", "INVALID_ARGS"),
|
|
2355
2356
|
domain: error.ErrorDomain.STORAGE,
|
|
2356
2357
|
category: error.ErrorCategory.USER,
|
|
2357
|
-
text:
|
|
2358
|
-
details: { tableName }
|
|
2358
|
+
text: error$1 instanceof Error ? error$1.message : "Invalid query arguments",
|
|
2359
|
+
details: { tableName: resolvedTableName }
|
|
2359
2360
|
},
|
|
2360
2361
|
error$1
|
|
2361
2362
|
);
|
|
2362
2363
|
}
|
|
2363
2364
|
try {
|
|
2364
|
-
const
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2365
|
+
const tables = await this.lanceClient.tableNames();
|
|
2366
|
+
if (!tables.includes(resolvedTableName)) {
|
|
2367
|
+
this.logger.debug(`Table ${resolvedTableName} does not exist. Returning empty results.`);
|
|
2368
|
+
return [];
|
|
2368
2369
|
}
|
|
2370
|
+
const table = await this.lanceClient.openTable(resolvedTableName);
|
|
2369
2371
|
let query = table.search(queryVector);
|
|
2370
2372
|
if (filter && Object.keys(filter).length > 0) {
|
|
2371
2373
|
const whereClause = this.filterTranslator(filter);
|
|
2372
2374
|
this.logger.debug(`Where clause generated: ${whereClause}`);
|
|
2373
2375
|
query = query.where(whereClause);
|
|
2374
2376
|
}
|
|
2375
|
-
if (!includeAllColumns &&
|
|
2377
|
+
if (!includeAllColumns && columns.length > 0) {
|
|
2378
|
+
const selectColumns = [...columns];
|
|
2379
|
+
if (!selectColumns.includes("id")) {
|
|
2380
|
+
selectColumns.push("id");
|
|
2381
|
+
}
|
|
2376
2382
|
query = query.select(selectColumns);
|
|
2377
2383
|
}
|
|
2378
2384
|
query = query.limit(topK);
|
|
@@ -2399,10 +2405,10 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2399
2405
|
} catch (error$1) {
|
|
2400
2406
|
throw new error.MastraError(
|
|
2401
2407
|
{
|
|
2402
|
-
id: "
|
|
2408
|
+
id: storage.createVectorErrorId("LANCE", "QUERY", "FAILED"),
|
|
2403
2409
|
domain: error.ErrorDomain.STORAGE,
|
|
2404
2410
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2405
|
-
details: { tableName, includeVector, columnsCount: columns?.length, includeAllColumns }
|
|
2411
|
+
details: { tableName: resolvedTableName, includeVector, columnsCount: columns?.length, includeAllColumns }
|
|
2406
2412
|
},
|
|
2407
2413
|
error$1
|
|
2408
2414
|
);
|
|
@@ -2437,13 +2443,14 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2437
2443
|
const translator = new LanceFilterTranslator();
|
|
2438
2444
|
return translator.translate(prefixedFilter);
|
|
2439
2445
|
}
|
|
2440
|
-
async upsert({ tableName, vectors, metadata = [], ids = [] }) {
|
|
2446
|
+
async upsert({ tableName, indexName, vectors, metadata = [], ids = [] }) {
|
|
2447
|
+
const resolvedTableName = tableName ?? indexName;
|
|
2441
2448
|
try {
|
|
2442
2449
|
if (!this.lanceClient) {
|
|
2443
2450
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2444
2451
|
}
|
|
2445
|
-
if (!
|
|
2446
|
-
throw new Error("tableName is required");
|
|
2452
|
+
if (!resolvedTableName) {
|
|
2453
|
+
throw new Error("tableName or indexName is required");
|
|
2447
2454
|
}
|
|
2448
2455
|
if (!vectors || !Array.isArray(vectors) || vectors.length === 0) {
|
|
2449
2456
|
throw new Error("vectors array is required and must not be empty");
|
|
@@ -2451,21 +2458,24 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2451
2458
|
} catch (error$1) {
|
|
2452
2459
|
throw new error.MastraError(
|
|
2453
2460
|
{
|
|
2454
|
-
id: "
|
|
2461
|
+
id: storage.createVectorErrorId("LANCE", "UPSERT", "INVALID_ARGS"),
|
|
2455
2462
|
domain: error.ErrorDomain.STORAGE,
|
|
2456
2463
|
category: error.ErrorCategory.USER,
|
|
2457
|
-
text:
|
|
2458
|
-
details: { tableName }
|
|
2464
|
+
text: error$1 instanceof Error ? error$1.message : "Invalid upsert arguments",
|
|
2465
|
+
details: { tableName: resolvedTableName }
|
|
2459
2466
|
},
|
|
2460
2467
|
error$1
|
|
2461
2468
|
);
|
|
2462
2469
|
}
|
|
2463
2470
|
try {
|
|
2464
2471
|
const tables = await this.lanceClient.tableNames();
|
|
2465
|
-
|
|
2466
|
-
|
|
2472
|
+
const tableExists = tables.includes(resolvedTableName);
|
|
2473
|
+
let table = null;
|
|
2474
|
+
if (!tableExists) {
|
|
2475
|
+
this.logger.debug(`Table ${resolvedTableName} does not exist. Creating it with the first upsert data.`);
|
|
2476
|
+
} else {
|
|
2477
|
+
table = await this.lanceClient.openTable(resolvedTableName);
|
|
2467
2478
|
}
|
|
2468
|
-
const table = await this.lanceClient.openTable(tableName);
|
|
2469
2479
|
const vectorIds = ids.length === vectors.length ? ids : vectors.map((_, i) => ids[i] || crypto.randomUUID());
|
|
2470
2480
|
const data = vectors.map((vector, i) => {
|
|
2471
2481
|
const id = String(vectorIds[i]);
|
|
@@ -2482,15 +2492,55 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2482
2492
|
}
|
|
2483
2493
|
return rowData;
|
|
2484
2494
|
});
|
|
2485
|
-
|
|
2495
|
+
if (table !== null) {
|
|
2496
|
+
const rowCount = await table.countRows();
|
|
2497
|
+
const schema = await table.schema();
|
|
2498
|
+
const existingColumns = new Set(schema.fields.map((f) => f.name));
|
|
2499
|
+
const dataColumns = new Set(Object.keys(data[0] || {}));
|
|
2500
|
+
const extraColumns = [...dataColumns].filter((col) => !existingColumns.has(col));
|
|
2501
|
+
const missingSchemaColumns = [...existingColumns].filter((col) => !dataColumns.has(col));
|
|
2502
|
+
const hasSchemaMismatch = extraColumns.length > 0 || missingSchemaColumns.length > 0;
|
|
2503
|
+
if (rowCount === 0 && extraColumns.length > 0) {
|
|
2504
|
+
this.logger.warn(
|
|
2505
|
+
`Table ${resolvedTableName} is empty and data has extra columns ${extraColumns.join(", ")}. Recreating with new schema.`
|
|
2506
|
+
);
|
|
2507
|
+
await this.lanceClient.dropTable(resolvedTableName);
|
|
2508
|
+
await this.lanceClient.createTable(resolvedTableName, data);
|
|
2509
|
+
} else if (hasSchemaMismatch) {
|
|
2510
|
+
if (extraColumns.length > 0) {
|
|
2511
|
+
this.logger.warn(
|
|
2512
|
+
`Table ${resolvedTableName} has ${rowCount} rows. Columns ${extraColumns.join(", ")} will be dropped from upsert.`
|
|
2513
|
+
);
|
|
2514
|
+
}
|
|
2515
|
+
const schemaFieldNames = schema.fields.map((f) => f.name);
|
|
2516
|
+
const normalizedData = data.map((row) => {
|
|
2517
|
+
const normalized = {};
|
|
2518
|
+
for (const col of schemaFieldNames) {
|
|
2519
|
+
normalized[col] = col in row ? row[col] : null;
|
|
2520
|
+
}
|
|
2521
|
+
return normalized;
|
|
2522
|
+
});
|
|
2523
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(normalizedData);
|
|
2524
|
+
} else {
|
|
2525
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(data);
|
|
2526
|
+
}
|
|
2527
|
+
} else {
|
|
2528
|
+
this.logger.debug(`Creating table ${resolvedTableName} with initial data`);
|
|
2529
|
+
await this.lanceClient.createTable(resolvedTableName, data);
|
|
2530
|
+
}
|
|
2486
2531
|
return vectorIds;
|
|
2487
2532
|
} catch (error$1) {
|
|
2488
2533
|
throw new error.MastraError(
|
|
2489
2534
|
{
|
|
2490
|
-
id: "
|
|
2535
|
+
id: storage.createVectorErrorId("LANCE", "UPSERT", "FAILED"),
|
|
2491
2536
|
domain: error.ErrorDomain.STORAGE,
|
|
2492
2537
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2493
|
-
details: {
|
|
2538
|
+
details: {
|
|
2539
|
+
tableName: resolvedTableName,
|
|
2540
|
+
vectorCount: vectors.length,
|
|
2541
|
+
metadataCount: metadata.length,
|
|
2542
|
+
idsCount: ids.length
|
|
2543
|
+
}
|
|
2494
2544
|
},
|
|
2495
2545
|
error$1
|
|
2496
2546
|
);
|
|
@@ -2514,7 +2564,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2514
2564
|
async createTable(tableName, data, options) {
|
|
2515
2565
|
if (!this.lanceClient) {
|
|
2516
2566
|
throw new error.MastraError({
|
|
2517
|
-
id: "
|
|
2567
|
+
id: storage.createVectorErrorId("LANCE", "CREATE_TABLE", "INVALID_ARGS"),
|
|
2518
2568
|
domain: error.ErrorDomain.STORAGE,
|
|
2519
2569
|
category: error.ErrorCategory.USER,
|
|
2520
2570
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2529,7 +2579,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2529
2579
|
} catch (error$1) {
|
|
2530
2580
|
throw new error.MastraError(
|
|
2531
2581
|
{
|
|
2532
|
-
id: "
|
|
2582
|
+
id: storage.createVectorErrorId("LANCE", "CREATE_TABLE", "FAILED"),
|
|
2533
2583
|
domain: error.ErrorDomain.STORAGE,
|
|
2534
2584
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2535
2585
|
details: { tableName }
|
|
@@ -2541,7 +2591,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2541
2591
|
async listTables() {
|
|
2542
2592
|
if (!this.lanceClient) {
|
|
2543
2593
|
throw new error.MastraError({
|
|
2544
|
-
id: "
|
|
2594
|
+
id: storage.createVectorErrorId("LANCE", "LIST_TABLES", "INVALID_ARGS"),
|
|
2545
2595
|
domain: error.ErrorDomain.STORAGE,
|
|
2546
2596
|
category: error.ErrorCategory.USER,
|
|
2547
2597
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2553,7 +2603,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2553
2603
|
} catch (error$1) {
|
|
2554
2604
|
throw new error.MastraError(
|
|
2555
2605
|
{
|
|
2556
|
-
id: "
|
|
2606
|
+
id: storage.createVectorErrorId("LANCE", "LIST_TABLES", "FAILED"),
|
|
2557
2607
|
domain: error.ErrorDomain.STORAGE,
|
|
2558
2608
|
category: error.ErrorCategory.THIRD_PARTY
|
|
2559
2609
|
},
|
|
@@ -2564,7 +2614,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2564
2614
|
async getTableSchema(tableName) {
|
|
2565
2615
|
if (!this.lanceClient) {
|
|
2566
2616
|
throw new error.MastraError({
|
|
2567
|
-
id: "
|
|
2617
|
+
id: storage.createVectorErrorId("LANCE", "GET_TABLE_SCHEMA", "INVALID_ARGS"),
|
|
2568
2618
|
domain: error.ErrorDomain.STORAGE,
|
|
2569
2619
|
category: error.ErrorCategory.USER,
|
|
2570
2620
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2577,7 +2627,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2577
2627
|
} catch (error$1) {
|
|
2578
2628
|
throw new error.MastraError(
|
|
2579
2629
|
{
|
|
2580
|
-
id: "
|
|
2630
|
+
id: storage.createVectorErrorId("LANCE", "GET_TABLE_SCHEMA", "FAILED"),
|
|
2581
2631
|
domain: error.ErrorDomain.STORAGE,
|
|
2582
2632
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2583
2633
|
details: { tableName }
|
|
@@ -2587,7 +2637,17 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2587
2637
|
}
|
|
2588
2638
|
}
|
|
2589
2639
|
/**
|
|
2590
|
-
*
|
|
2640
|
+
* Creates a vector index on a table.
|
|
2641
|
+
*
|
|
2642
|
+
* The behavior of `indexName` depends on whether `tableName` is provided:
|
|
2643
|
+
* - With `tableName`: `indexName` is the column to index (advanced use case)
|
|
2644
|
+
* - Without `tableName`: `indexName` becomes the table name, and 'vector' is used as the column (Memory compatibility)
|
|
2645
|
+
*
|
|
2646
|
+
* @param tableName - Optional table name. If not provided, defaults to indexName.
|
|
2647
|
+
* @param indexName - The index/column name, or table name if tableName is not provided.
|
|
2648
|
+
* @param dimension - Vector dimension size.
|
|
2649
|
+
* @param metric - Distance metric: 'cosine', 'euclidean', or 'dotproduct'.
|
|
2650
|
+
* @param indexConfig - Optional index configuration.
|
|
2591
2651
|
*/
|
|
2592
2652
|
async createIndex({
|
|
2593
2653
|
tableName,
|
|
@@ -2596,13 +2656,12 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2596
2656
|
metric = "cosine",
|
|
2597
2657
|
indexConfig = {}
|
|
2598
2658
|
}) {
|
|
2659
|
+
const resolvedTableName = tableName ?? indexName;
|
|
2660
|
+
const columnToIndex = tableName ? indexName : "vector";
|
|
2599
2661
|
try {
|
|
2600
2662
|
if (!this.lanceClient) {
|
|
2601
2663
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2602
2664
|
}
|
|
2603
|
-
if (!tableName) {
|
|
2604
|
-
throw new Error("tableName is required");
|
|
2605
|
-
}
|
|
2606
2665
|
if (!indexName) {
|
|
2607
2666
|
throw new Error("indexName is required");
|
|
2608
2667
|
}
|
|
@@ -2612,22 +2671,36 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2612
2671
|
} catch (err) {
|
|
2613
2672
|
throw new error.MastraError(
|
|
2614
2673
|
{
|
|
2615
|
-
id: "
|
|
2674
|
+
id: storage.createVectorErrorId("LANCE", "CREATE_INDEX", "INVALID_ARGS"),
|
|
2616
2675
|
domain: error.ErrorDomain.STORAGE,
|
|
2617
2676
|
category: error.ErrorCategory.USER,
|
|
2618
|
-
details: { tableName:
|
|
2677
|
+
details: { tableName: resolvedTableName, indexName, dimension, metric }
|
|
2619
2678
|
},
|
|
2620
2679
|
err
|
|
2621
2680
|
);
|
|
2622
2681
|
}
|
|
2623
2682
|
try {
|
|
2624
2683
|
const tables = await this.lanceClient.tableNames();
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2684
|
+
let table;
|
|
2685
|
+
if (!tables.includes(resolvedTableName)) {
|
|
2686
|
+
this.logger.debug(
|
|
2687
|
+
`Table ${resolvedTableName} does not exist. Creating empty table with dimension ${dimension}.`
|
|
2628
2688
|
);
|
|
2689
|
+
const initVector = new Array(dimension).fill(0);
|
|
2690
|
+
table = await this.lanceClient.createTable(resolvedTableName, [{ id: "__init__", vector: initVector }]);
|
|
2691
|
+
try {
|
|
2692
|
+
await table.delete("id = '__init__'");
|
|
2693
|
+
} catch (deleteError) {
|
|
2694
|
+
this.logger.warn(
|
|
2695
|
+
`Failed to delete initialization row from ${resolvedTableName}. Subsequent queries may include '__init__' row.`,
|
|
2696
|
+
deleteError
|
|
2697
|
+
);
|
|
2698
|
+
}
|
|
2699
|
+
this.logger.debug(`Table ${resolvedTableName} created. Index creation deferred until data is available.`);
|
|
2700
|
+
return;
|
|
2701
|
+
} else {
|
|
2702
|
+
table = await this.lanceClient.openTable(resolvedTableName);
|
|
2629
2703
|
}
|
|
2630
|
-
const table = await this.lanceClient.openTable(tableName);
|
|
2631
2704
|
let metricType;
|
|
2632
2705
|
if (metric === "euclidean") {
|
|
2633
2706
|
metricType = "l2";
|
|
@@ -2636,8 +2709,15 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2636
2709
|
} else if (metric === "cosine") {
|
|
2637
2710
|
metricType = "cosine";
|
|
2638
2711
|
}
|
|
2712
|
+
const rowCount = await table.countRows();
|
|
2713
|
+
if (rowCount < 256) {
|
|
2714
|
+
this.logger.warn(
|
|
2715
|
+
`Table ${resolvedTableName} has ${rowCount} rows, which is below the 256 row minimum for index creation. Skipping index creation.`
|
|
2716
|
+
);
|
|
2717
|
+
return;
|
|
2718
|
+
}
|
|
2639
2719
|
if (indexConfig.type === "ivfflat") {
|
|
2640
|
-
await table.createIndex(
|
|
2720
|
+
await table.createIndex(columnToIndex, {
|
|
2641
2721
|
config: lancedb.Index.ivfPq({
|
|
2642
2722
|
numPartitions: indexConfig.numPartitions || 128,
|
|
2643
2723
|
numSubVectors: indexConfig.numSubVectors || 16,
|
|
@@ -2646,7 +2726,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2646
2726
|
});
|
|
2647
2727
|
} else {
|
|
2648
2728
|
this.logger.debug("Creating HNSW PQ index with config:", indexConfig);
|
|
2649
|
-
await table.createIndex(
|
|
2729
|
+
await table.createIndex(columnToIndex, {
|
|
2650
2730
|
config: lancedb.Index.hnswPq({
|
|
2651
2731
|
m: indexConfig?.hnsw?.m || 16,
|
|
2652
2732
|
efConstruction: indexConfig?.hnsw?.efConstruction || 100,
|
|
@@ -2657,10 +2737,10 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2657
2737
|
} catch (error$1) {
|
|
2658
2738
|
throw new error.MastraError(
|
|
2659
2739
|
{
|
|
2660
|
-
id: "
|
|
2740
|
+
id: storage.createVectorErrorId("LANCE", "CREATE_INDEX", "FAILED"),
|
|
2661
2741
|
domain: error.ErrorDomain.STORAGE,
|
|
2662
2742
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2663
|
-
details: { tableName:
|
|
2743
|
+
details: { tableName: resolvedTableName, indexName, dimension }
|
|
2664
2744
|
},
|
|
2665
2745
|
error$1
|
|
2666
2746
|
);
|
|
@@ -2669,7 +2749,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2669
2749
|
async listIndexes() {
|
|
2670
2750
|
if (!this.lanceClient) {
|
|
2671
2751
|
throw new error.MastraError({
|
|
2672
|
-
id: "
|
|
2752
|
+
id: storage.createVectorErrorId("LANCE", "LIST_INDEXES", "INVALID_ARGS"),
|
|
2673
2753
|
domain: error.ErrorDomain.STORAGE,
|
|
2674
2754
|
category: error.ErrorCategory.USER,
|
|
2675
2755
|
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
@@ -2688,7 +2768,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2688
2768
|
} catch (error$1) {
|
|
2689
2769
|
throw new error.MastraError(
|
|
2690
2770
|
{
|
|
2691
|
-
id: "
|
|
2771
|
+
id: storage.createVectorErrorId("LANCE", "LIST_INDEXES", "FAILED"),
|
|
2692
2772
|
domain: error.ErrorDomain.STORAGE,
|
|
2693
2773
|
category: error.ErrorCategory.THIRD_PARTY
|
|
2694
2774
|
},
|
|
@@ -2707,7 +2787,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2707
2787
|
} catch (err) {
|
|
2708
2788
|
throw new error.MastraError(
|
|
2709
2789
|
{
|
|
2710
|
-
id: "
|
|
2790
|
+
id: storage.createVectorErrorId("LANCE", "DESCRIBE_INDEX", "INVALID_ARGS"),
|
|
2711
2791
|
domain: error.ErrorDomain.STORAGE,
|
|
2712
2792
|
category: error.ErrorCategory.USER,
|
|
2713
2793
|
details: { indexName }
|
|
@@ -2742,7 +2822,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2742
2822
|
} catch (error$1) {
|
|
2743
2823
|
throw new error.MastraError(
|
|
2744
2824
|
{
|
|
2745
|
-
id: "
|
|
2825
|
+
id: storage.createVectorErrorId("LANCE", "DESCRIBE_INDEX", "FAILED"),
|
|
2746
2826
|
domain: error.ErrorDomain.STORAGE,
|
|
2747
2827
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2748
2828
|
details: { indexName }
|
|
@@ -2762,7 +2842,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2762
2842
|
} catch (err) {
|
|
2763
2843
|
throw new error.MastraError(
|
|
2764
2844
|
{
|
|
2765
|
-
id: "
|
|
2845
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_INDEX", "INVALID_ARGS"),
|
|
2766
2846
|
domain: error.ErrorDomain.STORAGE,
|
|
2767
2847
|
category: error.ErrorCategory.USER,
|
|
2768
2848
|
details: { indexName }
|
|
@@ -2785,7 +2865,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2785
2865
|
} catch (error$1) {
|
|
2786
2866
|
throw new error.MastraError(
|
|
2787
2867
|
{
|
|
2788
|
-
id: "
|
|
2868
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_INDEX", "FAILED"),
|
|
2789
2869
|
domain: error.ErrorDomain.STORAGE,
|
|
2790
2870
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2791
2871
|
details: { indexName }
|
|
@@ -2800,7 +2880,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2800
2880
|
async deleteAllTables() {
|
|
2801
2881
|
if (!this.lanceClient) {
|
|
2802
2882
|
throw new error.MastraError({
|
|
2803
|
-
id: "
|
|
2883
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_ALL_TABLES", "INVALID_ARGS"),
|
|
2804
2884
|
domain: error.ErrorDomain.STORAGE,
|
|
2805
2885
|
category: error.ErrorCategory.USER,
|
|
2806
2886
|
details: { methodName: "deleteAllTables" },
|
|
@@ -2812,7 +2892,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2812
2892
|
} catch (error$1) {
|
|
2813
2893
|
throw new error.MastraError(
|
|
2814
2894
|
{
|
|
2815
|
-
id: "
|
|
2895
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_ALL_TABLES", "FAILED"),
|
|
2816
2896
|
domain: error.ErrorDomain.STORAGE,
|
|
2817
2897
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2818
2898
|
details: { methodName: "deleteAllTables" }
|
|
@@ -2824,7 +2904,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2824
2904
|
async deleteTable(tableName) {
|
|
2825
2905
|
if (!this.lanceClient) {
|
|
2826
2906
|
throw new error.MastraError({
|
|
2827
|
-
id: "
|
|
2907
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_TABLE", "INVALID_ARGS"),
|
|
2828
2908
|
domain: error.ErrorDomain.STORAGE,
|
|
2829
2909
|
category: error.ErrorCategory.USER,
|
|
2830
2910
|
details: { tableName },
|
|
@@ -2836,7 +2916,7 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2836
2916
|
} catch (error$1) {
|
|
2837
2917
|
throw new error.MastraError(
|
|
2838
2918
|
{
|
|
2839
|
-
id: "
|
|
2919
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_TABLE", "FAILED"),
|
|
2840
2920
|
domain: error.ErrorDomain.STORAGE,
|
|
2841
2921
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2842
2922
|
details: { tableName }
|
|
@@ -2845,7 +2925,44 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2845
2925
|
);
|
|
2846
2926
|
}
|
|
2847
2927
|
}
|
|
2848
|
-
async updateVector(
|
|
2928
|
+
async updateVector(params) {
|
|
2929
|
+
const { indexName, update } = params;
|
|
2930
|
+
if ("id" in params && "filter" in params && params.id && params.filter) {
|
|
2931
|
+
throw new error.MastraError({
|
|
2932
|
+
id: storage.createVectorErrorId("LANCE", "UPDATE_VECTOR", "MUTUALLY_EXCLUSIVE"),
|
|
2933
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2934
|
+
category: error.ErrorCategory.USER,
|
|
2935
|
+
text: "id and filter are mutually exclusive",
|
|
2936
|
+
details: { indexName }
|
|
2937
|
+
});
|
|
2938
|
+
}
|
|
2939
|
+
if (!("id" in params || "filter" in params) || !params.id && !params.filter) {
|
|
2940
|
+
throw new error.MastraError({
|
|
2941
|
+
id: storage.createVectorErrorId("LANCE", "UPDATE_VECTOR", "NO_TARGET"),
|
|
2942
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2943
|
+
category: error.ErrorCategory.USER,
|
|
2944
|
+
text: "Either id or filter must be provided",
|
|
2945
|
+
details: { indexName }
|
|
2946
|
+
});
|
|
2947
|
+
}
|
|
2948
|
+
if ("filter" in params && params.filter && Object.keys(params.filter).length === 0) {
|
|
2949
|
+
throw new error.MastraError({
|
|
2950
|
+
id: storage.createVectorErrorId("LANCE", "UPDATE_VECTOR", "EMPTY_FILTER"),
|
|
2951
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2952
|
+
category: error.ErrorCategory.USER,
|
|
2953
|
+
text: "Cannot update with empty filter",
|
|
2954
|
+
details: { indexName }
|
|
2955
|
+
});
|
|
2956
|
+
}
|
|
2957
|
+
if (!update.vector && !update.metadata) {
|
|
2958
|
+
throw new error.MastraError({
|
|
2959
|
+
id: storage.createVectorErrorId("LANCE", "UPDATE_VECTOR", "NO_PAYLOAD"),
|
|
2960
|
+
domain: error.ErrorDomain.STORAGE,
|
|
2961
|
+
category: error.ErrorCategory.USER,
|
|
2962
|
+
text: "No updates provided",
|
|
2963
|
+
details: { indexName }
|
|
2964
|
+
});
|
|
2965
|
+
}
|
|
2849
2966
|
try {
|
|
2850
2967
|
if (!this.lanceClient) {
|
|
2851
2968
|
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
@@ -2853,21 +2970,6 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2853
2970
|
if (!indexName) {
|
|
2854
2971
|
throw new Error("indexName is required");
|
|
2855
2972
|
}
|
|
2856
|
-
if (!id) {
|
|
2857
|
-
throw new Error("id is required");
|
|
2858
|
-
}
|
|
2859
|
-
} catch (err) {
|
|
2860
|
-
throw new error.MastraError(
|
|
2861
|
-
{
|
|
2862
|
-
id: "STORAGE_LANCE_VECTOR_UPDATE_VECTOR_FAILED_INVALID_ARGS",
|
|
2863
|
-
domain: error.ErrorDomain.STORAGE,
|
|
2864
|
-
category: error.ErrorCategory.USER,
|
|
2865
|
-
details: { indexName, id }
|
|
2866
|
-
},
|
|
2867
|
-
err
|
|
2868
|
-
);
|
|
2869
|
-
}
|
|
2870
|
-
try {
|
|
2871
2973
|
const tables = await this.lanceClient.tableNames();
|
|
2872
2974
|
for (const tableName of tables) {
|
|
2873
2975
|
this.logger.debug("Checking table:" + tableName);
|
|
@@ -2877,39 +2979,66 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2877
2979
|
const hasColumn = schema.fields.some((field) => field.name === indexName);
|
|
2878
2980
|
if (hasColumn) {
|
|
2879
2981
|
this.logger.debug(`Found column ${indexName} in table ${tableName}`);
|
|
2880
|
-
|
|
2881
|
-
if (
|
|
2882
|
-
|
|
2982
|
+
let whereClause;
|
|
2983
|
+
if ("id" in params && params.id) {
|
|
2984
|
+
whereClause = `id = '${params.id}'`;
|
|
2985
|
+
} else if ("filter" in params && params.filter) {
|
|
2986
|
+
const translator = new LanceFilterTranslator();
|
|
2987
|
+
const processFilterKeys = (filter) => {
|
|
2988
|
+
const processedFilter = {};
|
|
2989
|
+
Object.entries(filter).forEach(([key, value]) => {
|
|
2990
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
2991
|
+
Object.entries(value).forEach(([nestedKey, nestedValue]) => {
|
|
2992
|
+
processedFilter[`metadata_${key}_${nestedKey}`] = nestedValue;
|
|
2993
|
+
});
|
|
2994
|
+
} else {
|
|
2995
|
+
processedFilter[`metadata_${key}`] = value;
|
|
2996
|
+
}
|
|
2997
|
+
});
|
|
2998
|
+
return processedFilter;
|
|
2999
|
+
};
|
|
3000
|
+
const prefixedFilter = processFilterKeys(params.filter);
|
|
3001
|
+
whereClause = translator.translate(prefixedFilter) || "";
|
|
3002
|
+
if (!whereClause) {
|
|
3003
|
+
throw new Error("Failed to translate filter to SQL");
|
|
3004
|
+
}
|
|
3005
|
+
} else {
|
|
3006
|
+
throw new Error("Either id or filter must be provided");
|
|
2883
3007
|
}
|
|
2884
|
-
const
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2894
|
-
|
|
3008
|
+
const existingRecords = await table.query().where(whereClause).select(schema.fields.map((field) => field.name)).toArray();
|
|
3009
|
+
if (existingRecords.length === 0) {
|
|
3010
|
+
this.logger.info(`No records found matching criteria in table ${tableName}`);
|
|
3011
|
+
return;
|
|
3012
|
+
}
|
|
3013
|
+
const updatedRecords = existingRecords.map((record) => {
|
|
3014
|
+
const rowData = {};
|
|
3015
|
+
Object.entries(record).forEach(([key, value]) => {
|
|
3016
|
+
if (key !== "_distance") {
|
|
3017
|
+
if (key === indexName) {
|
|
3018
|
+
if (update.vector) {
|
|
3019
|
+
rowData[key] = update.vector;
|
|
2895
3020
|
} else {
|
|
2896
|
-
|
|
3021
|
+
if (Array.isArray(value)) {
|
|
3022
|
+
rowData[key] = [...value];
|
|
3023
|
+
} else if (typeof value === "object" && value !== null) {
|
|
3024
|
+
rowData[key] = Array.from(value);
|
|
3025
|
+
} else {
|
|
3026
|
+
rowData[key] = value;
|
|
3027
|
+
}
|
|
2897
3028
|
}
|
|
3029
|
+
} else {
|
|
3030
|
+
rowData[key] = value;
|
|
2898
3031
|
}
|
|
2899
|
-
} else {
|
|
2900
|
-
rowData[key] = value;
|
|
2901
3032
|
}
|
|
3033
|
+
});
|
|
3034
|
+
if (update.metadata) {
|
|
3035
|
+
Object.entries(update.metadata).forEach(([key, value]) => {
|
|
3036
|
+
rowData[`metadata_${key}`] = value;
|
|
3037
|
+
});
|
|
2902
3038
|
}
|
|
3039
|
+
return rowData;
|
|
2903
3040
|
});
|
|
2904
|
-
|
|
2905
|
-
rowData[indexName] = update.vector;
|
|
2906
|
-
}
|
|
2907
|
-
if (update.metadata) {
|
|
2908
|
-
Object.entries(update.metadata).forEach(([key, value]) => {
|
|
2909
|
-
rowData[`metadata_${key}`] = value;
|
|
2910
|
-
});
|
|
2911
|
-
}
|
|
2912
|
-
await table.add([rowData], { mode: "overwrite" });
|
|
3041
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(updatedRecords);
|
|
2913
3042
|
return;
|
|
2914
3043
|
}
|
|
2915
3044
|
} catch (err) {
|
|
@@ -2919,12 +3048,19 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2919
3048
|
}
|
|
2920
3049
|
throw new Error(`No table found with column/index '${indexName}'`);
|
|
2921
3050
|
} catch (error$1) {
|
|
3051
|
+
if (error$1 instanceof error.MastraError) throw error$1;
|
|
2922
3052
|
throw new error.MastraError(
|
|
2923
3053
|
{
|
|
2924
|
-
id: "
|
|
3054
|
+
id: storage.createVectorErrorId("LANCE", "UPDATE_VECTOR", "FAILED"),
|
|
2925
3055
|
domain: error.ErrorDomain.STORAGE,
|
|
2926
3056
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2927
|
-
details: {
|
|
3057
|
+
details: {
|
|
3058
|
+
indexName,
|
|
3059
|
+
..."id" in params && params.id && { id: params.id },
|
|
3060
|
+
..."filter" in params && params.filter && { filter: JSON.stringify(params.filter) },
|
|
3061
|
+
hasVector: !!update.vector,
|
|
3062
|
+
hasMetadata: !!update.metadata
|
|
3063
|
+
}
|
|
2928
3064
|
},
|
|
2929
3065
|
error$1
|
|
2930
3066
|
);
|
|
@@ -2944,10 +3080,13 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2944
3080
|
} catch (err) {
|
|
2945
3081
|
throw new error.MastraError(
|
|
2946
3082
|
{
|
|
2947
|
-
id: "
|
|
3083
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_VECTOR", "INVALID_ARGS"),
|
|
2948
3084
|
domain: error.ErrorDomain.STORAGE,
|
|
2949
3085
|
category: error.ErrorCategory.USER,
|
|
2950
|
-
details: {
|
|
3086
|
+
details: {
|
|
3087
|
+
indexName,
|
|
3088
|
+
...id && { id }
|
|
3089
|
+
}
|
|
2951
3090
|
},
|
|
2952
3091
|
err
|
|
2953
3092
|
);
|
|
@@ -2974,10 +3113,13 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
2974
3113
|
} catch (error$1) {
|
|
2975
3114
|
throw new error.MastraError(
|
|
2976
3115
|
{
|
|
2977
|
-
id: "
|
|
3116
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_VECTOR", "FAILED"),
|
|
2978
3117
|
domain: error.ErrorDomain.STORAGE,
|
|
2979
3118
|
category: error.ErrorCategory.THIRD_PARTY,
|
|
2980
|
-
details: {
|
|
3119
|
+
details: {
|
|
3120
|
+
indexName,
|
|
3121
|
+
...id && { id }
|
|
3122
|
+
}
|
|
2981
3123
|
},
|
|
2982
3124
|
error$1
|
|
2983
3125
|
);
|
|
@@ -3008,9 +3150,115 @@ var LanceVectorStore = class _LanceVectorStore extends vector.MastraVector {
|
|
|
3008
3150
|
});
|
|
3009
3151
|
return result;
|
|
3010
3152
|
}
|
|
3153
|
+
async deleteVectors({ indexName, filter, ids }) {
|
|
3154
|
+
if (ids && filter) {
|
|
3155
|
+
throw new error.MastraError({
|
|
3156
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_VECTORS", "MUTUALLY_EXCLUSIVE"),
|
|
3157
|
+
domain: error.ErrorDomain.STORAGE,
|
|
3158
|
+
category: error.ErrorCategory.USER,
|
|
3159
|
+
text: "ids and filter are mutually exclusive",
|
|
3160
|
+
details: { indexName }
|
|
3161
|
+
});
|
|
3162
|
+
}
|
|
3163
|
+
if (!ids && !filter) {
|
|
3164
|
+
throw new error.MastraError({
|
|
3165
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_VECTORS", "NO_TARGET"),
|
|
3166
|
+
domain: error.ErrorDomain.STORAGE,
|
|
3167
|
+
category: error.ErrorCategory.USER,
|
|
3168
|
+
text: "Either filter or ids must be provided",
|
|
3169
|
+
details: { indexName }
|
|
3170
|
+
});
|
|
3171
|
+
}
|
|
3172
|
+
if (ids && ids.length === 0) {
|
|
3173
|
+
throw new error.MastraError({
|
|
3174
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_VECTORS", "EMPTY_IDS"),
|
|
3175
|
+
domain: error.ErrorDomain.STORAGE,
|
|
3176
|
+
category: error.ErrorCategory.USER,
|
|
3177
|
+
text: "Cannot delete with empty ids array",
|
|
3178
|
+
details: { indexName }
|
|
3179
|
+
});
|
|
3180
|
+
}
|
|
3181
|
+
if (filter && Object.keys(filter).length === 0) {
|
|
3182
|
+
throw new error.MastraError({
|
|
3183
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_VECTORS", "EMPTY_FILTER"),
|
|
3184
|
+
domain: error.ErrorDomain.STORAGE,
|
|
3185
|
+
category: error.ErrorCategory.USER,
|
|
3186
|
+
text: "Cannot delete with empty filter",
|
|
3187
|
+
details: { indexName }
|
|
3188
|
+
});
|
|
3189
|
+
}
|
|
3190
|
+
try {
|
|
3191
|
+
if (!this.lanceClient) {
|
|
3192
|
+
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
3193
|
+
}
|
|
3194
|
+
if (!indexName) {
|
|
3195
|
+
throw new Error("indexName is required");
|
|
3196
|
+
}
|
|
3197
|
+
const tables = await this.lanceClient.tableNames();
|
|
3198
|
+
for (const tableName of tables) {
|
|
3199
|
+
this.logger.debug("Checking table:" + tableName);
|
|
3200
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
3201
|
+
try {
|
|
3202
|
+
const schema = await table.schema();
|
|
3203
|
+
const hasColumn = schema.fields.some((field) => field.name === indexName);
|
|
3204
|
+
if (hasColumn) {
|
|
3205
|
+
this.logger.debug(`Found column ${indexName} in table ${tableName}`);
|
|
3206
|
+
if (ids) {
|
|
3207
|
+
const idsConditions = ids.map((id) => `id = '${id}'`).join(" OR ");
|
|
3208
|
+
await table.delete(idsConditions);
|
|
3209
|
+
} else if (filter) {
|
|
3210
|
+
const translator = new LanceFilterTranslator();
|
|
3211
|
+
const processFilterKeys = (filter2) => {
|
|
3212
|
+
const processedFilter = {};
|
|
3213
|
+
Object.entries(filter2).forEach(([key, value]) => {
|
|
3214
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
3215
|
+
Object.entries(value).forEach(([nestedKey, nestedValue]) => {
|
|
3216
|
+
processedFilter[`metadata_${key}_${nestedKey}`] = nestedValue;
|
|
3217
|
+
});
|
|
3218
|
+
} else {
|
|
3219
|
+
processedFilter[`metadata_${key}`] = value;
|
|
3220
|
+
}
|
|
3221
|
+
});
|
|
3222
|
+
return processedFilter;
|
|
3223
|
+
};
|
|
3224
|
+
const prefixedFilter = processFilterKeys(filter);
|
|
3225
|
+
const whereClause = translator.translate(prefixedFilter);
|
|
3226
|
+
if (!whereClause) {
|
|
3227
|
+
throw new Error("Failed to translate filter to SQL");
|
|
3228
|
+
}
|
|
3229
|
+
await table.delete(whereClause);
|
|
3230
|
+
}
|
|
3231
|
+
return;
|
|
3232
|
+
}
|
|
3233
|
+
} catch (err) {
|
|
3234
|
+
this.logger.error(`Error checking schema for table ${tableName}:` + err);
|
|
3235
|
+
continue;
|
|
3236
|
+
}
|
|
3237
|
+
}
|
|
3238
|
+
throw new Error(`No table found with column/index '${indexName}'`);
|
|
3239
|
+
} catch (error$1) {
|
|
3240
|
+
if (error$1 instanceof error.MastraError) throw error$1;
|
|
3241
|
+
throw new error.MastraError(
|
|
3242
|
+
{
|
|
3243
|
+
id: storage.createVectorErrorId("LANCE", "DELETE_VECTORS", "FAILED"),
|
|
3244
|
+
domain: error.ErrorDomain.STORAGE,
|
|
3245
|
+
category: error.ErrorCategory.THIRD_PARTY,
|
|
3246
|
+
details: {
|
|
3247
|
+
indexName,
|
|
3248
|
+
...filter && { filter: JSON.stringify(filter) },
|
|
3249
|
+
...ids && { idsCount: ids.length }
|
|
3250
|
+
}
|
|
3251
|
+
},
|
|
3252
|
+
error$1
|
|
3253
|
+
);
|
|
3254
|
+
}
|
|
3255
|
+
}
|
|
3011
3256
|
};
|
|
3012
3257
|
|
|
3013
3258
|
exports.LanceStorage = LanceStorage;
|
|
3014
3259
|
exports.LanceVectorStore = LanceVectorStore;
|
|
3260
|
+
exports.StoreMemoryLance = StoreMemoryLance;
|
|
3261
|
+
exports.StoreScoresLance = StoreScoresLance;
|
|
3262
|
+
exports.StoreWorkflowsLance = StoreWorkflowsLance;
|
|
3015
3263
|
//# sourceMappingURL=index.cjs.map
|
|
3016
3264
|
//# sourceMappingURL=index.cjs.map
|