@mastra/lance 0.0.0-add-libsql-changeset-20250910154739
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +540 -0
- package/LICENSE.md +15 -0
- package/README.md +263 -0
- package/dist/index.cjs +3303 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3300 -0
- package/dist/index.js.map +1 -0
- package/dist/storage/domains/legacy-evals/index.d.ts +25 -0
- package/dist/storage/domains/legacy-evals/index.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +103 -0
- package/dist/storage/domains/memory/index.d.ts.map +1 -0
- package/dist/storage/domains/operations/index.d.ts +40 -0
- package/dist/storage/domains/operations/index.d.ts.map +1 -0
- package/dist/storage/domains/scores/index.d.ts +42 -0
- package/dist/storage/domains/scores/index.d.ts.map +1 -0
- package/dist/storage/domains/traces/index.d.ts +34 -0
- package/dist/storage/domains/traces/index.d.ts.map +1 -0
- package/dist/storage/domains/utils.d.ts +10 -0
- package/dist/storage/domains/utils.d.ts.map +1 -0
- package/dist/storage/domains/workflows/index.d.ts +56 -0
- package/dist/storage/domains/workflows/index.d.ts.map +1 -0
- package/dist/storage/index.d.ts +262 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/vector/filter.d.ts +41 -0
- package/dist/vector/filter.d.ts.map +1 -0
- package/dist/vector/index.d.ts +85 -0
- package/dist/vector/index.d.ts.map +1 -0
- package/dist/vector/types.d.ts +15 -0
- package/dist/vector/types.d.ts.map +1 -0
- package/package.json +60 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,3300 @@
|
|
|
1
|
+
import { connect, Index } from '@lancedb/lancedb';
|
|
2
|
+
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
3
|
+
import { MastraStorage, StoreOperations, LegacyEvalsStorage, TABLE_EVALS, MemoryStorage, TABLE_THREADS, TABLE_MESSAGES, resolveMessageLimit, TABLE_RESOURCES, ScoresStorage, TABLE_SCORERS, TracesStorage, TABLE_TRACES, WorkflowsStorage, TABLE_WORKFLOW_SNAPSHOT, ensureDate } from '@mastra/core/storage';
|
|
4
|
+
import { MessageList } from '@mastra/core/agent';
|
|
5
|
+
import { Utf8, Float64, Binary, Float32, Int32, Field, Schema } from 'apache-arrow';
|
|
6
|
+
import { MastraVector } from '@mastra/core/vector';
|
|
7
|
+
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
8
|
+
|
|
9
|
+
// src/storage/index.ts
|
|
10
|
+
var StoreLegacyEvalsLance = class extends LegacyEvalsStorage {
|
|
11
|
+
client;
|
|
12
|
+
constructor({ client }) {
|
|
13
|
+
super();
|
|
14
|
+
this.client = client;
|
|
15
|
+
}
|
|
16
|
+
async getEvalsByAgentName(agentName, type) {
|
|
17
|
+
try {
|
|
18
|
+
const table = await this.client.openTable(TABLE_EVALS);
|
|
19
|
+
const query = table.query().where(`agent_name = '${agentName}'`);
|
|
20
|
+
const records = await query.toArray();
|
|
21
|
+
let filteredRecords = records;
|
|
22
|
+
if (type === "live") {
|
|
23
|
+
filteredRecords = records.filter((record) => record.test_info === null);
|
|
24
|
+
} else if (type === "test") {
|
|
25
|
+
filteredRecords = records.filter((record) => record.test_info !== null);
|
|
26
|
+
}
|
|
27
|
+
return filteredRecords.map((record) => {
|
|
28
|
+
return {
|
|
29
|
+
id: record.id,
|
|
30
|
+
input: record.input,
|
|
31
|
+
output: record.output,
|
|
32
|
+
agentName: record.agent_name,
|
|
33
|
+
metricName: record.metric_name,
|
|
34
|
+
result: JSON.parse(record.result),
|
|
35
|
+
instructions: record.instructions,
|
|
36
|
+
testInfo: record.test_info ? JSON.parse(record.test_info) : null,
|
|
37
|
+
globalRunId: record.global_run_id,
|
|
38
|
+
runId: record.run_id,
|
|
39
|
+
createdAt: new Date(record.created_at).toString()
|
|
40
|
+
};
|
|
41
|
+
});
|
|
42
|
+
} catch (error) {
|
|
43
|
+
throw new MastraError(
|
|
44
|
+
{
|
|
45
|
+
id: "LANCE_STORE_GET_EVALS_BY_AGENT_NAME_FAILED",
|
|
46
|
+
domain: ErrorDomain.STORAGE,
|
|
47
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
48
|
+
details: { agentName }
|
|
49
|
+
},
|
|
50
|
+
error
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
async getEvals(options) {
|
|
55
|
+
try {
|
|
56
|
+
const table = await this.client.openTable(TABLE_EVALS);
|
|
57
|
+
const conditions = [];
|
|
58
|
+
if (options.agentName) {
|
|
59
|
+
conditions.push(`agent_name = '${options.agentName}'`);
|
|
60
|
+
}
|
|
61
|
+
if (options.type === "live") {
|
|
62
|
+
conditions.push("length(test_info) = 0");
|
|
63
|
+
} else if (options.type === "test") {
|
|
64
|
+
conditions.push("length(test_info) > 0");
|
|
65
|
+
}
|
|
66
|
+
const startDate = options.dateRange?.start || options.fromDate;
|
|
67
|
+
const endDate = options.dateRange?.end || options.toDate;
|
|
68
|
+
if (startDate) {
|
|
69
|
+
conditions.push(`\`created_at\` >= ${startDate.getTime()}`);
|
|
70
|
+
}
|
|
71
|
+
if (endDate) {
|
|
72
|
+
conditions.push(`\`created_at\` <= ${endDate.getTime()}`);
|
|
73
|
+
}
|
|
74
|
+
let total = 0;
|
|
75
|
+
if (conditions.length > 0) {
|
|
76
|
+
total = await table.countRows(conditions.join(" AND "));
|
|
77
|
+
} else {
|
|
78
|
+
total = await table.countRows();
|
|
79
|
+
}
|
|
80
|
+
const query = table.query();
|
|
81
|
+
if (conditions.length > 0) {
|
|
82
|
+
const whereClause = conditions.join(" AND ");
|
|
83
|
+
query.where(whereClause);
|
|
84
|
+
}
|
|
85
|
+
const records = await query.toArray();
|
|
86
|
+
const evals = records.sort((a, b) => b.created_at - a.created_at).map((record) => {
|
|
87
|
+
return {
|
|
88
|
+
id: record.id,
|
|
89
|
+
input: record.input,
|
|
90
|
+
output: record.output,
|
|
91
|
+
agentName: record.agent_name,
|
|
92
|
+
metricName: record.metric_name,
|
|
93
|
+
result: JSON.parse(record.result),
|
|
94
|
+
instructions: record.instructions,
|
|
95
|
+
testInfo: record.test_info ? JSON.parse(record.test_info) : null,
|
|
96
|
+
globalRunId: record.global_run_id,
|
|
97
|
+
runId: record.run_id,
|
|
98
|
+
createdAt: new Date(record.created_at).toISOString()
|
|
99
|
+
};
|
|
100
|
+
});
|
|
101
|
+
const page = options.page || 0;
|
|
102
|
+
const perPage = options.perPage || 10;
|
|
103
|
+
const pagedEvals = evals.slice(page * perPage, (page + 1) * perPage);
|
|
104
|
+
return {
|
|
105
|
+
evals: pagedEvals,
|
|
106
|
+
total,
|
|
107
|
+
page,
|
|
108
|
+
perPage,
|
|
109
|
+
hasMore: total > (page + 1) * perPage
|
|
110
|
+
};
|
|
111
|
+
} catch (error) {
|
|
112
|
+
throw new MastraError(
|
|
113
|
+
{
|
|
114
|
+
id: "LANCE_STORE_GET_EVALS_FAILED",
|
|
115
|
+
domain: ErrorDomain.STORAGE,
|
|
116
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
117
|
+
details: { agentName: options.agentName ?? "" }
|
|
118
|
+
},
|
|
119
|
+
error
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
function getPrimaryKeys(tableName) {
|
|
125
|
+
let primaryId = ["id"];
|
|
126
|
+
if (tableName === TABLE_WORKFLOW_SNAPSHOT) {
|
|
127
|
+
primaryId = ["workflow_name", "run_id"];
|
|
128
|
+
} else if (tableName === TABLE_EVALS) {
|
|
129
|
+
primaryId = ["agent_name", "metric_name", "run_id"];
|
|
130
|
+
}
|
|
131
|
+
return primaryId;
|
|
132
|
+
}
|
|
133
|
+
function validateKeyTypes(keys, tableSchema) {
|
|
134
|
+
const fieldTypes = new Map(
|
|
135
|
+
tableSchema.fields.map((field) => [field.name, field.type?.toString().toLowerCase()])
|
|
136
|
+
);
|
|
137
|
+
for (const [key, value] of Object.entries(keys)) {
|
|
138
|
+
const fieldType = fieldTypes.get(key);
|
|
139
|
+
if (!fieldType) {
|
|
140
|
+
throw new Error(`Field '${key}' does not exist in table schema`);
|
|
141
|
+
}
|
|
142
|
+
if (value !== null) {
|
|
143
|
+
if ((fieldType.includes("int") || fieldType.includes("bigint")) && typeof value !== "number") {
|
|
144
|
+
throw new Error(`Expected numeric value for field '${key}', got ${typeof value}`);
|
|
145
|
+
}
|
|
146
|
+
if (fieldType.includes("utf8") && typeof value !== "string") {
|
|
147
|
+
throw new Error(`Expected string value for field '${key}', got ${typeof value}`);
|
|
148
|
+
}
|
|
149
|
+
if (fieldType.includes("timestamp") && !(value instanceof Date) && typeof value !== "string") {
|
|
150
|
+
throw new Error(`Expected Date or string value for field '${key}', got ${typeof value}`);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
function processResultWithTypeConversion(rawResult, tableSchema) {
|
|
156
|
+
const fieldTypeMap = /* @__PURE__ */ new Map();
|
|
157
|
+
tableSchema.fields.forEach((field) => {
|
|
158
|
+
const fieldName = field.name;
|
|
159
|
+
const fieldTypeStr = field.type.toString().toLowerCase();
|
|
160
|
+
fieldTypeMap.set(fieldName, fieldTypeStr);
|
|
161
|
+
});
|
|
162
|
+
if (Array.isArray(rawResult)) {
|
|
163
|
+
return rawResult.map((item) => processResultWithTypeConversion(item, tableSchema));
|
|
164
|
+
}
|
|
165
|
+
const processedResult = { ...rawResult };
|
|
166
|
+
for (const key in processedResult) {
|
|
167
|
+
const fieldTypeStr = fieldTypeMap.get(key);
|
|
168
|
+
if (!fieldTypeStr) continue;
|
|
169
|
+
if (typeof processedResult[key] === "string") {
|
|
170
|
+
if (fieldTypeStr.includes("int32") || fieldTypeStr.includes("float32")) {
|
|
171
|
+
if (!isNaN(Number(processedResult[key]))) {
|
|
172
|
+
processedResult[key] = Number(processedResult[key]);
|
|
173
|
+
}
|
|
174
|
+
} else if (fieldTypeStr.includes("int64")) {
|
|
175
|
+
processedResult[key] = Number(processedResult[key]);
|
|
176
|
+
} else if (fieldTypeStr.includes("utf8") && key !== "id") {
|
|
177
|
+
try {
|
|
178
|
+
const parsed = JSON.parse(processedResult[key]);
|
|
179
|
+
if (typeof parsed === "object") {
|
|
180
|
+
processedResult[key] = JSON.parse(processedResult[key]);
|
|
181
|
+
}
|
|
182
|
+
} catch {
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
} else if (typeof processedResult[key] === "bigint") {
|
|
186
|
+
processedResult[key] = Number(processedResult[key]);
|
|
187
|
+
} else if (fieldTypeStr.includes("float64") && ["createdAt", "updatedAt"].includes(key)) {
|
|
188
|
+
processedResult[key] = new Date(processedResult[key]);
|
|
189
|
+
}
|
|
190
|
+
console.log(key, "processedResult", processedResult);
|
|
191
|
+
}
|
|
192
|
+
return processedResult;
|
|
193
|
+
}
|
|
194
|
+
async function getTableSchema({
|
|
195
|
+
tableName,
|
|
196
|
+
client
|
|
197
|
+
}) {
|
|
198
|
+
try {
|
|
199
|
+
if (!client) {
|
|
200
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
201
|
+
}
|
|
202
|
+
if (!tableName) {
|
|
203
|
+
throw new Error("tableName is required for getTableSchema.");
|
|
204
|
+
}
|
|
205
|
+
} catch (validationError) {
|
|
206
|
+
throw new MastraError(
|
|
207
|
+
{
|
|
208
|
+
id: "STORAGE_LANCE_STORAGE_GET_TABLE_SCHEMA_INVALID_ARGS",
|
|
209
|
+
domain: ErrorDomain.STORAGE,
|
|
210
|
+
category: ErrorCategory.USER,
|
|
211
|
+
text: validationError.message,
|
|
212
|
+
details: { tableName }
|
|
213
|
+
},
|
|
214
|
+
validationError
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
try {
|
|
218
|
+
const table = await client.openTable(tableName);
|
|
219
|
+
const rawSchema = await table.schema();
|
|
220
|
+
const fields = rawSchema.fields;
|
|
221
|
+
return {
|
|
222
|
+
fields,
|
|
223
|
+
metadata: /* @__PURE__ */ new Map(),
|
|
224
|
+
get names() {
|
|
225
|
+
return fields.map((field) => field.name);
|
|
226
|
+
}
|
|
227
|
+
};
|
|
228
|
+
} catch (error) {
|
|
229
|
+
throw new MastraError(
|
|
230
|
+
{
|
|
231
|
+
id: "STORAGE_LANCE_STORAGE_GET_TABLE_SCHEMA_FAILED",
|
|
232
|
+
domain: ErrorDomain.STORAGE,
|
|
233
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
234
|
+
details: { tableName }
|
|
235
|
+
},
|
|
236
|
+
error
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// src/storage/domains/memory/index.ts
|
|
242
|
+
var StoreMemoryLance = class extends MemoryStorage {
|
|
243
|
+
client;
|
|
244
|
+
operations;
|
|
245
|
+
constructor({ client, operations }) {
|
|
246
|
+
super();
|
|
247
|
+
this.client = client;
|
|
248
|
+
this.operations = operations;
|
|
249
|
+
}
|
|
250
|
+
async getThreadById({ threadId }) {
|
|
251
|
+
try {
|
|
252
|
+
const thread = await this.operations.load({ tableName: TABLE_THREADS, keys: { id: threadId } });
|
|
253
|
+
if (!thread) {
|
|
254
|
+
return null;
|
|
255
|
+
}
|
|
256
|
+
return {
|
|
257
|
+
...thread,
|
|
258
|
+
createdAt: new Date(thread.createdAt),
|
|
259
|
+
updatedAt: new Date(thread.updatedAt)
|
|
260
|
+
};
|
|
261
|
+
} catch (error) {
|
|
262
|
+
throw new MastraError(
|
|
263
|
+
{
|
|
264
|
+
id: "LANCE_STORE_GET_THREAD_BY_ID_FAILED",
|
|
265
|
+
domain: ErrorDomain.STORAGE,
|
|
266
|
+
category: ErrorCategory.THIRD_PARTY
|
|
267
|
+
},
|
|
268
|
+
error
|
|
269
|
+
);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
273
|
+
try {
|
|
274
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
275
|
+
const query = table.query().where(`\`resourceId\` = '${resourceId}'`);
|
|
276
|
+
const records = await query.toArray();
|
|
277
|
+
return processResultWithTypeConversion(
|
|
278
|
+
records,
|
|
279
|
+
await getTableSchema({ tableName: TABLE_THREADS, client: this.client })
|
|
280
|
+
);
|
|
281
|
+
} catch (error) {
|
|
282
|
+
throw new MastraError(
|
|
283
|
+
{
|
|
284
|
+
id: "LANCE_STORE_GET_THREADS_BY_RESOURCE_ID_FAILED",
|
|
285
|
+
domain: ErrorDomain.STORAGE,
|
|
286
|
+
category: ErrorCategory.THIRD_PARTY
|
|
287
|
+
},
|
|
288
|
+
error
|
|
289
|
+
);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
294
|
+
* @param thread - The thread to save
|
|
295
|
+
* @returns The saved thread
|
|
296
|
+
*/
|
|
297
|
+
async saveThread({ thread }) {
|
|
298
|
+
try {
|
|
299
|
+
const record = { ...thread, metadata: JSON.stringify(thread.metadata) };
|
|
300
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
301
|
+
await table.add([record], { mode: "append" });
|
|
302
|
+
return thread;
|
|
303
|
+
} catch (error) {
|
|
304
|
+
throw new MastraError(
|
|
305
|
+
{
|
|
306
|
+
id: "LANCE_STORE_SAVE_THREAD_FAILED",
|
|
307
|
+
domain: ErrorDomain.STORAGE,
|
|
308
|
+
category: ErrorCategory.THIRD_PARTY
|
|
309
|
+
},
|
|
310
|
+
error
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
async updateThread({
|
|
315
|
+
id,
|
|
316
|
+
title,
|
|
317
|
+
metadata
|
|
318
|
+
}) {
|
|
319
|
+
const maxRetries = 5;
|
|
320
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
321
|
+
try {
|
|
322
|
+
const current = await this.getThreadById({ threadId: id });
|
|
323
|
+
if (!current) {
|
|
324
|
+
throw new Error(`Thread with id ${id} not found`);
|
|
325
|
+
}
|
|
326
|
+
const mergedMetadata = { ...current.metadata, ...metadata };
|
|
327
|
+
const record = {
|
|
328
|
+
id,
|
|
329
|
+
title,
|
|
330
|
+
metadata: JSON.stringify(mergedMetadata),
|
|
331
|
+
updatedAt: (/* @__PURE__ */ new Date()).getTime()
|
|
332
|
+
};
|
|
333
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
334
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
335
|
+
const updatedThread = await this.getThreadById({ threadId: id });
|
|
336
|
+
if (!updatedThread) {
|
|
337
|
+
throw new Error(`Failed to retrieve updated thread ${id}`);
|
|
338
|
+
}
|
|
339
|
+
return updatedThread;
|
|
340
|
+
} catch (error) {
|
|
341
|
+
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
342
|
+
const delay = Math.pow(2, attempt) * 10;
|
|
343
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
throw new MastraError(
|
|
347
|
+
{
|
|
348
|
+
id: "LANCE_STORE_UPDATE_THREAD_FAILED",
|
|
349
|
+
domain: ErrorDomain.STORAGE,
|
|
350
|
+
category: ErrorCategory.THIRD_PARTY
|
|
351
|
+
},
|
|
352
|
+
error
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
throw new MastraError(
|
|
357
|
+
{
|
|
358
|
+
id: "LANCE_STORE_UPDATE_THREAD_FAILED",
|
|
359
|
+
domain: ErrorDomain.STORAGE,
|
|
360
|
+
category: ErrorCategory.THIRD_PARTY
|
|
361
|
+
},
|
|
362
|
+
new Error("All retries exhausted")
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
async deleteThread({ threadId }) {
|
|
366
|
+
try {
|
|
367
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
368
|
+
await table.delete(`id = '${threadId}'`);
|
|
369
|
+
const messagesTable = await this.client.openTable(TABLE_MESSAGES);
|
|
370
|
+
await messagesTable.delete(`thread_id = '${threadId}'`);
|
|
371
|
+
} catch (error) {
|
|
372
|
+
throw new MastraError(
|
|
373
|
+
{
|
|
374
|
+
id: "LANCE_STORE_DELETE_THREAD_FAILED",
|
|
375
|
+
domain: ErrorDomain.STORAGE,
|
|
376
|
+
category: ErrorCategory.THIRD_PARTY
|
|
377
|
+
},
|
|
378
|
+
error
|
|
379
|
+
);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
normalizeMessage(message) {
|
|
383
|
+
const { thread_id, ...rest } = message;
|
|
384
|
+
return {
|
|
385
|
+
...rest,
|
|
386
|
+
threadId: thread_id,
|
|
387
|
+
content: typeof message.content === "string" ? (() => {
|
|
388
|
+
try {
|
|
389
|
+
return JSON.parse(message.content);
|
|
390
|
+
} catch {
|
|
391
|
+
return message.content;
|
|
392
|
+
}
|
|
393
|
+
})() : message.content
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
async getMessages({
|
|
397
|
+
threadId,
|
|
398
|
+
resourceId,
|
|
399
|
+
selectBy,
|
|
400
|
+
format,
|
|
401
|
+
threadConfig
|
|
402
|
+
}) {
|
|
403
|
+
try {
|
|
404
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
405
|
+
if (threadConfig) {
|
|
406
|
+
throw new Error("ThreadConfig is not supported by LanceDB storage");
|
|
407
|
+
}
|
|
408
|
+
const limit = resolveMessageLimit({ last: selectBy?.last, defaultLimit: Number.MAX_SAFE_INTEGER });
|
|
409
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
410
|
+
let allRecords = [];
|
|
411
|
+
if (selectBy?.include && selectBy.include.length > 0) {
|
|
412
|
+
const threadIds = [...new Set(selectBy.include.map((item) => item.threadId))];
|
|
413
|
+
for (const threadId2 of threadIds) {
|
|
414
|
+
const threadQuery = table.query().where(`thread_id = '${threadId2}'`);
|
|
415
|
+
let threadRecords = await threadQuery.toArray();
|
|
416
|
+
allRecords.push(...threadRecords);
|
|
417
|
+
}
|
|
418
|
+
} else {
|
|
419
|
+
let query = table.query().where(`\`thread_id\` = '${threadId}'`);
|
|
420
|
+
allRecords = await query.toArray();
|
|
421
|
+
}
|
|
422
|
+
allRecords.sort((a, b) => {
|
|
423
|
+
const dateA = new Date(a.createdAt).getTime();
|
|
424
|
+
const dateB = new Date(b.createdAt).getTime();
|
|
425
|
+
return dateA - dateB;
|
|
426
|
+
});
|
|
427
|
+
if (selectBy?.include && selectBy.include.length > 0) {
|
|
428
|
+
allRecords = this.processMessagesWithContext(allRecords, selectBy.include);
|
|
429
|
+
}
|
|
430
|
+
if (limit !== Number.MAX_SAFE_INTEGER) {
|
|
431
|
+
allRecords = allRecords.slice(-limit);
|
|
432
|
+
}
|
|
433
|
+
const messages = processResultWithTypeConversion(
|
|
434
|
+
allRecords,
|
|
435
|
+
await getTableSchema({ tableName: TABLE_MESSAGES, client: this.client })
|
|
436
|
+
);
|
|
437
|
+
const list = new MessageList({ threadId, resourceId }).add(messages.map(this.normalizeMessage), "memory");
|
|
438
|
+
if (format === "v2") return list.get.all.v2();
|
|
439
|
+
return list.get.all.v1();
|
|
440
|
+
} catch (error) {
|
|
441
|
+
throw new MastraError(
|
|
442
|
+
{
|
|
443
|
+
id: "LANCE_STORE_GET_MESSAGES_FAILED",
|
|
444
|
+
domain: ErrorDomain.STORAGE,
|
|
445
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
446
|
+
details: {
|
|
447
|
+
threadId,
|
|
448
|
+
resourceId: resourceId ?? ""
|
|
449
|
+
}
|
|
450
|
+
},
|
|
451
|
+
error
|
|
452
|
+
);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
async getMessagesById({
|
|
456
|
+
messageIds,
|
|
457
|
+
format
|
|
458
|
+
}) {
|
|
459
|
+
if (messageIds.length === 0) return [];
|
|
460
|
+
try {
|
|
461
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
462
|
+
const quotedIds = messageIds.map((id) => `'${id}'`).join(", ");
|
|
463
|
+
const allRecords = await table.query().where(`id IN (${quotedIds})`).toArray();
|
|
464
|
+
const messages = processResultWithTypeConversion(
|
|
465
|
+
allRecords,
|
|
466
|
+
await getTableSchema({ tableName: TABLE_MESSAGES, client: this.client })
|
|
467
|
+
);
|
|
468
|
+
const list = new MessageList().add(messages.map(this.normalizeMessage), "memory");
|
|
469
|
+
if (format === `v1`) return list.get.all.v1();
|
|
470
|
+
return list.get.all.v2();
|
|
471
|
+
} catch (error) {
|
|
472
|
+
throw new MastraError(
|
|
473
|
+
{
|
|
474
|
+
id: "LANCE_STORE_GET_MESSAGES_BY_ID_FAILED",
|
|
475
|
+
domain: ErrorDomain.STORAGE,
|
|
476
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
477
|
+
details: {
|
|
478
|
+
messageIds: JSON.stringify(messageIds)
|
|
479
|
+
}
|
|
480
|
+
},
|
|
481
|
+
error
|
|
482
|
+
);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
async saveMessages(args) {
|
|
486
|
+
try {
|
|
487
|
+
const { messages, format = "v1" } = args;
|
|
488
|
+
if (messages.length === 0) {
|
|
489
|
+
return [];
|
|
490
|
+
}
|
|
491
|
+
const threadId = messages[0]?.threadId;
|
|
492
|
+
if (!threadId) {
|
|
493
|
+
throw new Error("Thread ID is required");
|
|
494
|
+
}
|
|
495
|
+
for (const message of messages) {
|
|
496
|
+
if (!message.id) {
|
|
497
|
+
throw new Error("Message ID is required");
|
|
498
|
+
}
|
|
499
|
+
if (!message.threadId) {
|
|
500
|
+
throw new Error("Thread ID is required for all messages");
|
|
501
|
+
}
|
|
502
|
+
if (message.resourceId === null || message.resourceId === void 0) {
|
|
503
|
+
throw new Error("Resource ID cannot be null or undefined");
|
|
504
|
+
}
|
|
505
|
+
if (!message.content) {
|
|
506
|
+
throw new Error("Message content is required");
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
const transformedMessages = messages.map((message) => {
|
|
510
|
+
const { threadId: threadId2, type, ...rest } = message;
|
|
511
|
+
return {
|
|
512
|
+
...rest,
|
|
513
|
+
thread_id: threadId2,
|
|
514
|
+
type: type ?? "v2",
|
|
515
|
+
content: JSON.stringify(message.content)
|
|
516
|
+
};
|
|
517
|
+
});
|
|
518
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
519
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(transformedMessages);
|
|
520
|
+
const threadsTable = await this.client.openTable(TABLE_THREADS);
|
|
521
|
+
const currentTime = (/* @__PURE__ */ new Date()).getTime();
|
|
522
|
+
const updateRecord = { id: threadId, updatedAt: currentTime };
|
|
523
|
+
await threadsTable.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([updateRecord]);
|
|
524
|
+
const list = new MessageList().add(messages, "memory");
|
|
525
|
+
if (format === `v2`) return list.get.all.v2();
|
|
526
|
+
return list.get.all.v1();
|
|
527
|
+
} catch (error) {
|
|
528
|
+
throw new MastraError(
|
|
529
|
+
{
|
|
530
|
+
id: "LANCE_STORE_SAVE_MESSAGES_FAILED",
|
|
531
|
+
domain: ErrorDomain.STORAGE,
|
|
532
|
+
category: ErrorCategory.THIRD_PARTY
|
|
533
|
+
},
|
|
534
|
+
error
|
|
535
|
+
);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
539
|
+
try {
|
|
540
|
+
const { resourceId, page = 0, perPage = 10 } = args;
|
|
541
|
+
const table = await this.client.openTable(TABLE_THREADS);
|
|
542
|
+
const total = await table.countRows(`\`resourceId\` = '${resourceId}'`);
|
|
543
|
+
const query = table.query().where(`\`resourceId\` = '${resourceId}'`);
|
|
544
|
+
const offset = page * perPage;
|
|
545
|
+
query.limit(perPage);
|
|
546
|
+
if (offset > 0) {
|
|
547
|
+
query.offset(offset);
|
|
548
|
+
}
|
|
549
|
+
const records = await query.toArray();
|
|
550
|
+
records.sort((a, b) => new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime());
|
|
551
|
+
const schema = await getTableSchema({ tableName: TABLE_THREADS, client: this.client });
|
|
552
|
+
const threads = records.map((record) => processResultWithTypeConversion(record, schema));
|
|
553
|
+
return {
|
|
554
|
+
threads,
|
|
555
|
+
total,
|
|
556
|
+
page,
|
|
557
|
+
perPage,
|
|
558
|
+
hasMore: total > (page + 1) * perPage
|
|
559
|
+
};
|
|
560
|
+
} catch (error) {
|
|
561
|
+
throw new MastraError(
|
|
562
|
+
{
|
|
563
|
+
id: "LANCE_STORE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
564
|
+
domain: ErrorDomain.STORAGE,
|
|
565
|
+
category: ErrorCategory.THIRD_PARTY
|
|
566
|
+
},
|
|
567
|
+
error
|
|
568
|
+
);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
/**
|
|
572
|
+
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
573
|
+
* @param records - The sorted array of records to process
|
|
574
|
+
* @param include - The array of include specifications with context parameters
|
|
575
|
+
* @returns The processed array with context messages included
|
|
576
|
+
*/
|
|
577
|
+
processMessagesWithContext(records, include) {
|
|
578
|
+
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
579
|
+
if (messagesWithContext.length === 0) {
|
|
580
|
+
return records;
|
|
581
|
+
}
|
|
582
|
+
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
583
|
+
records.forEach((message, index) => {
|
|
584
|
+
messageIndexMap.set(message.id, index);
|
|
585
|
+
});
|
|
586
|
+
const additionalIndices = /* @__PURE__ */ new Set();
|
|
587
|
+
for (const item of messagesWithContext) {
|
|
588
|
+
const messageIndex = messageIndexMap.get(item.id);
|
|
589
|
+
if (messageIndex !== void 0) {
|
|
590
|
+
if (item.withPreviousMessages) {
|
|
591
|
+
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
592
|
+
for (let i = startIdx; i < messageIndex; i++) {
|
|
593
|
+
additionalIndices.add(i);
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
if (item.withNextMessages) {
|
|
597
|
+
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
598
|
+
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
599
|
+
additionalIndices.add(i);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
if (additionalIndices.size === 0) {
|
|
605
|
+
return records;
|
|
606
|
+
}
|
|
607
|
+
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
608
|
+
const allIndices = /* @__PURE__ */ new Set();
|
|
609
|
+
records.forEach((record, index) => {
|
|
610
|
+
if (originalMatchIds.has(record.id)) {
|
|
611
|
+
allIndices.add(index);
|
|
612
|
+
}
|
|
613
|
+
});
|
|
614
|
+
additionalIndices.forEach((index) => {
|
|
615
|
+
allIndices.add(index);
|
|
616
|
+
});
|
|
617
|
+
return Array.from(allIndices).sort((a, b) => a - b).map((index) => records[index]);
|
|
618
|
+
}
|
|
619
|
+
async getMessagesPaginated(args) {
|
|
620
|
+
const { threadId, resourceId, selectBy, format = "v1" } = args;
|
|
621
|
+
const page = selectBy?.pagination?.page ?? 0;
|
|
622
|
+
const perPage = selectBy?.pagination?.perPage ?? 10;
|
|
623
|
+
try {
|
|
624
|
+
if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
|
|
625
|
+
const dateRange = selectBy?.pagination?.dateRange;
|
|
626
|
+
const fromDate = dateRange?.start;
|
|
627
|
+
const toDate = dateRange?.end;
|
|
628
|
+
const table = await this.client.openTable(TABLE_MESSAGES);
|
|
629
|
+
const messages = [];
|
|
630
|
+
if (selectBy?.include && Array.isArray(selectBy.include)) {
|
|
631
|
+
const threadIds = [...new Set(selectBy.include.map((item) => item.threadId))];
|
|
632
|
+
const allThreadMessages = [];
|
|
633
|
+
for (const threadId2 of threadIds) {
|
|
634
|
+
const threadQuery = table.query().where(`thread_id = '${threadId2}'`);
|
|
635
|
+
let threadRecords = await threadQuery.toArray();
|
|
636
|
+
if (fromDate) threadRecords = threadRecords.filter((m) => m.createdAt >= fromDate.getTime());
|
|
637
|
+
if (toDate) threadRecords = threadRecords.filter((m) => m.createdAt <= toDate.getTime());
|
|
638
|
+
allThreadMessages.push(...threadRecords);
|
|
639
|
+
}
|
|
640
|
+
allThreadMessages.sort((a, b) => a.createdAt - b.createdAt);
|
|
641
|
+
const contextMessages = this.processMessagesWithContext(allThreadMessages, selectBy.include);
|
|
642
|
+
messages.push(...contextMessages);
|
|
643
|
+
}
|
|
644
|
+
const conditions = [`thread_id = '${threadId}'`];
|
|
645
|
+
if (resourceId) {
|
|
646
|
+
conditions.push(`\`resourceId\` = '${resourceId}'`);
|
|
647
|
+
}
|
|
648
|
+
if (fromDate) {
|
|
649
|
+
conditions.push(`\`createdAt\` >= ${fromDate.getTime()}`);
|
|
650
|
+
}
|
|
651
|
+
if (toDate) {
|
|
652
|
+
conditions.push(`\`createdAt\` <= ${toDate.getTime()}`);
|
|
653
|
+
}
|
|
654
|
+
let total = 0;
|
|
655
|
+
if (conditions.length > 0) {
|
|
656
|
+
total = await table.countRows(conditions.join(" AND "));
|
|
657
|
+
} else {
|
|
658
|
+
total = await table.countRows();
|
|
659
|
+
}
|
|
660
|
+
if (total === 0 && messages.length === 0) {
|
|
661
|
+
return {
|
|
662
|
+
messages: [],
|
|
663
|
+
total: 0,
|
|
664
|
+
page,
|
|
665
|
+
perPage,
|
|
666
|
+
hasMore: false
|
|
667
|
+
};
|
|
668
|
+
}
|
|
669
|
+
const excludeIds = messages.map((m) => m.id);
|
|
670
|
+
let selectedMessages = [];
|
|
671
|
+
if (selectBy?.last && selectBy.last > 0) {
|
|
672
|
+
const query = table.query();
|
|
673
|
+
if (conditions.length > 0) {
|
|
674
|
+
query.where(conditions.join(" AND "));
|
|
675
|
+
}
|
|
676
|
+
let records = await query.toArray();
|
|
677
|
+
records = records.sort((a, b) => a.createdAt - b.createdAt);
|
|
678
|
+
if (excludeIds.length > 0) {
|
|
679
|
+
records = records.filter((m) => !excludeIds.includes(m.id));
|
|
680
|
+
}
|
|
681
|
+
selectedMessages = records.slice(-selectBy.last);
|
|
682
|
+
} else {
|
|
683
|
+
const query = table.query();
|
|
684
|
+
if (conditions.length > 0) {
|
|
685
|
+
query.where(conditions.join(" AND "));
|
|
686
|
+
}
|
|
687
|
+
let records = await query.toArray();
|
|
688
|
+
records = records.sort((a, b) => a.createdAt - b.createdAt);
|
|
689
|
+
if (excludeIds.length > 0) {
|
|
690
|
+
records = records.filter((m) => !excludeIds.includes(m.id));
|
|
691
|
+
}
|
|
692
|
+
selectedMessages = records.slice(page * perPage, (page + 1) * perPage);
|
|
693
|
+
}
|
|
694
|
+
const allMessages = [...messages, ...selectedMessages];
|
|
695
|
+
const seen = /* @__PURE__ */ new Set();
|
|
696
|
+
const dedupedMessages = allMessages.filter((m) => {
|
|
697
|
+
const key = `${m.id}:${m.thread_id}`;
|
|
698
|
+
if (seen.has(key)) return false;
|
|
699
|
+
seen.add(key);
|
|
700
|
+
return true;
|
|
701
|
+
});
|
|
702
|
+
const formattedMessages = dedupedMessages.map((msg) => {
|
|
703
|
+
const { thread_id, ...rest } = msg;
|
|
704
|
+
return {
|
|
705
|
+
...rest,
|
|
706
|
+
threadId: thread_id,
|
|
707
|
+
content: typeof msg.content === "string" ? (() => {
|
|
708
|
+
try {
|
|
709
|
+
return JSON.parse(msg.content);
|
|
710
|
+
} catch {
|
|
711
|
+
return msg.content;
|
|
712
|
+
}
|
|
713
|
+
})() : msg.content
|
|
714
|
+
};
|
|
715
|
+
});
|
|
716
|
+
const list = new MessageList().add(formattedMessages, "memory");
|
|
717
|
+
return {
|
|
718
|
+
messages: format === "v2" ? list.get.all.v2() : list.get.all.v1(),
|
|
719
|
+
total,
|
|
720
|
+
// Total should be the count of messages matching the filters
|
|
721
|
+
page,
|
|
722
|
+
perPage,
|
|
723
|
+
hasMore: total > (page + 1) * perPage
|
|
724
|
+
};
|
|
725
|
+
} catch (error) {
|
|
726
|
+
const mastraError = new MastraError(
|
|
727
|
+
{
|
|
728
|
+
id: "LANCE_STORE_GET_MESSAGES_PAGINATED_FAILED",
|
|
729
|
+
domain: ErrorDomain.STORAGE,
|
|
730
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
731
|
+
details: {
|
|
732
|
+
threadId,
|
|
733
|
+
resourceId: resourceId ?? ""
|
|
734
|
+
}
|
|
735
|
+
},
|
|
736
|
+
error
|
|
737
|
+
);
|
|
738
|
+
this.logger?.trackException?.(mastraError);
|
|
739
|
+
this.logger?.error?.(mastraError.toString());
|
|
740
|
+
return { messages: [], total: 0, page, perPage, hasMore: false };
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
/**
|
|
744
|
+
* Parse message data from LanceDB record format to MastraMessageV2 format
|
|
745
|
+
*/
|
|
746
|
+
parseMessageData(data) {
|
|
747
|
+
const { thread_id, ...rest } = data;
|
|
748
|
+
return {
|
|
749
|
+
...rest,
|
|
750
|
+
threadId: thread_id,
|
|
751
|
+
content: typeof data.content === "string" ? (() => {
|
|
752
|
+
try {
|
|
753
|
+
return JSON.parse(data.content);
|
|
754
|
+
} catch {
|
|
755
|
+
return data.content;
|
|
756
|
+
}
|
|
757
|
+
})() : data.content,
|
|
758
|
+
createdAt: new Date(data.createdAt),
|
|
759
|
+
updatedAt: new Date(data.updatedAt)
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
async updateMessages(args) {
|
|
763
|
+
const { messages } = args;
|
|
764
|
+
this.logger.debug("Updating messages", { count: messages.length });
|
|
765
|
+
if (!messages.length) {
|
|
766
|
+
return [];
|
|
767
|
+
}
|
|
768
|
+
const updatedMessages = [];
|
|
769
|
+
const affectedThreadIds = /* @__PURE__ */ new Set();
|
|
770
|
+
try {
|
|
771
|
+
for (const updateData of messages) {
|
|
772
|
+
const { id, ...updates } = updateData;
|
|
773
|
+
const existingMessage = await this.operations.load({ tableName: TABLE_MESSAGES, keys: { id } });
|
|
774
|
+
if (!existingMessage) {
|
|
775
|
+
this.logger.warn("Message not found for update", { id });
|
|
776
|
+
continue;
|
|
777
|
+
}
|
|
778
|
+
const existingMsg = this.parseMessageData(existingMessage);
|
|
779
|
+
const originalThreadId = existingMsg.threadId;
|
|
780
|
+
affectedThreadIds.add(originalThreadId);
|
|
781
|
+
const updatePayload = {};
|
|
782
|
+
if ("role" in updates && updates.role !== void 0) updatePayload.role = updates.role;
|
|
783
|
+
if ("type" in updates && updates.type !== void 0) updatePayload.type = updates.type;
|
|
784
|
+
if ("resourceId" in updates && updates.resourceId !== void 0) updatePayload.resourceId = updates.resourceId;
|
|
785
|
+
if ("threadId" in updates && updates.threadId !== void 0 && updates.threadId !== null) {
|
|
786
|
+
updatePayload.thread_id = updates.threadId;
|
|
787
|
+
affectedThreadIds.add(updates.threadId);
|
|
788
|
+
}
|
|
789
|
+
if (updates.content) {
|
|
790
|
+
const existingContent = existingMsg.content;
|
|
791
|
+
let newContent = { ...existingContent };
|
|
792
|
+
if (updates.content.metadata !== void 0) {
|
|
793
|
+
newContent.metadata = {
|
|
794
|
+
...existingContent.metadata || {},
|
|
795
|
+
...updates.content.metadata || {}
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
if (updates.content.content !== void 0) {
|
|
799
|
+
newContent.content = updates.content.content;
|
|
800
|
+
}
|
|
801
|
+
if ("parts" in updates.content && updates.content.parts !== void 0) {
|
|
802
|
+
newContent.parts = updates.content.parts;
|
|
803
|
+
}
|
|
804
|
+
updatePayload.content = JSON.stringify(newContent);
|
|
805
|
+
}
|
|
806
|
+
await this.operations.insert({ tableName: TABLE_MESSAGES, record: { id, ...updatePayload } });
|
|
807
|
+
const updatedMessage = await this.operations.load({ tableName: TABLE_MESSAGES, keys: { id } });
|
|
808
|
+
if (updatedMessage) {
|
|
809
|
+
updatedMessages.push(this.parseMessageData(updatedMessage));
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
for (const threadId of affectedThreadIds) {
|
|
813
|
+
await this.operations.insert({
|
|
814
|
+
tableName: TABLE_THREADS,
|
|
815
|
+
record: { id: threadId, updatedAt: Date.now() }
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
return updatedMessages;
|
|
819
|
+
} catch (error) {
|
|
820
|
+
throw new MastraError(
|
|
821
|
+
{
|
|
822
|
+
id: "LANCE_STORE_UPDATE_MESSAGES_FAILED",
|
|
823
|
+
domain: ErrorDomain.STORAGE,
|
|
824
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
825
|
+
details: { count: messages.length }
|
|
826
|
+
},
|
|
827
|
+
error
|
|
828
|
+
);
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
async getResourceById({ resourceId }) {
|
|
832
|
+
try {
|
|
833
|
+
const resource = await this.operations.load({ tableName: TABLE_RESOURCES, keys: { id: resourceId } });
|
|
834
|
+
if (!resource) {
|
|
835
|
+
return null;
|
|
836
|
+
}
|
|
837
|
+
let createdAt;
|
|
838
|
+
let updatedAt;
|
|
839
|
+
try {
|
|
840
|
+
if (resource.createdAt instanceof Date) {
|
|
841
|
+
createdAt = resource.createdAt;
|
|
842
|
+
} else if (typeof resource.createdAt === "string") {
|
|
843
|
+
createdAt = new Date(resource.createdAt);
|
|
844
|
+
} else if (typeof resource.createdAt === "number") {
|
|
845
|
+
createdAt = new Date(resource.createdAt);
|
|
846
|
+
} else {
|
|
847
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
848
|
+
}
|
|
849
|
+
if (isNaN(createdAt.getTime())) {
|
|
850
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
851
|
+
}
|
|
852
|
+
} catch {
|
|
853
|
+
createdAt = /* @__PURE__ */ new Date();
|
|
854
|
+
}
|
|
855
|
+
try {
|
|
856
|
+
if (resource.updatedAt instanceof Date) {
|
|
857
|
+
updatedAt = resource.updatedAt;
|
|
858
|
+
} else if (typeof resource.updatedAt === "string") {
|
|
859
|
+
updatedAt = new Date(resource.updatedAt);
|
|
860
|
+
} else if (typeof resource.updatedAt === "number") {
|
|
861
|
+
updatedAt = new Date(resource.updatedAt);
|
|
862
|
+
} else {
|
|
863
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
864
|
+
}
|
|
865
|
+
if (isNaN(updatedAt.getTime())) {
|
|
866
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
867
|
+
}
|
|
868
|
+
} catch {
|
|
869
|
+
updatedAt = /* @__PURE__ */ new Date();
|
|
870
|
+
}
|
|
871
|
+
let workingMemory = resource.workingMemory;
|
|
872
|
+
if (workingMemory === null || workingMemory === void 0) {
|
|
873
|
+
workingMemory = void 0;
|
|
874
|
+
} else if (workingMemory === "") {
|
|
875
|
+
workingMemory = "";
|
|
876
|
+
} else if (typeof workingMemory === "object") {
|
|
877
|
+
workingMemory = JSON.stringify(workingMemory);
|
|
878
|
+
}
|
|
879
|
+
let metadata = resource.metadata;
|
|
880
|
+
if (metadata === "" || metadata === null || metadata === void 0) {
|
|
881
|
+
metadata = void 0;
|
|
882
|
+
} else if (typeof metadata === "string") {
|
|
883
|
+
try {
|
|
884
|
+
metadata = JSON.parse(metadata);
|
|
885
|
+
} catch {
|
|
886
|
+
metadata = metadata;
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
return {
|
|
890
|
+
...resource,
|
|
891
|
+
createdAt,
|
|
892
|
+
updatedAt,
|
|
893
|
+
workingMemory,
|
|
894
|
+
metadata
|
|
895
|
+
};
|
|
896
|
+
} catch (error) {
|
|
897
|
+
throw new MastraError(
|
|
898
|
+
{
|
|
899
|
+
id: "LANCE_STORE_GET_RESOURCE_BY_ID_FAILED",
|
|
900
|
+
domain: ErrorDomain.STORAGE,
|
|
901
|
+
category: ErrorCategory.THIRD_PARTY
|
|
902
|
+
},
|
|
903
|
+
error
|
|
904
|
+
);
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
async saveResource({ resource }) {
|
|
908
|
+
try {
|
|
909
|
+
const record = {
|
|
910
|
+
...resource,
|
|
911
|
+
metadata: resource.metadata ? JSON.stringify(resource.metadata) : "",
|
|
912
|
+
createdAt: resource.createdAt.getTime(),
|
|
913
|
+
// Store as timestamp (milliseconds)
|
|
914
|
+
updatedAt: resource.updatedAt.getTime()
|
|
915
|
+
// Store as timestamp (milliseconds)
|
|
916
|
+
};
|
|
917
|
+
const table = await this.client.openTable(TABLE_RESOURCES);
|
|
918
|
+
await table.add([record], { mode: "append" });
|
|
919
|
+
return resource;
|
|
920
|
+
} catch (error) {
|
|
921
|
+
throw new MastraError(
|
|
922
|
+
{
|
|
923
|
+
id: "LANCE_STORE_SAVE_RESOURCE_FAILED",
|
|
924
|
+
domain: ErrorDomain.STORAGE,
|
|
925
|
+
category: ErrorCategory.THIRD_PARTY
|
|
926
|
+
},
|
|
927
|
+
error
|
|
928
|
+
);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
async updateResource({
|
|
932
|
+
resourceId,
|
|
933
|
+
workingMemory,
|
|
934
|
+
metadata
|
|
935
|
+
}) {
|
|
936
|
+
const maxRetries = 3;
|
|
937
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
938
|
+
try {
|
|
939
|
+
const existingResource = await this.getResourceById({ resourceId });
|
|
940
|
+
if (!existingResource) {
|
|
941
|
+
const newResource = {
|
|
942
|
+
id: resourceId,
|
|
943
|
+
workingMemory,
|
|
944
|
+
metadata: metadata || {},
|
|
945
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
946
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
947
|
+
};
|
|
948
|
+
return this.saveResource({ resource: newResource });
|
|
949
|
+
}
|
|
950
|
+
const updatedResource = {
|
|
951
|
+
...existingResource,
|
|
952
|
+
workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
|
|
953
|
+
metadata: {
|
|
954
|
+
...existingResource.metadata,
|
|
955
|
+
...metadata
|
|
956
|
+
},
|
|
957
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
958
|
+
};
|
|
959
|
+
const record = {
|
|
960
|
+
id: resourceId,
|
|
961
|
+
workingMemory: updatedResource.workingMemory || "",
|
|
962
|
+
metadata: updatedResource.metadata ? JSON.stringify(updatedResource.metadata) : "",
|
|
963
|
+
updatedAt: updatedResource.updatedAt.getTime()
|
|
964
|
+
// Store as timestamp (milliseconds)
|
|
965
|
+
};
|
|
966
|
+
const table = await this.client.openTable(TABLE_RESOURCES);
|
|
967
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
968
|
+
return updatedResource;
|
|
969
|
+
} catch (error) {
|
|
970
|
+
if (error.message?.includes("Commit conflict") && attempt < maxRetries - 1) {
|
|
971
|
+
const delay = Math.pow(2, attempt) * 10;
|
|
972
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
973
|
+
continue;
|
|
974
|
+
}
|
|
975
|
+
throw new MastraError(
|
|
976
|
+
{
|
|
977
|
+
id: "LANCE_STORE_UPDATE_RESOURCE_FAILED",
|
|
978
|
+
domain: ErrorDomain.STORAGE,
|
|
979
|
+
category: ErrorCategory.THIRD_PARTY
|
|
980
|
+
},
|
|
981
|
+
error
|
|
982
|
+
);
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
throw new Error("Unexpected end of retry loop");
|
|
986
|
+
}
|
|
987
|
+
};
|
|
988
|
+
var StoreOperationsLance = class extends StoreOperations {
|
|
989
|
+
client;
|
|
990
|
+
constructor({ client }) {
|
|
991
|
+
super();
|
|
992
|
+
this.client = client;
|
|
993
|
+
}
|
|
994
|
+
getDefaultValue(type) {
|
|
995
|
+
switch (type) {
|
|
996
|
+
case "text":
|
|
997
|
+
return "''";
|
|
998
|
+
case "timestamp":
|
|
999
|
+
return "CURRENT_TIMESTAMP";
|
|
1000
|
+
case "integer":
|
|
1001
|
+
case "bigint":
|
|
1002
|
+
return "0";
|
|
1003
|
+
case "jsonb":
|
|
1004
|
+
return "'{}'";
|
|
1005
|
+
case "uuid":
|
|
1006
|
+
return "''";
|
|
1007
|
+
default:
|
|
1008
|
+
return super.getDefaultValue(type);
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
async hasColumn(tableName, columnName) {
|
|
1012
|
+
const table = await this.client.openTable(tableName);
|
|
1013
|
+
const schema = await table.schema();
|
|
1014
|
+
return schema.fields.some((field) => field.name === columnName);
|
|
1015
|
+
}
|
|
1016
|
+
translateSchema(schema) {
|
|
1017
|
+
const fields = Object.entries(schema).map(([name, column]) => {
|
|
1018
|
+
let arrowType;
|
|
1019
|
+
switch (column.type.toLowerCase()) {
|
|
1020
|
+
case "text":
|
|
1021
|
+
case "uuid":
|
|
1022
|
+
arrowType = new Utf8();
|
|
1023
|
+
break;
|
|
1024
|
+
case "int":
|
|
1025
|
+
case "integer":
|
|
1026
|
+
arrowType = new Int32();
|
|
1027
|
+
break;
|
|
1028
|
+
case "bigint":
|
|
1029
|
+
arrowType = new Float64();
|
|
1030
|
+
break;
|
|
1031
|
+
case "float":
|
|
1032
|
+
arrowType = new Float32();
|
|
1033
|
+
break;
|
|
1034
|
+
case "jsonb":
|
|
1035
|
+
case "json":
|
|
1036
|
+
arrowType = new Utf8();
|
|
1037
|
+
break;
|
|
1038
|
+
case "binary":
|
|
1039
|
+
arrowType = new Binary();
|
|
1040
|
+
break;
|
|
1041
|
+
case "timestamp":
|
|
1042
|
+
arrowType = new Float64();
|
|
1043
|
+
break;
|
|
1044
|
+
default:
|
|
1045
|
+
arrowType = new Utf8();
|
|
1046
|
+
}
|
|
1047
|
+
return new Field(name, arrowType, column.nullable ?? true);
|
|
1048
|
+
});
|
|
1049
|
+
return new Schema(fields);
|
|
1050
|
+
}
|
|
1051
|
+
async createTable({
|
|
1052
|
+
tableName,
|
|
1053
|
+
schema
|
|
1054
|
+
}) {
|
|
1055
|
+
try {
|
|
1056
|
+
if (!this.client) {
|
|
1057
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1058
|
+
}
|
|
1059
|
+
if (!tableName) {
|
|
1060
|
+
throw new Error("tableName is required for createTable.");
|
|
1061
|
+
}
|
|
1062
|
+
if (!schema) {
|
|
1063
|
+
throw new Error("schema is required for createTable.");
|
|
1064
|
+
}
|
|
1065
|
+
} catch (error) {
|
|
1066
|
+
throw new MastraError(
|
|
1067
|
+
{
|
|
1068
|
+
id: "STORAGE_LANCE_STORAGE_CREATE_TABLE_INVALID_ARGS",
|
|
1069
|
+
domain: ErrorDomain.STORAGE,
|
|
1070
|
+
category: ErrorCategory.USER,
|
|
1071
|
+
details: { tableName }
|
|
1072
|
+
},
|
|
1073
|
+
error
|
|
1074
|
+
);
|
|
1075
|
+
}
|
|
1076
|
+
try {
|
|
1077
|
+
const arrowSchema = this.translateSchema(schema);
|
|
1078
|
+
await this.client.createEmptyTable(tableName, arrowSchema);
|
|
1079
|
+
} catch (error) {
|
|
1080
|
+
if (error.message?.includes("already exists")) {
|
|
1081
|
+
this.logger.debug(`Table '${tableName}' already exists, skipping create`);
|
|
1082
|
+
return;
|
|
1083
|
+
}
|
|
1084
|
+
throw new MastraError(
|
|
1085
|
+
{
|
|
1086
|
+
id: "STORAGE_LANCE_STORAGE_CREATE_TABLE_FAILED",
|
|
1087
|
+
domain: ErrorDomain.STORAGE,
|
|
1088
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1089
|
+
details: { tableName }
|
|
1090
|
+
},
|
|
1091
|
+
error
|
|
1092
|
+
);
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
async dropTable({ tableName }) {
|
|
1096
|
+
try {
|
|
1097
|
+
if (!this.client) {
|
|
1098
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1099
|
+
}
|
|
1100
|
+
if (!tableName) {
|
|
1101
|
+
throw new Error("tableName is required for dropTable.");
|
|
1102
|
+
}
|
|
1103
|
+
} catch (validationError) {
|
|
1104
|
+
throw new MastraError(
|
|
1105
|
+
{
|
|
1106
|
+
id: "STORAGE_LANCE_STORAGE_DROP_TABLE_INVALID_ARGS",
|
|
1107
|
+
domain: ErrorDomain.STORAGE,
|
|
1108
|
+
category: ErrorCategory.USER,
|
|
1109
|
+
text: validationError.message,
|
|
1110
|
+
details: { tableName }
|
|
1111
|
+
},
|
|
1112
|
+
validationError
|
|
1113
|
+
);
|
|
1114
|
+
}
|
|
1115
|
+
try {
|
|
1116
|
+
await this.client.dropTable(tableName);
|
|
1117
|
+
} catch (error) {
|
|
1118
|
+
if (error.toString().includes("was not found") || error.message?.includes("Table not found")) {
|
|
1119
|
+
this.logger.debug(`Table '${tableName}' does not exist, skipping drop`);
|
|
1120
|
+
return;
|
|
1121
|
+
}
|
|
1122
|
+
throw new MastraError(
|
|
1123
|
+
{
|
|
1124
|
+
id: "STORAGE_LANCE_STORAGE_DROP_TABLE_FAILED",
|
|
1125
|
+
domain: ErrorDomain.STORAGE,
|
|
1126
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1127
|
+
details: { tableName }
|
|
1128
|
+
},
|
|
1129
|
+
error
|
|
1130
|
+
);
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
async alterTable({
|
|
1134
|
+
tableName,
|
|
1135
|
+
schema,
|
|
1136
|
+
ifNotExists
|
|
1137
|
+
}) {
|
|
1138
|
+
try {
|
|
1139
|
+
if (!this.client) {
|
|
1140
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1141
|
+
}
|
|
1142
|
+
if (!tableName) {
|
|
1143
|
+
throw new Error("tableName is required for alterTable.");
|
|
1144
|
+
}
|
|
1145
|
+
if (!schema) {
|
|
1146
|
+
throw new Error("schema is required for alterTable.");
|
|
1147
|
+
}
|
|
1148
|
+
if (!ifNotExists || ifNotExists.length === 0) {
|
|
1149
|
+
this.logger.debug("No columns specified to add in alterTable, skipping.");
|
|
1150
|
+
return;
|
|
1151
|
+
}
|
|
1152
|
+
} catch (validationError) {
|
|
1153
|
+
throw new MastraError(
|
|
1154
|
+
{
|
|
1155
|
+
id: "STORAGE_LANCE_STORAGE_ALTER_TABLE_INVALID_ARGS",
|
|
1156
|
+
domain: ErrorDomain.STORAGE,
|
|
1157
|
+
category: ErrorCategory.USER,
|
|
1158
|
+
text: validationError.message,
|
|
1159
|
+
details: { tableName }
|
|
1160
|
+
},
|
|
1161
|
+
validationError
|
|
1162
|
+
);
|
|
1163
|
+
}
|
|
1164
|
+
try {
|
|
1165
|
+
const table = await this.client.openTable(tableName);
|
|
1166
|
+
const currentSchema = await table.schema();
|
|
1167
|
+
const existingFields = new Set(currentSchema.fields.map((f) => f.name));
|
|
1168
|
+
const typeMap = {
|
|
1169
|
+
text: "string",
|
|
1170
|
+
integer: "int",
|
|
1171
|
+
bigint: "bigint",
|
|
1172
|
+
timestamp: "timestamp",
|
|
1173
|
+
jsonb: "string",
|
|
1174
|
+
uuid: "string"
|
|
1175
|
+
};
|
|
1176
|
+
const columnsToAdd = ifNotExists.filter((col) => schema[col] && !existingFields.has(col)).map((col) => {
|
|
1177
|
+
const colDef = schema[col];
|
|
1178
|
+
return {
|
|
1179
|
+
name: col,
|
|
1180
|
+
valueSql: colDef?.nullable ? `cast(NULL as ${typeMap[colDef.type ?? "text"]})` : `cast(${this.getDefaultValue(colDef?.type ?? "text")} as ${typeMap[colDef?.type ?? "text"]})`
|
|
1181
|
+
};
|
|
1182
|
+
});
|
|
1183
|
+
if (columnsToAdd.length > 0) {
|
|
1184
|
+
await table.addColumns(columnsToAdd);
|
|
1185
|
+
this.logger?.info?.(`Added columns [${columnsToAdd.map((c) => c.name).join(", ")}] to table ${tableName}`);
|
|
1186
|
+
}
|
|
1187
|
+
} catch (error) {
|
|
1188
|
+
throw new MastraError(
|
|
1189
|
+
{
|
|
1190
|
+
id: "STORAGE_LANCE_STORAGE_ALTER_TABLE_FAILED",
|
|
1191
|
+
domain: ErrorDomain.STORAGE,
|
|
1192
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1193
|
+
details: { tableName }
|
|
1194
|
+
},
|
|
1195
|
+
error
|
|
1196
|
+
);
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
async clearTable({ tableName }) {
|
|
1200
|
+
try {
|
|
1201
|
+
if (!this.client) {
|
|
1202
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1203
|
+
}
|
|
1204
|
+
if (!tableName) {
|
|
1205
|
+
throw new Error("tableName is required for clearTable.");
|
|
1206
|
+
}
|
|
1207
|
+
} catch (validationError) {
|
|
1208
|
+
throw new MastraError(
|
|
1209
|
+
{
|
|
1210
|
+
id: "STORAGE_LANCE_STORAGE_CLEAR_TABLE_INVALID_ARGS",
|
|
1211
|
+
domain: ErrorDomain.STORAGE,
|
|
1212
|
+
category: ErrorCategory.USER,
|
|
1213
|
+
text: validationError.message,
|
|
1214
|
+
details: { tableName }
|
|
1215
|
+
},
|
|
1216
|
+
validationError
|
|
1217
|
+
);
|
|
1218
|
+
}
|
|
1219
|
+
try {
|
|
1220
|
+
const table = await this.client.openTable(tableName);
|
|
1221
|
+
await table.delete("1=1");
|
|
1222
|
+
} catch (error) {
|
|
1223
|
+
throw new MastraError(
|
|
1224
|
+
{
|
|
1225
|
+
id: "STORAGE_LANCE_STORAGE_CLEAR_TABLE_FAILED",
|
|
1226
|
+
domain: ErrorDomain.STORAGE,
|
|
1227
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1228
|
+
details: { tableName }
|
|
1229
|
+
},
|
|
1230
|
+
error
|
|
1231
|
+
);
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
async insert({ tableName, record }) {
|
|
1235
|
+
try {
|
|
1236
|
+
if (!this.client) {
|
|
1237
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1238
|
+
}
|
|
1239
|
+
if (!tableName) {
|
|
1240
|
+
throw new Error("tableName is required for insert.");
|
|
1241
|
+
}
|
|
1242
|
+
if (!record || Object.keys(record).length === 0) {
|
|
1243
|
+
throw new Error("record is required and cannot be empty for insert.");
|
|
1244
|
+
}
|
|
1245
|
+
} catch (validationError) {
|
|
1246
|
+
throw new MastraError(
|
|
1247
|
+
{
|
|
1248
|
+
id: "STORAGE_LANCE_STORAGE_INSERT_INVALID_ARGS",
|
|
1249
|
+
domain: ErrorDomain.STORAGE,
|
|
1250
|
+
category: ErrorCategory.USER,
|
|
1251
|
+
text: validationError.message,
|
|
1252
|
+
details: { tableName }
|
|
1253
|
+
},
|
|
1254
|
+
validationError
|
|
1255
|
+
);
|
|
1256
|
+
}
|
|
1257
|
+
try {
|
|
1258
|
+
const table = await this.client.openTable(tableName);
|
|
1259
|
+
const primaryId = getPrimaryKeys(tableName);
|
|
1260
|
+
const processedRecord = { ...record };
|
|
1261
|
+
for (const key in processedRecord) {
|
|
1262
|
+
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
1263
|
+
this.logger.debug("Converting object to JSON string: ", processedRecord[key]);
|
|
1264
|
+
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
console.log(await table.schema());
|
|
1268
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([processedRecord]);
|
|
1269
|
+
} catch (error) {
|
|
1270
|
+
throw new MastraError(
|
|
1271
|
+
{
|
|
1272
|
+
id: "STORAGE_LANCE_STORAGE_INSERT_FAILED",
|
|
1273
|
+
domain: ErrorDomain.STORAGE,
|
|
1274
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1275
|
+
details: { tableName }
|
|
1276
|
+
},
|
|
1277
|
+
error
|
|
1278
|
+
);
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
async batchInsert({ tableName, records }) {
|
|
1282
|
+
try {
|
|
1283
|
+
if (!this.client) {
|
|
1284
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1285
|
+
}
|
|
1286
|
+
if (!tableName) {
|
|
1287
|
+
throw new Error("tableName is required for batchInsert.");
|
|
1288
|
+
}
|
|
1289
|
+
if (!records || records.length === 0) {
|
|
1290
|
+
throw new Error("records array is required and cannot be empty for batchInsert.");
|
|
1291
|
+
}
|
|
1292
|
+
} catch (validationError) {
|
|
1293
|
+
throw new MastraError(
|
|
1294
|
+
{
|
|
1295
|
+
id: "STORAGE_LANCE_STORAGE_BATCH_INSERT_INVALID_ARGS",
|
|
1296
|
+
domain: ErrorDomain.STORAGE,
|
|
1297
|
+
category: ErrorCategory.USER,
|
|
1298
|
+
text: validationError.message,
|
|
1299
|
+
details: { tableName }
|
|
1300
|
+
},
|
|
1301
|
+
validationError
|
|
1302
|
+
);
|
|
1303
|
+
}
|
|
1304
|
+
try {
|
|
1305
|
+
const table = await this.client.openTable(tableName);
|
|
1306
|
+
const primaryId = getPrimaryKeys(tableName);
|
|
1307
|
+
const processedRecords = records.map((record) => {
|
|
1308
|
+
const processedRecord = { ...record };
|
|
1309
|
+
for (const key in processedRecord) {
|
|
1310
|
+
if (processedRecord[key] == null) continue;
|
|
1311
|
+
if (processedRecord[key] !== null && typeof processedRecord[key] === "object" && !(processedRecord[key] instanceof Date)) {
|
|
1312
|
+
processedRecord[key] = JSON.stringify(processedRecord[key]);
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
return processedRecord;
|
|
1316
|
+
});
|
|
1317
|
+
console.log(processedRecords);
|
|
1318
|
+
await table.mergeInsert(primaryId).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(processedRecords);
|
|
1319
|
+
} catch (error) {
|
|
1320
|
+
throw new MastraError(
|
|
1321
|
+
{
|
|
1322
|
+
id: "STORAGE_LANCE_STORAGE_BATCH_INSERT_FAILED",
|
|
1323
|
+
domain: ErrorDomain.STORAGE,
|
|
1324
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1325
|
+
details: { tableName }
|
|
1326
|
+
},
|
|
1327
|
+
error
|
|
1328
|
+
);
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
async load({ tableName, keys }) {
|
|
1332
|
+
try {
|
|
1333
|
+
if (!this.client) {
|
|
1334
|
+
throw new Error("LanceDB client not initialized. Call LanceStorage.create() first.");
|
|
1335
|
+
}
|
|
1336
|
+
if (!tableName) {
|
|
1337
|
+
throw new Error("tableName is required for load.");
|
|
1338
|
+
}
|
|
1339
|
+
if (!keys || Object.keys(keys).length === 0) {
|
|
1340
|
+
throw new Error("keys are required and cannot be empty for load.");
|
|
1341
|
+
}
|
|
1342
|
+
} catch (validationError) {
|
|
1343
|
+
throw new MastraError(
|
|
1344
|
+
{
|
|
1345
|
+
id: "STORAGE_LANCE_STORAGE_LOAD_INVALID_ARGS",
|
|
1346
|
+
domain: ErrorDomain.STORAGE,
|
|
1347
|
+
category: ErrorCategory.USER,
|
|
1348
|
+
text: validationError.message,
|
|
1349
|
+
details: { tableName }
|
|
1350
|
+
},
|
|
1351
|
+
validationError
|
|
1352
|
+
);
|
|
1353
|
+
}
|
|
1354
|
+
try {
|
|
1355
|
+
const table = await this.client.openTable(tableName);
|
|
1356
|
+
const tableSchema = await getTableSchema({ tableName, client: this.client });
|
|
1357
|
+
const query = table.query();
|
|
1358
|
+
if (Object.keys(keys).length > 0) {
|
|
1359
|
+
validateKeyTypes(keys, tableSchema);
|
|
1360
|
+
const filterConditions = Object.entries(keys).map(([key, value]) => {
|
|
1361
|
+
const isCamelCase = /^[a-z][a-zA-Z]*$/.test(key) && /[A-Z]/.test(key);
|
|
1362
|
+
const quotedKey = isCamelCase ? `\`${key}\`` : key;
|
|
1363
|
+
if (typeof value === "string") {
|
|
1364
|
+
return `${quotedKey} = '${value}'`;
|
|
1365
|
+
} else if (value === null) {
|
|
1366
|
+
return `${quotedKey} IS NULL`;
|
|
1367
|
+
} else {
|
|
1368
|
+
return `${quotedKey} = ${value}`;
|
|
1369
|
+
}
|
|
1370
|
+
}).join(" AND ");
|
|
1371
|
+
this.logger.debug("where clause generated: " + filterConditions);
|
|
1372
|
+
query.where(filterConditions);
|
|
1373
|
+
}
|
|
1374
|
+
const result = await query.limit(1).toArray();
|
|
1375
|
+
if (result.length === 0) {
|
|
1376
|
+
this.logger.debug("No record found");
|
|
1377
|
+
return null;
|
|
1378
|
+
}
|
|
1379
|
+
return processResultWithTypeConversion(result[0], tableSchema);
|
|
1380
|
+
} catch (error) {
|
|
1381
|
+
if (error instanceof MastraError) throw error;
|
|
1382
|
+
throw new MastraError(
|
|
1383
|
+
{
|
|
1384
|
+
id: "STORAGE_LANCE_STORAGE_LOAD_FAILED",
|
|
1385
|
+
domain: ErrorDomain.STORAGE,
|
|
1386
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1387
|
+
details: { tableName, keyCount: Object.keys(keys).length, firstKey: Object.keys(keys)[0] ?? "" }
|
|
1388
|
+
},
|
|
1389
|
+
error
|
|
1390
|
+
);
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
};
|
|
1394
|
+
var StoreScoresLance = class extends ScoresStorage {
|
|
1395
|
+
client;
|
|
1396
|
+
constructor({ client }) {
|
|
1397
|
+
super();
|
|
1398
|
+
this.client = client;
|
|
1399
|
+
}
|
|
1400
|
+
async saveScore(score) {
|
|
1401
|
+
try {
|
|
1402
|
+
const id = crypto.randomUUID();
|
|
1403
|
+
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1404
|
+
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1405
|
+
const allowedFields = new Set(schema.fields.map((f) => f.name));
|
|
1406
|
+
const filteredScore = {};
|
|
1407
|
+
Object.keys(score).forEach((key) => {
|
|
1408
|
+
if (allowedFields.has(key)) {
|
|
1409
|
+
filteredScore[key] = score[key];
|
|
1410
|
+
}
|
|
1411
|
+
});
|
|
1412
|
+
for (const key in filteredScore) {
|
|
1413
|
+
if (filteredScore[key] !== null && typeof filteredScore[key] === "object" && !(filteredScore[key] instanceof Date)) {
|
|
1414
|
+
filteredScore[key] = JSON.stringify(filteredScore[key]);
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
filteredScore.id = id;
|
|
1418
|
+
await table.add([filteredScore], { mode: "append" });
|
|
1419
|
+
return { score };
|
|
1420
|
+
} catch (error) {
|
|
1421
|
+
throw new MastraError(
|
|
1422
|
+
{
|
|
1423
|
+
id: "LANCE_STORAGE_SAVE_SCORE_FAILED",
|
|
1424
|
+
text: "Failed to save score in LanceStorage",
|
|
1425
|
+
domain: ErrorDomain.STORAGE,
|
|
1426
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1427
|
+
details: { error: error?.message }
|
|
1428
|
+
},
|
|
1429
|
+
error
|
|
1430
|
+
);
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
async getScoreById({ id }) {
|
|
1434
|
+
try {
|
|
1435
|
+
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1436
|
+
const query = table.query().where(`id = '${id}'`).limit(1);
|
|
1437
|
+
const records = await query.toArray();
|
|
1438
|
+
if (records.length === 0) return null;
|
|
1439
|
+
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1440
|
+
return processResultWithTypeConversion(records[0], schema);
|
|
1441
|
+
} catch (error) {
|
|
1442
|
+
throw new MastraError(
|
|
1443
|
+
{
|
|
1444
|
+
id: "LANCE_STORAGE_GET_SCORE_BY_ID_FAILED",
|
|
1445
|
+
text: "Failed to get score by id in LanceStorage",
|
|
1446
|
+
domain: ErrorDomain.STORAGE,
|
|
1447
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1448
|
+
details: { error: error?.message }
|
|
1449
|
+
},
|
|
1450
|
+
error
|
|
1451
|
+
);
|
|
1452
|
+
}
|
|
1453
|
+
}
|
|
1454
|
+
async getScoresByScorerId({
|
|
1455
|
+
scorerId,
|
|
1456
|
+
pagination,
|
|
1457
|
+
entityId,
|
|
1458
|
+
entityType,
|
|
1459
|
+
source
|
|
1460
|
+
}) {
|
|
1461
|
+
try {
|
|
1462
|
+
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1463
|
+
const { page = 0, perPage = 10 } = pagination || {};
|
|
1464
|
+
const offset = page * perPage;
|
|
1465
|
+
let query = table.query().where(`\`scorerId\` = '${scorerId}'`);
|
|
1466
|
+
if (source) {
|
|
1467
|
+
query = query.where(`\`source\` = '${source}'`);
|
|
1468
|
+
}
|
|
1469
|
+
if (entityId) {
|
|
1470
|
+
query = query.where(`\`entityId\` = '${entityId}'`);
|
|
1471
|
+
}
|
|
1472
|
+
if (entityType) {
|
|
1473
|
+
query = query.where(`\`entityType\` = '${entityType}'`);
|
|
1474
|
+
}
|
|
1475
|
+
query = query.limit(perPage);
|
|
1476
|
+
if (offset > 0) query.offset(offset);
|
|
1477
|
+
const records = await query.toArray();
|
|
1478
|
+
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1479
|
+
const scores = processResultWithTypeConversion(records, schema);
|
|
1480
|
+
let totalQuery = table.query().where(`\`scorerId\` = '${scorerId}'`);
|
|
1481
|
+
if (source) {
|
|
1482
|
+
totalQuery = totalQuery.where(`\`source\` = '${source}'`);
|
|
1483
|
+
}
|
|
1484
|
+
const allRecords = await totalQuery.toArray();
|
|
1485
|
+
const total = allRecords.length;
|
|
1486
|
+
return {
|
|
1487
|
+
pagination: {
|
|
1488
|
+
page,
|
|
1489
|
+
perPage,
|
|
1490
|
+
total,
|
|
1491
|
+
hasMore: offset + scores.length < total
|
|
1492
|
+
},
|
|
1493
|
+
scores
|
|
1494
|
+
};
|
|
1495
|
+
} catch (error) {
|
|
1496
|
+
throw new MastraError(
|
|
1497
|
+
{
|
|
1498
|
+
id: "LANCE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
|
|
1499
|
+
text: "Failed to get scores by scorerId in LanceStorage",
|
|
1500
|
+
domain: ErrorDomain.STORAGE,
|
|
1501
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1502
|
+
details: { error: error?.message }
|
|
1503
|
+
},
|
|
1504
|
+
error
|
|
1505
|
+
);
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
async getScoresByRunId({
|
|
1509
|
+
runId,
|
|
1510
|
+
pagination
|
|
1511
|
+
}) {
|
|
1512
|
+
try {
|
|
1513
|
+
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1514
|
+
const { page = 0, perPage = 10 } = pagination || {};
|
|
1515
|
+
const offset = page * perPage;
|
|
1516
|
+
const query = table.query().where(`\`runId\` = '${runId}'`).limit(perPage);
|
|
1517
|
+
if (offset > 0) query.offset(offset);
|
|
1518
|
+
const records = await query.toArray();
|
|
1519
|
+
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1520
|
+
const scores = processResultWithTypeConversion(records, schema);
|
|
1521
|
+
const allRecords = await table.query().where(`\`runId\` = '${runId}'`).toArray();
|
|
1522
|
+
const total = allRecords.length;
|
|
1523
|
+
return {
|
|
1524
|
+
pagination: {
|
|
1525
|
+
page,
|
|
1526
|
+
perPage,
|
|
1527
|
+
total,
|
|
1528
|
+
hasMore: offset + scores.length < total
|
|
1529
|
+
},
|
|
1530
|
+
scores
|
|
1531
|
+
};
|
|
1532
|
+
} catch (error) {
|
|
1533
|
+
throw new MastraError(
|
|
1534
|
+
{
|
|
1535
|
+
id: "LANCE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
|
|
1536
|
+
text: "Failed to get scores by runId in LanceStorage",
|
|
1537
|
+
domain: ErrorDomain.STORAGE,
|
|
1538
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1539
|
+
details: { error: error?.message }
|
|
1540
|
+
},
|
|
1541
|
+
error
|
|
1542
|
+
);
|
|
1543
|
+
}
|
|
1544
|
+
}
|
|
1545
|
+
async getScoresByEntityId({
|
|
1546
|
+
entityId,
|
|
1547
|
+
entityType,
|
|
1548
|
+
pagination
|
|
1549
|
+
}) {
|
|
1550
|
+
try {
|
|
1551
|
+
const table = await this.client.openTable(TABLE_SCORERS);
|
|
1552
|
+
const { page = 0, perPage = 10 } = pagination || {};
|
|
1553
|
+
const offset = page * perPage;
|
|
1554
|
+
const query = table.query().where(`\`entityId\` = '${entityId}' AND \`entityType\` = '${entityType}'`).limit(perPage);
|
|
1555
|
+
if (offset > 0) query.offset(offset);
|
|
1556
|
+
const records = await query.toArray();
|
|
1557
|
+
const schema = await getTableSchema({ tableName: TABLE_SCORERS, client: this.client });
|
|
1558
|
+
const scores = processResultWithTypeConversion(records, schema);
|
|
1559
|
+
const allRecords = await table.query().where(`\`entityId\` = '${entityId}' AND \`entityType\` = '${entityType}'`).toArray();
|
|
1560
|
+
const total = allRecords.length;
|
|
1561
|
+
return {
|
|
1562
|
+
pagination: {
|
|
1563
|
+
page,
|
|
1564
|
+
perPage,
|
|
1565
|
+
total,
|
|
1566
|
+
hasMore: offset + scores.length < total
|
|
1567
|
+
},
|
|
1568
|
+
scores
|
|
1569
|
+
};
|
|
1570
|
+
} catch (error) {
|
|
1571
|
+
throw new MastraError(
|
|
1572
|
+
{
|
|
1573
|
+
id: "LANCE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
|
|
1574
|
+
text: "Failed to get scores by entityId and entityType in LanceStorage",
|
|
1575
|
+
domain: ErrorDomain.STORAGE,
|
|
1576
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1577
|
+
details: { error: error?.message }
|
|
1578
|
+
},
|
|
1579
|
+
error
|
|
1580
|
+
);
|
|
1581
|
+
}
|
|
1582
|
+
}
|
|
1583
|
+
};
|
|
1584
|
+
var StoreTracesLance = class extends TracesStorage {
|
|
1585
|
+
client;
|
|
1586
|
+
operations;
|
|
1587
|
+
constructor({ client, operations }) {
|
|
1588
|
+
super();
|
|
1589
|
+
this.client = client;
|
|
1590
|
+
this.operations = operations;
|
|
1591
|
+
}
|
|
1592
|
+
async saveTrace({ trace }) {
|
|
1593
|
+
try {
|
|
1594
|
+
const table = await this.client.openTable(TABLE_TRACES);
|
|
1595
|
+
const record = {
|
|
1596
|
+
...trace,
|
|
1597
|
+
attributes: JSON.stringify(trace.attributes),
|
|
1598
|
+
status: JSON.stringify(trace.status),
|
|
1599
|
+
events: JSON.stringify(trace.events),
|
|
1600
|
+
links: JSON.stringify(trace.links),
|
|
1601
|
+
other: JSON.stringify(trace.other)
|
|
1602
|
+
};
|
|
1603
|
+
await table.add([record], { mode: "append" });
|
|
1604
|
+
return trace;
|
|
1605
|
+
} catch (error) {
|
|
1606
|
+
throw new MastraError(
|
|
1607
|
+
{
|
|
1608
|
+
id: "LANCE_STORE_SAVE_TRACE_FAILED",
|
|
1609
|
+
domain: ErrorDomain.STORAGE,
|
|
1610
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1611
|
+
},
|
|
1612
|
+
error
|
|
1613
|
+
);
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
async getTraceById({ traceId }) {
|
|
1617
|
+
try {
|
|
1618
|
+
const table = await this.client.openTable(TABLE_TRACES);
|
|
1619
|
+
const query = table.query().where(`id = '${traceId}'`);
|
|
1620
|
+
const records = await query.toArray();
|
|
1621
|
+
return records[0];
|
|
1622
|
+
} catch (error) {
|
|
1623
|
+
throw new MastraError(
|
|
1624
|
+
{
|
|
1625
|
+
id: "LANCE_STORE_GET_TRACE_BY_ID_FAILED",
|
|
1626
|
+
domain: ErrorDomain.STORAGE,
|
|
1627
|
+
category: ErrorCategory.THIRD_PARTY
|
|
1628
|
+
},
|
|
1629
|
+
error
|
|
1630
|
+
);
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
async getTraces({
|
|
1634
|
+
name,
|
|
1635
|
+
scope,
|
|
1636
|
+
page = 1,
|
|
1637
|
+
perPage = 10,
|
|
1638
|
+
attributes
|
|
1639
|
+
}) {
|
|
1640
|
+
try {
|
|
1641
|
+
const table = await this.client.openTable(TABLE_TRACES);
|
|
1642
|
+
const query = table.query();
|
|
1643
|
+
if (name) {
|
|
1644
|
+
query.where(`name = '${name}'`);
|
|
1645
|
+
}
|
|
1646
|
+
if (scope) {
|
|
1647
|
+
query.where(`scope = '${scope}'`);
|
|
1648
|
+
}
|
|
1649
|
+
if (attributes) {
|
|
1650
|
+
query.where(`attributes = '${JSON.stringify(attributes)}'`);
|
|
1651
|
+
}
|
|
1652
|
+
const offset = (page - 1) * perPage;
|
|
1653
|
+
query.limit(perPage);
|
|
1654
|
+
if (offset > 0) {
|
|
1655
|
+
query.offset(offset);
|
|
1656
|
+
}
|
|
1657
|
+
const records = await query.toArray();
|
|
1658
|
+
return records.map((record) => {
|
|
1659
|
+
const processed = {
|
|
1660
|
+
...record,
|
|
1661
|
+
attributes: record.attributes ? JSON.parse(record.attributes) : {},
|
|
1662
|
+
status: record.status ? JSON.parse(record.status) : {},
|
|
1663
|
+
events: record.events ? JSON.parse(record.events) : [],
|
|
1664
|
+
links: record.links ? JSON.parse(record.links) : [],
|
|
1665
|
+
other: record.other ? JSON.parse(record.other) : {},
|
|
1666
|
+
startTime: new Date(record.startTime),
|
|
1667
|
+
endTime: new Date(record.endTime),
|
|
1668
|
+
createdAt: new Date(record.createdAt)
|
|
1669
|
+
};
|
|
1670
|
+
if (processed.parentSpanId === null || processed.parentSpanId === void 0) {
|
|
1671
|
+
processed.parentSpanId = "";
|
|
1672
|
+
} else {
|
|
1673
|
+
processed.parentSpanId = String(processed.parentSpanId);
|
|
1674
|
+
}
|
|
1675
|
+
return processed;
|
|
1676
|
+
});
|
|
1677
|
+
} catch (error) {
|
|
1678
|
+
throw new MastraError(
|
|
1679
|
+
{
|
|
1680
|
+
id: "LANCE_STORE_GET_TRACES_FAILED",
|
|
1681
|
+
domain: ErrorDomain.STORAGE,
|
|
1682
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1683
|
+
details: { name: name ?? "", scope: scope ?? "" }
|
|
1684
|
+
},
|
|
1685
|
+
error
|
|
1686
|
+
);
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
async getTracesPaginated(args) {
|
|
1690
|
+
try {
|
|
1691
|
+
const table = await this.client.openTable(TABLE_TRACES);
|
|
1692
|
+
const query = table.query();
|
|
1693
|
+
const conditions = [];
|
|
1694
|
+
if (args.name) {
|
|
1695
|
+
conditions.push(`name = '${args.name}'`);
|
|
1696
|
+
}
|
|
1697
|
+
if (args.scope) {
|
|
1698
|
+
conditions.push(`scope = '${args.scope}'`);
|
|
1699
|
+
}
|
|
1700
|
+
if (args.attributes) {
|
|
1701
|
+
const attributesStr = JSON.stringify(args.attributes);
|
|
1702
|
+
conditions.push(`attributes LIKE '%${attributesStr.replace(/"/g, '\\"')}%'`);
|
|
1703
|
+
}
|
|
1704
|
+
if (args.dateRange?.start) {
|
|
1705
|
+
conditions.push(`\`createdAt\` >= ${args.dateRange.start.getTime()}`);
|
|
1706
|
+
}
|
|
1707
|
+
if (args.dateRange?.end) {
|
|
1708
|
+
conditions.push(`\`createdAt\` <= ${args.dateRange.end.getTime()}`);
|
|
1709
|
+
}
|
|
1710
|
+
if (conditions.length > 0) {
|
|
1711
|
+
const whereClause = conditions.join(" AND ");
|
|
1712
|
+
query.where(whereClause);
|
|
1713
|
+
}
|
|
1714
|
+
let total = 0;
|
|
1715
|
+
if (conditions.length > 0) {
|
|
1716
|
+
const countQuery = table.query().where(conditions.join(" AND "));
|
|
1717
|
+
const allRecords = await countQuery.toArray();
|
|
1718
|
+
total = allRecords.length;
|
|
1719
|
+
} else {
|
|
1720
|
+
total = await table.countRows();
|
|
1721
|
+
}
|
|
1722
|
+
const page = args.page || 0;
|
|
1723
|
+
const perPage = args.perPage || 10;
|
|
1724
|
+
const offset = page * perPage;
|
|
1725
|
+
query.limit(perPage);
|
|
1726
|
+
if (offset > 0) {
|
|
1727
|
+
query.offset(offset);
|
|
1728
|
+
}
|
|
1729
|
+
const records = await query.toArray();
|
|
1730
|
+
const traces = records.map((record) => {
|
|
1731
|
+
const processed = {
|
|
1732
|
+
...record,
|
|
1733
|
+
attributes: record.attributes ? JSON.parse(record.attributes) : {},
|
|
1734
|
+
status: record.status ? JSON.parse(record.status) : {},
|
|
1735
|
+
events: record.events ? JSON.parse(record.events) : [],
|
|
1736
|
+
links: record.links ? JSON.parse(record.links) : [],
|
|
1737
|
+
other: record.other ? JSON.parse(record.other) : {},
|
|
1738
|
+
startTime: new Date(record.startTime),
|
|
1739
|
+
endTime: new Date(record.endTime),
|
|
1740
|
+
createdAt: new Date(record.createdAt)
|
|
1741
|
+
};
|
|
1742
|
+
if (processed.parentSpanId === null || processed.parentSpanId === void 0) {
|
|
1743
|
+
processed.parentSpanId = "";
|
|
1744
|
+
} else {
|
|
1745
|
+
processed.parentSpanId = String(processed.parentSpanId);
|
|
1746
|
+
}
|
|
1747
|
+
return processed;
|
|
1748
|
+
});
|
|
1749
|
+
return {
|
|
1750
|
+
traces,
|
|
1751
|
+
total,
|
|
1752
|
+
page,
|
|
1753
|
+
perPage,
|
|
1754
|
+
hasMore: total > (page + 1) * perPage
|
|
1755
|
+
};
|
|
1756
|
+
} catch (error) {
|
|
1757
|
+
throw new MastraError(
|
|
1758
|
+
{
|
|
1759
|
+
id: "LANCE_STORE_GET_TRACES_PAGINATED_FAILED",
|
|
1760
|
+
domain: ErrorDomain.STORAGE,
|
|
1761
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1762
|
+
details: { name: args.name ?? "", scope: args.scope ?? "" }
|
|
1763
|
+
},
|
|
1764
|
+
error
|
|
1765
|
+
);
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
async batchTraceInsert({ records }) {
|
|
1769
|
+
this.logger.debug("Batch inserting traces", { count: records.length });
|
|
1770
|
+
await this.operations.batchInsert({
|
|
1771
|
+
tableName: TABLE_TRACES,
|
|
1772
|
+
records
|
|
1773
|
+
});
|
|
1774
|
+
}
|
|
1775
|
+
};
|
|
1776
|
+
function parseWorkflowRun(row) {
|
|
1777
|
+
let parsedSnapshot = row.snapshot;
|
|
1778
|
+
if (typeof parsedSnapshot === "string") {
|
|
1779
|
+
try {
|
|
1780
|
+
parsedSnapshot = JSON.parse(row.snapshot);
|
|
1781
|
+
} catch (e) {
|
|
1782
|
+
console.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
return {
|
|
1786
|
+
workflowName: row.workflow_name,
|
|
1787
|
+
runId: row.run_id,
|
|
1788
|
+
snapshot: parsedSnapshot,
|
|
1789
|
+
createdAt: ensureDate(row.createdAt),
|
|
1790
|
+
updatedAt: ensureDate(row.updatedAt),
|
|
1791
|
+
resourceId: row.resourceId
|
|
1792
|
+
};
|
|
1793
|
+
}
|
|
1794
|
+
var StoreWorkflowsLance = class extends WorkflowsStorage {
|
|
1795
|
+
client;
|
|
1796
|
+
constructor({ client }) {
|
|
1797
|
+
super();
|
|
1798
|
+
this.client = client;
|
|
1799
|
+
}
|
|
1800
|
+
updateWorkflowResults({
|
|
1801
|
+
// workflowName,
|
|
1802
|
+
// runId,
|
|
1803
|
+
// stepId,
|
|
1804
|
+
// result,
|
|
1805
|
+
// runtimeContext,
|
|
1806
|
+
}) {
|
|
1807
|
+
throw new Error("Method not implemented.");
|
|
1808
|
+
}
|
|
1809
|
+
updateWorkflowState({
|
|
1810
|
+
// workflowName,
|
|
1811
|
+
// runId,
|
|
1812
|
+
// opts,
|
|
1813
|
+
}) {
|
|
1814
|
+
throw new Error("Method not implemented.");
|
|
1815
|
+
}
|
|
1816
|
+
async persistWorkflowSnapshot({
|
|
1817
|
+
workflowName,
|
|
1818
|
+
runId,
|
|
1819
|
+
snapshot
|
|
1820
|
+
}) {
|
|
1821
|
+
try {
|
|
1822
|
+
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1823
|
+
const query = table.query().where(`workflow_name = '${workflowName}' AND run_id = '${runId}'`);
|
|
1824
|
+
const records = await query.toArray();
|
|
1825
|
+
let createdAt;
|
|
1826
|
+
const now = Date.now();
|
|
1827
|
+
if (records.length > 0) {
|
|
1828
|
+
createdAt = records[0].createdAt ?? now;
|
|
1829
|
+
} else {
|
|
1830
|
+
createdAt = now;
|
|
1831
|
+
}
|
|
1832
|
+
const record = {
|
|
1833
|
+
workflow_name: workflowName,
|
|
1834
|
+
run_id: runId,
|
|
1835
|
+
snapshot: JSON.stringify(snapshot),
|
|
1836
|
+
createdAt,
|
|
1837
|
+
updatedAt: now
|
|
1838
|
+
};
|
|
1839
|
+
await table.mergeInsert(["workflow_name", "run_id"]).whenMatchedUpdateAll().whenNotMatchedInsertAll().execute([record]);
|
|
1840
|
+
} catch (error) {
|
|
1841
|
+
throw new MastraError(
|
|
1842
|
+
{
|
|
1843
|
+
id: "LANCE_STORE_PERSIST_WORKFLOW_SNAPSHOT_FAILED",
|
|
1844
|
+
domain: ErrorDomain.STORAGE,
|
|
1845
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1846
|
+
details: { workflowName, runId }
|
|
1847
|
+
},
|
|
1848
|
+
error
|
|
1849
|
+
);
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
async loadWorkflowSnapshot({
|
|
1853
|
+
workflowName,
|
|
1854
|
+
runId
|
|
1855
|
+
}) {
|
|
1856
|
+
try {
|
|
1857
|
+
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1858
|
+
const query = table.query().where(`workflow_name = '${workflowName}' AND run_id = '${runId}'`);
|
|
1859
|
+
const records = await query.toArray();
|
|
1860
|
+
return records.length > 0 ? JSON.parse(records[0].snapshot) : null;
|
|
1861
|
+
} catch (error) {
|
|
1862
|
+
throw new MastraError(
|
|
1863
|
+
{
|
|
1864
|
+
id: "LANCE_STORE_LOAD_WORKFLOW_SNAPSHOT_FAILED",
|
|
1865
|
+
domain: ErrorDomain.STORAGE,
|
|
1866
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1867
|
+
details: { workflowName, runId }
|
|
1868
|
+
},
|
|
1869
|
+
error
|
|
1870
|
+
);
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
async getWorkflowRunById(args) {
|
|
1874
|
+
try {
|
|
1875
|
+
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1876
|
+
let whereClause = `run_id = '${args.runId}'`;
|
|
1877
|
+
if (args.workflowName) {
|
|
1878
|
+
whereClause += ` AND workflow_name = '${args.workflowName}'`;
|
|
1879
|
+
}
|
|
1880
|
+
const query = table.query().where(whereClause);
|
|
1881
|
+
const records = await query.toArray();
|
|
1882
|
+
if (records.length === 0) return null;
|
|
1883
|
+
const record = records[0];
|
|
1884
|
+
return parseWorkflowRun(record);
|
|
1885
|
+
} catch (error) {
|
|
1886
|
+
throw new MastraError(
|
|
1887
|
+
{
|
|
1888
|
+
id: "LANCE_STORE_GET_WORKFLOW_RUN_BY_ID_FAILED",
|
|
1889
|
+
domain: ErrorDomain.STORAGE,
|
|
1890
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1891
|
+
details: { runId: args.runId, workflowName: args.workflowName ?? "" }
|
|
1892
|
+
},
|
|
1893
|
+
error
|
|
1894
|
+
);
|
|
1895
|
+
}
|
|
1896
|
+
}
|
|
1897
|
+
async getWorkflowRuns(args) {
|
|
1898
|
+
try {
|
|
1899
|
+
const table = await this.client.openTable(TABLE_WORKFLOW_SNAPSHOT);
|
|
1900
|
+
let query = table.query();
|
|
1901
|
+
const conditions = [];
|
|
1902
|
+
if (args?.workflowName) {
|
|
1903
|
+
conditions.push(`workflow_name = '${args.workflowName.replace(/'/g, "''")}'`);
|
|
1904
|
+
}
|
|
1905
|
+
if (args?.resourceId) {
|
|
1906
|
+
conditions.push(`\`resourceId\` = '${args.resourceId}'`);
|
|
1907
|
+
}
|
|
1908
|
+
if (args?.fromDate instanceof Date) {
|
|
1909
|
+
conditions.push(`\`createdAt\` >= ${args.fromDate.getTime()}`);
|
|
1910
|
+
}
|
|
1911
|
+
if (args?.toDate instanceof Date) {
|
|
1912
|
+
conditions.push(`\`createdAt\` <= ${args.toDate.getTime()}`);
|
|
1913
|
+
}
|
|
1914
|
+
let total = 0;
|
|
1915
|
+
if (conditions.length > 0) {
|
|
1916
|
+
query = query.where(conditions.join(" AND "));
|
|
1917
|
+
total = await table.countRows(conditions.join(" AND "));
|
|
1918
|
+
} else {
|
|
1919
|
+
total = await table.countRows();
|
|
1920
|
+
}
|
|
1921
|
+
if (args?.limit) {
|
|
1922
|
+
query.limit(args.limit);
|
|
1923
|
+
}
|
|
1924
|
+
if (args?.offset) {
|
|
1925
|
+
query.offset(args.offset);
|
|
1926
|
+
}
|
|
1927
|
+
const records = await query.toArray();
|
|
1928
|
+
return {
|
|
1929
|
+
runs: records.map((record) => parseWorkflowRun(record)),
|
|
1930
|
+
total: total || records.length
|
|
1931
|
+
};
|
|
1932
|
+
} catch (error) {
|
|
1933
|
+
throw new MastraError(
|
|
1934
|
+
{
|
|
1935
|
+
id: "LANCE_STORE_GET_WORKFLOW_RUNS_FAILED",
|
|
1936
|
+
domain: ErrorDomain.STORAGE,
|
|
1937
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1938
|
+
details: { namespace: args?.namespace ?? "", workflowName: args?.workflowName ?? "" }
|
|
1939
|
+
},
|
|
1940
|
+
error
|
|
1941
|
+
);
|
|
1942
|
+
}
|
|
1943
|
+
}
|
|
1944
|
+
};
|
|
1945
|
+
|
|
1946
|
+
// src/storage/index.ts
|
|
1947
|
+
var LanceStorage = class _LanceStorage extends MastraStorage {
|
|
1948
|
+
stores;
|
|
1949
|
+
lanceClient;
|
|
1950
|
+
/**
|
|
1951
|
+
* Creates a new instance of LanceStorage
|
|
1952
|
+
* @param uri The URI to connect to LanceDB
|
|
1953
|
+
* @param options connection options
|
|
1954
|
+
*
|
|
1955
|
+
* Usage:
|
|
1956
|
+
*
|
|
1957
|
+
* Connect to a local database
|
|
1958
|
+
* ```ts
|
|
1959
|
+
* const store = await LanceStorage.create('/path/to/db');
|
|
1960
|
+
* ```
|
|
1961
|
+
*
|
|
1962
|
+
* Connect to a LanceDB cloud database
|
|
1963
|
+
* ```ts
|
|
1964
|
+
* const store = await LanceStorage.create('db://host:port');
|
|
1965
|
+
* ```
|
|
1966
|
+
*
|
|
1967
|
+
* Connect to a cloud database
|
|
1968
|
+
* ```ts
|
|
1969
|
+
* const store = await LanceStorage.create('s3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
1970
|
+
* ```
|
|
1971
|
+
*/
|
|
1972
|
+
static async create(name, uri, options) {
|
|
1973
|
+
const instance = new _LanceStorage(name);
|
|
1974
|
+
try {
|
|
1975
|
+
instance.lanceClient = await connect(uri, options);
|
|
1976
|
+
const operations = new StoreOperationsLance({ client: instance.lanceClient });
|
|
1977
|
+
instance.stores = {
|
|
1978
|
+
operations: new StoreOperationsLance({ client: instance.lanceClient }),
|
|
1979
|
+
workflows: new StoreWorkflowsLance({ client: instance.lanceClient }),
|
|
1980
|
+
traces: new StoreTracesLance({ client: instance.lanceClient, operations }),
|
|
1981
|
+
scores: new StoreScoresLance({ client: instance.lanceClient }),
|
|
1982
|
+
memory: new StoreMemoryLance({ client: instance.lanceClient, operations }),
|
|
1983
|
+
legacyEvals: new StoreLegacyEvalsLance({ client: instance.lanceClient })
|
|
1984
|
+
};
|
|
1985
|
+
return instance;
|
|
1986
|
+
} catch (e) {
|
|
1987
|
+
throw new MastraError(
|
|
1988
|
+
{
|
|
1989
|
+
id: "STORAGE_LANCE_STORAGE_CONNECT_FAILED",
|
|
1990
|
+
domain: ErrorDomain.STORAGE,
|
|
1991
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1992
|
+
text: `Failed to connect to LanceDB: ${e.message || e}`,
|
|
1993
|
+
details: { uri, optionsProvided: !!options }
|
|
1994
|
+
},
|
|
1995
|
+
e
|
|
1996
|
+
);
|
|
1997
|
+
}
|
|
1998
|
+
}
|
|
1999
|
+
/**
|
|
2000
|
+
* @internal
|
|
2001
|
+
* Private constructor to enforce using the create factory method
|
|
2002
|
+
*/
|
|
2003
|
+
constructor(name) {
|
|
2004
|
+
super({ name });
|
|
2005
|
+
const operations = new StoreOperationsLance({ client: this.lanceClient });
|
|
2006
|
+
this.stores = {
|
|
2007
|
+
operations: new StoreOperationsLance({ client: this.lanceClient }),
|
|
2008
|
+
workflows: new StoreWorkflowsLance({ client: this.lanceClient }),
|
|
2009
|
+
traces: new StoreTracesLance({ client: this.lanceClient, operations }),
|
|
2010
|
+
scores: new StoreScoresLance({ client: this.lanceClient }),
|
|
2011
|
+
legacyEvals: new StoreLegacyEvalsLance({ client: this.lanceClient }),
|
|
2012
|
+
memory: new StoreMemoryLance({ client: this.lanceClient, operations })
|
|
2013
|
+
};
|
|
2014
|
+
}
|
|
2015
|
+
async createTable({
|
|
2016
|
+
tableName,
|
|
2017
|
+
schema
|
|
2018
|
+
}) {
|
|
2019
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
2020
|
+
}
|
|
2021
|
+
async dropTable({ tableName }) {
|
|
2022
|
+
return this.stores.operations.dropTable({ tableName });
|
|
2023
|
+
}
|
|
2024
|
+
async alterTable({
|
|
2025
|
+
tableName,
|
|
2026
|
+
schema,
|
|
2027
|
+
ifNotExists
|
|
2028
|
+
}) {
|
|
2029
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
2030
|
+
}
|
|
2031
|
+
async clearTable({ tableName }) {
|
|
2032
|
+
return this.stores.operations.clearTable({ tableName });
|
|
2033
|
+
}
|
|
2034
|
+
async insert({ tableName, record }) {
|
|
2035
|
+
return this.stores.operations.insert({ tableName, record });
|
|
2036
|
+
}
|
|
2037
|
+
async batchInsert({ tableName, records }) {
|
|
2038
|
+
return this.stores.operations.batchInsert({ tableName, records });
|
|
2039
|
+
}
|
|
2040
|
+
async load({ tableName, keys }) {
|
|
2041
|
+
return this.stores.operations.load({ tableName, keys });
|
|
2042
|
+
}
|
|
2043
|
+
async getThreadById({ threadId }) {
|
|
2044
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
2045
|
+
}
|
|
2046
|
+
async getThreadsByResourceId({ resourceId }) {
|
|
2047
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
2048
|
+
}
|
|
2049
|
+
/**
|
|
2050
|
+
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
2051
|
+
* @param thread - The thread to save
|
|
2052
|
+
* @returns The saved thread
|
|
2053
|
+
*/
|
|
2054
|
+
async saveThread({ thread }) {
|
|
2055
|
+
return this.stores.memory.saveThread({ thread });
|
|
2056
|
+
}
|
|
2057
|
+
async updateThread({
|
|
2058
|
+
id,
|
|
2059
|
+
title,
|
|
2060
|
+
metadata
|
|
2061
|
+
}) {
|
|
2062
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
2063
|
+
}
|
|
2064
|
+
async deleteThread({ threadId }) {
|
|
2065
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
2066
|
+
}
|
|
2067
|
+
get supports() {
|
|
2068
|
+
return {
|
|
2069
|
+
selectByIncludeResourceScope: true,
|
|
2070
|
+
resourceWorkingMemory: true,
|
|
2071
|
+
hasColumn: true,
|
|
2072
|
+
createTable: true,
|
|
2073
|
+
deleteMessages: false
|
|
2074
|
+
};
|
|
2075
|
+
}
|
|
2076
|
+
async getResourceById({ resourceId }) {
|
|
2077
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
2078
|
+
}
|
|
2079
|
+
async saveResource({ resource }) {
|
|
2080
|
+
return this.stores.memory.saveResource({ resource });
|
|
2081
|
+
}
|
|
2082
|
+
async updateResource({
|
|
2083
|
+
resourceId,
|
|
2084
|
+
workingMemory,
|
|
2085
|
+
metadata
|
|
2086
|
+
}) {
|
|
2087
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
2088
|
+
}
|
|
2089
|
+
/**
|
|
2090
|
+
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
2091
|
+
* @param records - The sorted array of records to process
|
|
2092
|
+
* @param include - The array of include specifications with context parameters
|
|
2093
|
+
* @returns The processed array with context messages included
|
|
2094
|
+
*/
|
|
2095
|
+
processMessagesWithContext(records, include) {
|
|
2096
|
+
const messagesWithContext = include.filter((item) => item.withPreviousMessages || item.withNextMessages);
|
|
2097
|
+
if (messagesWithContext.length === 0) {
|
|
2098
|
+
return records;
|
|
2099
|
+
}
|
|
2100
|
+
const messageIndexMap = /* @__PURE__ */ new Map();
|
|
2101
|
+
records.forEach((message, index) => {
|
|
2102
|
+
messageIndexMap.set(message.id, index);
|
|
2103
|
+
});
|
|
2104
|
+
const additionalIndices = /* @__PURE__ */ new Set();
|
|
2105
|
+
for (const item of messagesWithContext) {
|
|
2106
|
+
const messageIndex = messageIndexMap.get(item.id);
|
|
2107
|
+
if (messageIndex !== void 0) {
|
|
2108
|
+
if (item.withPreviousMessages) {
|
|
2109
|
+
const startIdx = Math.max(0, messageIndex - item.withPreviousMessages);
|
|
2110
|
+
for (let i = startIdx; i < messageIndex; i++) {
|
|
2111
|
+
additionalIndices.add(i);
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
if (item.withNextMessages) {
|
|
2115
|
+
const endIdx = Math.min(records.length - 1, messageIndex + item.withNextMessages);
|
|
2116
|
+
for (let i = messageIndex + 1; i <= endIdx; i++) {
|
|
2117
|
+
additionalIndices.add(i);
|
|
2118
|
+
}
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
}
|
|
2122
|
+
if (additionalIndices.size === 0) {
|
|
2123
|
+
return records;
|
|
2124
|
+
}
|
|
2125
|
+
const originalMatchIds = new Set(include.map((item) => item.id));
|
|
2126
|
+
const allIndices = /* @__PURE__ */ new Set();
|
|
2127
|
+
records.forEach((record, index) => {
|
|
2128
|
+
if (originalMatchIds.has(record.id)) {
|
|
2129
|
+
allIndices.add(index);
|
|
2130
|
+
}
|
|
2131
|
+
});
|
|
2132
|
+
additionalIndices.forEach((index) => {
|
|
2133
|
+
allIndices.add(index);
|
|
2134
|
+
});
|
|
2135
|
+
return Array.from(allIndices).sort((a, b) => a - b).map((index) => records[index]);
|
|
2136
|
+
}
|
|
2137
|
+
async getMessages({
|
|
2138
|
+
threadId,
|
|
2139
|
+
resourceId,
|
|
2140
|
+
selectBy,
|
|
2141
|
+
format,
|
|
2142
|
+
threadConfig
|
|
2143
|
+
}) {
|
|
2144
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format, threadConfig });
|
|
2145
|
+
}
|
|
2146
|
+
async getMessagesById({
|
|
2147
|
+
messageIds,
|
|
2148
|
+
format
|
|
2149
|
+
}) {
|
|
2150
|
+
return this.stores.memory.getMessagesById({ messageIds, format });
|
|
2151
|
+
}
|
|
2152
|
+
async saveMessages(args) {
|
|
2153
|
+
return this.stores.memory.saveMessages(args);
|
|
2154
|
+
}
|
|
2155
|
+
async getThreadsByResourceIdPaginated(args) {
|
|
2156
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
2157
|
+
}
|
|
2158
|
+
async getMessagesPaginated(args) {
|
|
2159
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
2160
|
+
}
|
|
2161
|
+
async updateMessages(_args) {
|
|
2162
|
+
return this.stores.memory.updateMessages(_args);
|
|
2163
|
+
}
|
|
2164
|
+
async getTraceById(args) {
|
|
2165
|
+
return this.stores.traces.getTraceById(args);
|
|
2166
|
+
}
|
|
2167
|
+
async getTraces(args) {
|
|
2168
|
+
return this.stores.traces.getTraces(args);
|
|
2169
|
+
}
|
|
2170
|
+
async getTracesPaginated(args) {
|
|
2171
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
2172
|
+
}
|
|
2173
|
+
async getEvalsByAgentName(agentName, type) {
|
|
2174
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
2175
|
+
}
|
|
2176
|
+
async getEvals(options) {
|
|
2177
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
2178
|
+
}
|
|
2179
|
+
async getWorkflowRuns(args) {
|
|
2180
|
+
return this.stores.workflows.getWorkflowRuns(args);
|
|
2181
|
+
}
|
|
2182
|
+
async getWorkflowRunById(args) {
|
|
2183
|
+
return this.stores.workflows.getWorkflowRunById(args);
|
|
2184
|
+
}
|
|
2185
|
+
async updateWorkflowResults({
|
|
2186
|
+
workflowName,
|
|
2187
|
+
runId,
|
|
2188
|
+
stepId,
|
|
2189
|
+
result,
|
|
2190
|
+
runtimeContext
|
|
2191
|
+
}) {
|
|
2192
|
+
return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, runtimeContext });
|
|
2193
|
+
}
|
|
2194
|
+
async updateWorkflowState({
|
|
2195
|
+
workflowName,
|
|
2196
|
+
runId,
|
|
2197
|
+
opts
|
|
2198
|
+
}) {
|
|
2199
|
+
return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
|
|
2200
|
+
}
|
|
2201
|
+
async persistWorkflowSnapshot({
|
|
2202
|
+
workflowName,
|
|
2203
|
+
runId,
|
|
2204
|
+
snapshot
|
|
2205
|
+
}) {
|
|
2206
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, snapshot });
|
|
2207
|
+
}
|
|
2208
|
+
async loadWorkflowSnapshot({
|
|
2209
|
+
workflowName,
|
|
2210
|
+
runId
|
|
2211
|
+
}) {
|
|
2212
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
2213
|
+
}
|
|
2214
|
+
async getScoreById({ id: _id }) {
|
|
2215
|
+
return this.stores.scores.getScoreById({ id: _id });
|
|
2216
|
+
}
|
|
2217
|
+
async getScoresByScorerId({
|
|
2218
|
+
scorerId,
|
|
2219
|
+
source,
|
|
2220
|
+
entityId,
|
|
2221
|
+
entityType,
|
|
2222
|
+
pagination
|
|
2223
|
+
}) {
|
|
2224
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, source, pagination, entityId, entityType });
|
|
2225
|
+
}
|
|
2226
|
+
async saveScore(_score) {
|
|
2227
|
+
return this.stores.scores.saveScore(_score);
|
|
2228
|
+
}
|
|
2229
|
+
async getScoresByRunId({
|
|
2230
|
+
runId,
|
|
2231
|
+
pagination
|
|
2232
|
+
}) {
|
|
2233
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
2234
|
+
}
|
|
2235
|
+
async getScoresByEntityId({
|
|
2236
|
+
entityId,
|
|
2237
|
+
entityType,
|
|
2238
|
+
pagination
|
|
2239
|
+
}) {
|
|
2240
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
2241
|
+
}
|
|
2242
|
+
};
|
|
2243
|
+
var LanceFilterTranslator = class extends BaseFilterTranslator {
|
|
2244
|
+
translate(filter) {
|
|
2245
|
+
if (!filter || Object.keys(filter).length === 0) {
|
|
2246
|
+
return "";
|
|
2247
|
+
}
|
|
2248
|
+
if (typeof filter === "object" && filter !== null) {
|
|
2249
|
+
const keys = Object.keys(filter);
|
|
2250
|
+
for (const key of keys) {
|
|
2251
|
+
if (key.includes(".") && !this.isNormalNestedField(key)) {
|
|
2252
|
+
throw new Error(`Field names containing periods (.) are not supported: ${key}`);
|
|
2253
|
+
}
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
return this.processFilter(filter);
|
|
2257
|
+
}
|
|
2258
|
+
processFilter(filter, parentPath = "") {
|
|
2259
|
+
if (filter === null) {
|
|
2260
|
+
return `${parentPath} IS NULL`;
|
|
2261
|
+
}
|
|
2262
|
+
if (filter instanceof Date) {
|
|
2263
|
+
return `${parentPath} = ${this.formatValue(filter)}`;
|
|
2264
|
+
}
|
|
2265
|
+
if (typeof filter === "object" && filter !== null) {
|
|
2266
|
+
const obj = filter;
|
|
2267
|
+
const keys = Object.keys(obj);
|
|
2268
|
+
if (keys.length === 1 && this.isOperator(keys[0])) {
|
|
2269
|
+
const operator = keys[0];
|
|
2270
|
+
const operatorValue = obj[operator];
|
|
2271
|
+
if (this.isLogicalOperator(operator)) {
|
|
2272
|
+
if (operator === "$and" || operator === "$or") {
|
|
2273
|
+
return this.processLogicalOperator(operator, operatorValue);
|
|
2274
|
+
}
|
|
2275
|
+
throw new Error(BaseFilterTranslator.ErrorMessages.UNSUPPORTED_OPERATOR(operator));
|
|
2276
|
+
}
|
|
2277
|
+
throw new Error(BaseFilterTranslator.ErrorMessages.INVALID_TOP_LEVEL_OPERATOR(operator));
|
|
2278
|
+
}
|
|
2279
|
+
for (const key of keys) {
|
|
2280
|
+
if (key.includes(".") && !this.isNormalNestedField(key)) {
|
|
2281
|
+
throw new Error(`Field names containing periods (.) are not supported: ${key}`);
|
|
2282
|
+
}
|
|
2283
|
+
}
|
|
2284
|
+
if (keys.length > 1) {
|
|
2285
|
+
const conditions = keys.map((key) => {
|
|
2286
|
+
const value = obj[key];
|
|
2287
|
+
if (this.isNestedObject(value) && !this.isDateObject(value)) {
|
|
2288
|
+
return this.processNestedObject(key, value);
|
|
2289
|
+
} else {
|
|
2290
|
+
return this.processField(key, value);
|
|
2291
|
+
}
|
|
2292
|
+
});
|
|
2293
|
+
return conditions.join(" AND ");
|
|
2294
|
+
}
|
|
2295
|
+
if (keys.length === 1) {
|
|
2296
|
+
const key = keys[0];
|
|
2297
|
+
const value = obj[key];
|
|
2298
|
+
if (this.isNestedObject(value) && !this.isDateObject(value)) {
|
|
2299
|
+
return this.processNestedObject(key, value);
|
|
2300
|
+
} else {
|
|
2301
|
+
return this.processField(key, value);
|
|
2302
|
+
}
|
|
2303
|
+
}
|
|
2304
|
+
}
|
|
2305
|
+
return "";
|
|
2306
|
+
}
|
|
2307
|
+
processLogicalOperator(operator, conditions) {
|
|
2308
|
+
if (!Array.isArray(conditions)) {
|
|
2309
|
+
throw new Error(`Logical operator ${operator} must have an array value`);
|
|
2310
|
+
}
|
|
2311
|
+
if (conditions.length === 0) {
|
|
2312
|
+
return operator === "$and" ? "true" : "false";
|
|
2313
|
+
}
|
|
2314
|
+
const sqlOperator = operator === "$and" ? "AND" : "OR";
|
|
2315
|
+
const processedConditions = conditions.map((condition) => {
|
|
2316
|
+
if (typeof condition !== "object" || condition === null) {
|
|
2317
|
+
throw new Error(BaseFilterTranslator.ErrorMessages.INVALID_LOGICAL_OPERATOR_CONTENT(operator));
|
|
2318
|
+
}
|
|
2319
|
+
const condObj = condition;
|
|
2320
|
+
const keys = Object.keys(condObj);
|
|
2321
|
+
if (keys.length === 1 && this.isOperator(keys[0])) {
|
|
2322
|
+
if (this.isLogicalOperator(keys[0])) {
|
|
2323
|
+
return `(${this.processLogicalOperator(keys[0], condObj[keys[0]])})`;
|
|
2324
|
+
} else {
|
|
2325
|
+
throw new Error(BaseFilterTranslator.ErrorMessages.UNSUPPORTED_OPERATOR(keys[0]));
|
|
2326
|
+
}
|
|
2327
|
+
}
|
|
2328
|
+
if (keys.length > 1) {
|
|
2329
|
+
return `(${this.processFilter(condition)})`;
|
|
2330
|
+
}
|
|
2331
|
+
return this.processFilter(condition);
|
|
2332
|
+
});
|
|
2333
|
+
return processedConditions.join(` ${sqlOperator} `);
|
|
2334
|
+
}
|
|
2335
|
+
processNestedObject(path, value) {
|
|
2336
|
+
if (typeof value !== "object" || value === null) {
|
|
2337
|
+
throw new Error(`Expected object for nested path ${path}`);
|
|
2338
|
+
}
|
|
2339
|
+
const obj = value;
|
|
2340
|
+
const keys = Object.keys(obj);
|
|
2341
|
+
if (keys.length === 0) {
|
|
2342
|
+
return `${path} = {}`;
|
|
2343
|
+
}
|
|
2344
|
+
if (keys.every((k) => this.isOperator(k))) {
|
|
2345
|
+
return this.processOperators(path, obj);
|
|
2346
|
+
}
|
|
2347
|
+
const conditions = keys.map((key) => {
|
|
2348
|
+
const nestedPath = key.includes(".") ? `${path}.${key}` : `${path}.${key}`;
|
|
2349
|
+
if (this.isNestedObject(obj[key]) && !this.isDateObject(obj[key])) {
|
|
2350
|
+
return this.processNestedObject(nestedPath, obj[key]);
|
|
2351
|
+
} else {
|
|
2352
|
+
return this.processField(nestedPath, obj[key]);
|
|
2353
|
+
}
|
|
2354
|
+
});
|
|
2355
|
+
return conditions.join(" AND ");
|
|
2356
|
+
}
|
|
2357
|
+
processField(field, value) {
|
|
2358
|
+
if (field.includes(".") && !this.isNormalNestedField(field)) {
|
|
2359
|
+
throw new Error(`Field names containing periods (.) are not supported: ${field}`);
|
|
2360
|
+
}
|
|
2361
|
+
const escapedField = this.escapeFieldName(field);
|
|
2362
|
+
if (value === null) {
|
|
2363
|
+
return `${escapedField} IS NULL`;
|
|
2364
|
+
}
|
|
2365
|
+
if (value instanceof Date) {
|
|
2366
|
+
return `${escapedField} = ${this.formatValue(value)}`;
|
|
2367
|
+
}
|
|
2368
|
+
if (Array.isArray(value)) {
|
|
2369
|
+
if (value.length === 0) {
|
|
2370
|
+
return "false";
|
|
2371
|
+
}
|
|
2372
|
+
const normalizedValues = this.normalizeArrayValues(value);
|
|
2373
|
+
return `${escapedField} IN (${this.formatArrayValues(normalizedValues)})`;
|
|
2374
|
+
}
|
|
2375
|
+
if (this.isOperatorObject(value)) {
|
|
2376
|
+
return this.processOperators(field, value);
|
|
2377
|
+
}
|
|
2378
|
+
return `${escapedField} = ${this.formatValue(this.normalizeComparisonValue(value))}`;
|
|
2379
|
+
}
|
|
2380
|
+
processOperators(field, operators) {
|
|
2381
|
+
const escapedField = this.escapeFieldName(field);
|
|
2382
|
+
const operatorKeys = Object.keys(operators);
|
|
2383
|
+
if (operatorKeys.some((op) => this.isLogicalOperator(op))) {
|
|
2384
|
+
const logicalOp = operatorKeys.find((op) => this.isLogicalOperator(op)) || "";
|
|
2385
|
+
throw new Error(`Unsupported operator: ${logicalOp} cannot be used at field level`);
|
|
2386
|
+
}
|
|
2387
|
+
return operatorKeys.map((op) => {
|
|
2388
|
+
const value = operators[op];
|
|
2389
|
+
if (!this.isFieldOperator(op) && !this.isCustomOperator(op)) {
|
|
2390
|
+
throw new Error(BaseFilterTranslator.ErrorMessages.UNSUPPORTED_OPERATOR(op));
|
|
2391
|
+
}
|
|
2392
|
+
switch (op) {
|
|
2393
|
+
case "$eq":
|
|
2394
|
+
if (value === null) {
|
|
2395
|
+
return `${escapedField} IS NULL`;
|
|
2396
|
+
}
|
|
2397
|
+
return `${escapedField} = ${this.formatValue(this.normalizeComparisonValue(value))}`;
|
|
2398
|
+
case "$ne":
|
|
2399
|
+
if (value === null) {
|
|
2400
|
+
return `${escapedField} IS NOT NULL`;
|
|
2401
|
+
}
|
|
2402
|
+
return `${escapedField} != ${this.formatValue(this.normalizeComparisonValue(value))}`;
|
|
2403
|
+
case "$gt":
|
|
2404
|
+
return `${escapedField} > ${this.formatValue(this.normalizeComparisonValue(value))}`;
|
|
2405
|
+
case "$gte":
|
|
2406
|
+
return `${escapedField} >= ${this.formatValue(this.normalizeComparisonValue(value))}`;
|
|
2407
|
+
case "$lt":
|
|
2408
|
+
return `${escapedField} < ${this.formatValue(this.normalizeComparisonValue(value))}`;
|
|
2409
|
+
case "$lte":
|
|
2410
|
+
return `${escapedField} <= ${this.formatValue(this.normalizeComparisonValue(value))}`;
|
|
2411
|
+
case "$in":
|
|
2412
|
+
if (!Array.isArray(value)) {
|
|
2413
|
+
throw new Error(`$in operator requires array value for field: ${field}`);
|
|
2414
|
+
}
|
|
2415
|
+
if (value.length === 0) {
|
|
2416
|
+
return "false";
|
|
2417
|
+
}
|
|
2418
|
+
const normalizedValues = this.normalizeArrayValues(value);
|
|
2419
|
+
return `${escapedField} IN (${this.formatArrayValues(normalizedValues)})`;
|
|
2420
|
+
case "$like":
|
|
2421
|
+
return `${escapedField} LIKE ${this.formatValue(value)}`;
|
|
2422
|
+
case "$notLike":
|
|
2423
|
+
return `${escapedField} NOT LIKE ${this.formatValue(value)}`;
|
|
2424
|
+
case "$regex":
|
|
2425
|
+
return `regexp_match(${escapedField}, ${this.formatValue(value)})`;
|
|
2426
|
+
default:
|
|
2427
|
+
throw new Error(BaseFilterTranslator.ErrorMessages.UNSUPPORTED_OPERATOR(op));
|
|
2428
|
+
}
|
|
2429
|
+
}).join(" AND ");
|
|
2430
|
+
}
|
|
2431
|
+
formatValue(value) {
|
|
2432
|
+
if (value === null) {
|
|
2433
|
+
return "NULL";
|
|
2434
|
+
}
|
|
2435
|
+
if (typeof value === "string") {
|
|
2436
|
+
return `'${value.replace(/'/g, "''")}'`;
|
|
2437
|
+
}
|
|
2438
|
+
if (typeof value === "number") {
|
|
2439
|
+
return value.toString();
|
|
2440
|
+
}
|
|
2441
|
+
if (typeof value === "boolean") {
|
|
2442
|
+
return value ? "true" : "false";
|
|
2443
|
+
}
|
|
2444
|
+
if (value instanceof Date) {
|
|
2445
|
+
return `timestamp '${value.toISOString()}'`;
|
|
2446
|
+
}
|
|
2447
|
+
if (typeof value === "object") {
|
|
2448
|
+
if (value instanceof Date) {
|
|
2449
|
+
return `timestamp '${value.toISOString()}'`;
|
|
2450
|
+
}
|
|
2451
|
+
return JSON.stringify(value);
|
|
2452
|
+
}
|
|
2453
|
+
return String(value);
|
|
2454
|
+
}
|
|
2455
|
+
formatArrayValues(array) {
|
|
2456
|
+
return array.map((item) => this.formatValue(item)).join(", ");
|
|
2457
|
+
}
|
|
2458
|
+
normalizeArrayValues(array) {
|
|
2459
|
+
return array.map((item) => {
|
|
2460
|
+
if (item instanceof Date) {
|
|
2461
|
+
return item;
|
|
2462
|
+
}
|
|
2463
|
+
return this.normalizeComparisonValue(item);
|
|
2464
|
+
});
|
|
2465
|
+
}
|
|
2466
|
+
normalizeComparisonValue(value) {
|
|
2467
|
+
if (value instanceof Date) {
|
|
2468
|
+
return value;
|
|
2469
|
+
}
|
|
2470
|
+
return super.normalizeComparisonValue(value);
|
|
2471
|
+
}
|
|
2472
|
+
isOperatorObject(value) {
|
|
2473
|
+
if (typeof value !== "object" || value === null) {
|
|
2474
|
+
return false;
|
|
2475
|
+
}
|
|
2476
|
+
const obj = value;
|
|
2477
|
+
const keys = Object.keys(obj);
|
|
2478
|
+
return keys.length > 0 && keys.some((key) => this.isOperator(key));
|
|
2479
|
+
}
|
|
2480
|
+
isNestedObject(value) {
|
|
2481
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2482
|
+
}
|
|
2483
|
+
isNormalNestedField(field) {
|
|
2484
|
+
const parts = field.split(".");
|
|
2485
|
+
return !field.startsWith(".") && !field.endsWith(".") && parts.every((part) => part.trim().length > 0);
|
|
2486
|
+
}
|
|
2487
|
+
escapeFieldName(field) {
|
|
2488
|
+
if (field.includes(" ") || field.includes("-") || /^[A-Z]+$/.test(field) || this.isSqlKeyword(field)) {
|
|
2489
|
+
if (field.includes(".")) {
|
|
2490
|
+
return field.split(".").map((part) => `\`${part}\``).join(".");
|
|
2491
|
+
}
|
|
2492
|
+
return `\`${field}\``;
|
|
2493
|
+
}
|
|
2494
|
+
return field;
|
|
2495
|
+
}
|
|
2496
|
+
isSqlKeyword(str) {
|
|
2497
|
+
const sqlKeywords = [
|
|
2498
|
+
"SELECT",
|
|
2499
|
+
"FROM",
|
|
2500
|
+
"WHERE",
|
|
2501
|
+
"AND",
|
|
2502
|
+
"OR",
|
|
2503
|
+
"NOT",
|
|
2504
|
+
"INSERT",
|
|
2505
|
+
"UPDATE",
|
|
2506
|
+
"DELETE",
|
|
2507
|
+
"CREATE",
|
|
2508
|
+
"ALTER",
|
|
2509
|
+
"DROP",
|
|
2510
|
+
"TABLE",
|
|
2511
|
+
"VIEW",
|
|
2512
|
+
"INDEX",
|
|
2513
|
+
"JOIN",
|
|
2514
|
+
"INNER",
|
|
2515
|
+
"OUTER",
|
|
2516
|
+
"LEFT",
|
|
2517
|
+
"RIGHT",
|
|
2518
|
+
"FULL",
|
|
2519
|
+
"UNION",
|
|
2520
|
+
"ALL",
|
|
2521
|
+
"DISTINCT",
|
|
2522
|
+
"AS",
|
|
2523
|
+
"ON",
|
|
2524
|
+
"BETWEEN",
|
|
2525
|
+
"LIKE",
|
|
2526
|
+
"IN",
|
|
2527
|
+
"IS",
|
|
2528
|
+
"NULL",
|
|
2529
|
+
"TRUE",
|
|
2530
|
+
"FALSE",
|
|
2531
|
+
"ASC",
|
|
2532
|
+
"DESC",
|
|
2533
|
+
"GROUP",
|
|
2534
|
+
"ORDER",
|
|
2535
|
+
"BY",
|
|
2536
|
+
"HAVING",
|
|
2537
|
+
"LIMIT",
|
|
2538
|
+
"OFFSET",
|
|
2539
|
+
"CASE",
|
|
2540
|
+
"WHEN",
|
|
2541
|
+
"THEN",
|
|
2542
|
+
"ELSE",
|
|
2543
|
+
"END",
|
|
2544
|
+
"CAST",
|
|
2545
|
+
"CUBE"
|
|
2546
|
+
];
|
|
2547
|
+
return sqlKeywords.includes(str.toUpperCase());
|
|
2548
|
+
}
|
|
2549
|
+
isDateObject(value) {
|
|
2550
|
+
return value instanceof Date;
|
|
2551
|
+
}
|
|
2552
|
+
/**
|
|
2553
|
+
* Override getSupportedOperators to add custom operators for LanceDB
|
|
2554
|
+
*/
|
|
2555
|
+
getSupportedOperators() {
|
|
2556
|
+
return {
|
|
2557
|
+
...BaseFilterTranslator.DEFAULT_OPERATORS,
|
|
2558
|
+
custom: ["$like", "$notLike", "$regex"]
|
|
2559
|
+
};
|
|
2560
|
+
}
|
|
2561
|
+
};
|
|
2562
|
+
|
|
2563
|
+
// src/vector/index.ts
|
|
2564
|
+
var LanceVectorStore = class _LanceVectorStore extends MastraVector {
|
|
2565
|
+
lanceClient;
|
|
2566
|
+
/**
|
|
2567
|
+
* Creates a new instance of LanceVectorStore
|
|
2568
|
+
* @param uri The URI to connect to LanceDB
|
|
2569
|
+
* @param options connection options
|
|
2570
|
+
*
|
|
2571
|
+
* Usage:
|
|
2572
|
+
*
|
|
2573
|
+
* Connect to a local database
|
|
2574
|
+
* ```ts
|
|
2575
|
+
* const store = await LanceVectorStore.create('/path/to/db');
|
|
2576
|
+
* ```
|
|
2577
|
+
*
|
|
2578
|
+
* Connect to a LanceDB cloud database
|
|
2579
|
+
* ```ts
|
|
2580
|
+
* const store = await LanceVectorStore.create('db://host:port');
|
|
2581
|
+
* ```
|
|
2582
|
+
*
|
|
2583
|
+
* Connect to a cloud database
|
|
2584
|
+
* ```ts
|
|
2585
|
+
* const store = await LanceVectorStore.create('s3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
2586
|
+
* ```
|
|
2587
|
+
*/
|
|
2588
|
+
static async create(uri, options) {
|
|
2589
|
+
const instance = new _LanceVectorStore();
|
|
2590
|
+
try {
|
|
2591
|
+
instance.lanceClient = await connect(uri, options);
|
|
2592
|
+
return instance;
|
|
2593
|
+
} catch (e) {
|
|
2594
|
+
throw new MastraError(
|
|
2595
|
+
{
|
|
2596
|
+
id: "STORAGE_LANCE_VECTOR_CONNECT_FAILED",
|
|
2597
|
+
domain: ErrorDomain.STORAGE,
|
|
2598
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2599
|
+
details: { uri }
|
|
2600
|
+
},
|
|
2601
|
+
e
|
|
2602
|
+
);
|
|
2603
|
+
}
|
|
2604
|
+
}
|
|
2605
|
+
/**
|
|
2606
|
+
* @internal
|
|
2607
|
+
* Private constructor to enforce using the create factory method
|
|
2608
|
+
*/
|
|
2609
|
+
constructor() {
|
|
2610
|
+
super();
|
|
2611
|
+
}
|
|
2612
|
+
close() {
|
|
2613
|
+
if (this.lanceClient) {
|
|
2614
|
+
this.lanceClient.close();
|
|
2615
|
+
}
|
|
2616
|
+
}
|
|
2617
|
+
async query({
|
|
2618
|
+
tableName,
|
|
2619
|
+
queryVector,
|
|
2620
|
+
filter,
|
|
2621
|
+
includeVector = false,
|
|
2622
|
+
topK = 10,
|
|
2623
|
+
columns = [],
|
|
2624
|
+
includeAllColumns = false
|
|
2625
|
+
}) {
|
|
2626
|
+
try {
|
|
2627
|
+
if (!this.lanceClient) {
|
|
2628
|
+
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2629
|
+
}
|
|
2630
|
+
if (!tableName) {
|
|
2631
|
+
throw new Error("tableName is required");
|
|
2632
|
+
}
|
|
2633
|
+
if (!queryVector) {
|
|
2634
|
+
throw new Error("queryVector is required");
|
|
2635
|
+
}
|
|
2636
|
+
} catch (error) {
|
|
2637
|
+
throw new MastraError(
|
|
2638
|
+
{
|
|
2639
|
+
id: "STORAGE_LANCE_VECTOR_QUERY_FAILED_INVALID_ARGS",
|
|
2640
|
+
domain: ErrorDomain.STORAGE,
|
|
2641
|
+
category: ErrorCategory.USER,
|
|
2642
|
+
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
2643
|
+
details: { tableName }
|
|
2644
|
+
},
|
|
2645
|
+
error
|
|
2646
|
+
);
|
|
2647
|
+
}
|
|
2648
|
+
try {
|
|
2649
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
2650
|
+
const selectColumns = [...columns];
|
|
2651
|
+
if (!selectColumns.includes("id")) {
|
|
2652
|
+
selectColumns.push("id");
|
|
2653
|
+
}
|
|
2654
|
+
let query = table.search(queryVector);
|
|
2655
|
+
if (filter && Object.keys(filter).length > 0) {
|
|
2656
|
+
const whereClause = this.filterTranslator(filter);
|
|
2657
|
+
this.logger.debug(`Where clause generated: ${whereClause}`);
|
|
2658
|
+
query = query.where(whereClause);
|
|
2659
|
+
}
|
|
2660
|
+
if (!includeAllColumns && selectColumns.length > 0) {
|
|
2661
|
+
query = query.select(selectColumns);
|
|
2662
|
+
}
|
|
2663
|
+
query = query.limit(topK);
|
|
2664
|
+
const results = await query.toArray();
|
|
2665
|
+
return results.map((result) => {
|
|
2666
|
+
const flatMetadata = {};
|
|
2667
|
+
Object.keys(result).forEach((key) => {
|
|
2668
|
+
if (key !== "id" && key !== "score" && key !== "vector" && key !== "_distance") {
|
|
2669
|
+
if (key.startsWith("metadata_")) {
|
|
2670
|
+
const metadataKey = key.substring("metadata_".length);
|
|
2671
|
+
flatMetadata[metadataKey] = result[key];
|
|
2672
|
+
}
|
|
2673
|
+
}
|
|
2674
|
+
});
|
|
2675
|
+
const metadata = this.unflattenObject(flatMetadata);
|
|
2676
|
+
return {
|
|
2677
|
+
id: String(result.id || ""),
|
|
2678
|
+
metadata,
|
|
2679
|
+
vector: includeVector && result.vector ? Array.isArray(result.vector) ? result.vector : Array.from(result.vector) : void 0,
|
|
2680
|
+
document: result.document,
|
|
2681
|
+
score: result._distance
|
|
2682
|
+
};
|
|
2683
|
+
});
|
|
2684
|
+
} catch (error) {
|
|
2685
|
+
throw new MastraError(
|
|
2686
|
+
{
|
|
2687
|
+
id: "STORAGE_LANCE_VECTOR_QUERY_FAILED",
|
|
2688
|
+
domain: ErrorDomain.STORAGE,
|
|
2689
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2690
|
+
details: { tableName, includeVector, columnsCount: columns?.length, includeAllColumns }
|
|
2691
|
+
},
|
|
2692
|
+
error
|
|
2693
|
+
);
|
|
2694
|
+
}
|
|
2695
|
+
}
|
|
2696
|
+
filterTranslator(filter) {
|
|
2697
|
+
const processFilterKeys = (filterObj) => {
|
|
2698
|
+
const result = {};
|
|
2699
|
+
Object.entries(filterObj).forEach(([key, value]) => {
|
|
2700
|
+
if (key === "$or" || key === "$and" || key === "$not" || key === "$in") {
|
|
2701
|
+
if (Array.isArray(value)) {
|
|
2702
|
+
result[key] = value.map(
|
|
2703
|
+
(item) => typeof item === "object" && item !== null ? processFilterKeys(item) : item
|
|
2704
|
+
);
|
|
2705
|
+
} else {
|
|
2706
|
+
result[key] = value;
|
|
2707
|
+
}
|
|
2708
|
+
} else if (key.startsWith("metadata_")) {
|
|
2709
|
+
result[key] = value;
|
|
2710
|
+
} else {
|
|
2711
|
+
if (key.includes(".")) {
|
|
2712
|
+
const convertedKey = `metadata_${key.replace(/\./g, "_")}`;
|
|
2713
|
+
result[convertedKey] = value;
|
|
2714
|
+
} else {
|
|
2715
|
+
result[`metadata_${key}`] = value;
|
|
2716
|
+
}
|
|
2717
|
+
}
|
|
2718
|
+
});
|
|
2719
|
+
return result;
|
|
2720
|
+
};
|
|
2721
|
+
const prefixedFilter = filter && typeof filter === "object" ? processFilterKeys(filter) : {};
|
|
2722
|
+
const translator = new LanceFilterTranslator();
|
|
2723
|
+
return translator.translate(prefixedFilter);
|
|
2724
|
+
}
|
|
2725
|
+
async upsert({ tableName, vectors, metadata = [], ids = [] }) {
|
|
2726
|
+
try {
|
|
2727
|
+
if (!this.lanceClient) {
|
|
2728
|
+
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2729
|
+
}
|
|
2730
|
+
if (!tableName) {
|
|
2731
|
+
throw new Error("tableName is required");
|
|
2732
|
+
}
|
|
2733
|
+
if (!vectors || !Array.isArray(vectors) || vectors.length === 0) {
|
|
2734
|
+
throw new Error("vectors array is required and must not be empty");
|
|
2735
|
+
}
|
|
2736
|
+
} catch (error) {
|
|
2737
|
+
throw new MastraError(
|
|
2738
|
+
{
|
|
2739
|
+
id: "STORAGE_LANCE_VECTOR_UPSERT_FAILED_INVALID_ARGS",
|
|
2740
|
+
domain: ErrorDomain.STORAGE,
|
|
2741
|
+
category: ErrorCategory.USER,
|
|
2742
|
+
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
2743
|
+
details: { tableName }
|
|
2744
|
+
},
|
|
2745
|
+
error
|
|
2746
|
+
);
|
|
2747
|
+
}
|
|
2748
|
+
try {
|
|
2749
|
+
const tables = await this.lanceClient.tableNames();
|
|
2750
|
+
if (!tables.includes(tableName)) {
|
|
2751
|
+
throw new Error(`Table ${tableName} does not exist`);
|
|
2752
|
+
}
|
|
2753
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
2754
|
+
const vectorIds = ids.length === vectors.length ? ids : vectors.map((_, i) => ids[i] || crypto.randomUUID());
|
|
2755
|
+
const data = vectors.map((vector, i) => {
|
|
2756
|
+
const id = String(vectorIds[i]);
|
|
2757
|
+
const metadataItem = metadata[i] || {};
|
|
2758
|
+
const rowData = {
|
|
2759
|
+
id,
|
|
2760
|
+
vector
|
|
2761
|
+
};
|
|
2762
|
+
if (Object.keys(metadataItem).length > 0) {
|
|
2763
|
+
const flattenedMetadata = this.flattenObject(metadataItem, "metadata");
|
|
2764
|
+
Object.entries(flattenedMetadata).forEach(([key, value]) => {
|
|
2765
|
+
rowData[key] = value;
|
|
2766
|
+
});
|
|
2767
|
+
}
|
|
2768
|
+
return rowData;
|
|
2769
|
+
});
|
|
2770
|
+
await table.add(data, { mode: "overwrite" });
|
|
2771
|
+
return vectorIds;
|
|
2772
|
+
} catch (error) {
|
|
2773
|
+
throw new MastraError(
|
|
2774
|
+
{
|
|
2775
|
+
id: "STORAGE_LANCE_VECTOR_UPSERT_FAILED",
|
|
2776
|
+
domain: ErrorDomain.STORAGE,
|
|
2777
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2778
|
+
details: { tableName, vectorCount: vectors.length, metadataCount: metadata.length, idsCount: ids.length }
|
|
2779
|
+
},
|
|
2780
|
+
error
|
|
2781
|
+
);
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2784
|
+
/**
|
|
2785
|
+
* Flattens a nested object, creating new keys with underscores for nested properties.
|
|
2786
|
+
* Example: { metadata: { text: 'test' } } → { metadata_text: 'test' }
|
|
2787
|
+
*/
|
|
2788
|
+
flattenObject(obj, prefix = "") {
|
|
2789
|
+
return Object.keys(obj).reduce((acc, k) => {
|
|
2790
|
+
const pre = prefix.length ? `${prefix}_` : "";
|
|
2791
|
+
if (typeof obj[k] === "object" && obj[k] !== null && !Array.isArray(obj[k])) {
|
|
2792
|
+
Object.assign(acc, this.flattenObject(obj[k], pre + k));
|
|
2793
|
+
} else {
|
|
2794
|
+
acc[pre + k] = obj[k];
|
|
2795
|
+
}
|
|
2796
|
+
return acc;
|
|
2797
|
+
}, {});
|
|
2798
|
+
}
|
|
2799
|
+
async createTable(tableName, data, options) {
|
|
2800
|
+
if (!this.lanceClient) {
|
|
2801
|
+
throw new MastraError({
|
|
2802
|
+
id: "STORAGE_LANCE_VECTOR_CREATE_TABLE_FAILED_INVALID_ARGS",
|
|
2803
|
+
domain: ErrorDomain.STORAGE,
|
|
2804
|
+
category: ErrorCategory.USER,
|
|
2805
|
+
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
2806
|
+
details: { tableName }
|
|
2807
|
+
});
|
|
2808
|
+
}
|
|
2809
|
+
if (Array.isArray(data)) {
|
|
2810
|
+
data = data.map((record) => this.flattenObject(record));
|
|
2811
|
+
}
|
|
2812
|
+
try {
|
|
2813
|
+
return await this.lanceClient.createTable(tableName, data, options);
|
|
2814
|
+
} catch (error) {
|
|
2815
|
+
throw new MastraError(
|
|
2816
|
+
{
|
|
2817
|
+
id: "STORAGE_LANCE_VECTOR_CREATE_TABLE_FAILED",
|
|
2818
|
+
domain: ErrorDomain.STORAGE,
|
|
2819
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2820
|
+
details: { tableName }
|
|
2821
|
+
},
|
|
2822
|
+
error
|
|
2823
|
+
);
|
|
2824
|
+
}
|
|
2825
|
+
}
|
|
2826
|
+
async listTables() {
|
|
2827
|
+
if (!this.lanceClient) {
|
|
2828
|
+
throw new MastraError({
|
|
2829
|
+
id: "STORAGE_LANCE_VECTOR_LIST_TABLES_FAILED_INVALID_ARGS",
|
|
2830
|
+
domain: ErrorDomain.STORAGE,
|
|
2831
|
+
category: ErrorCategory.USER,
|
|
2832
|
+
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
2833
|
+
details: { methodName: "listTables" }
|
|
2834
|
+
});
|
|
2835
|
+
}
|
|
2836
|
+
try {
|
|
2837
|
+
return await this.lanceClient.tableNames();
|
|
2838
|
+
} catch (error) {
|
|
2839
|
+
throw new MastraError(
|
|
2840
|
+
{
|
|
2841
|
+
id: "STORAGE_LANCE_VECTOR_LIST_TABLES_FAILED",
|
|
2842
|
+
domain: ErrorDomain.STORAGE,
|
|
2843
|
+
category: ErrorCategory.THIRD_PARTY
|
|
2844
|
+
},
|
|
2845
|
+
error
|
|
2846
|
+
);
|
|
2847
|
+
}
|
|
2848
|
+
}
|
|
2849
|
+
async getTableSchema(tableName) {
|
|
2850
|
+
if (!this.lanceClient) {
|
|
2851
|
+
throw new MastraError({
|
|
2852
|
+
id: "STORAGE_LANCE_VECTOR_GET_TABLE_SCHEMA_FAILED_INVALID_ARGS",
|
|
2853
|
+
domain: ErrorDomain.STORAGE,
|
|
2854
|
+
category: ErrorCategory.USER,
|
|
2855
|
+
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
2856
|
+
details: { tableName }
|
|
2857
|
+
});
|
|
2858
|
+
}
|
|
2859
|
+
try {
|
|
2860
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
2861
|
+
return await table.schema();
|
|
2862
|
+
} catch (error) {
|
|
2863
|
+
throw new MastraError(
|
|
2864
|
+
{
|
|
2865
|
+
id: "STORAGE_LANCE_VECTOR_GET_TABLE_SCHEMA_FAILED",
|
|
2866
|
+
domain: ErrorDomain.STORAGE,
|
|
2867
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2868
|
+
details: { tableName }
|
|
2869
|
+
},
|
|
2870
|
+
error
|
|
2871
|
+
);
|
|
2872
|
+
}
|
|
2873
|
+
}
|
|
2874
|
+
/**
|
|
2875
|
+
* indexName is actually a column name in a table in lanceDB
|
|
2876
|
+
*/
|
|
2877
|
+
async createIndex({
|
|
2878
|
+
tableName,
|
|
2879
|
+
indexName,
|
|
2880
|
+
dimension,
|
|
2881
|
+
metric = "cosine",
|
|
2882
|
+
indexConfig = {}
|
|
2883
|
+
}) {
|
|
2884
|
+
try {
|
|
2885
|
+
if (!this.lanceClient) {
|
|
2886
|
+
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2887
|
+
}
|
|
2888
|
+
if (!tableName) {
|
|
2889
|
+
throw new Error("tableName is required");
|
|
2890
|
+
}
|
|
2891
|
+
if (!indexName) {
|
|
2892
|
+
throw new Error("indexName is required");
|
|
2893
|
+
}
|
|
2894
|
+
if (typeof dimension !== "number" || dimension <= 0) {
|
|
2895
|
+
throw new Error("dimension must be a positive number");
|
|
2896
|
+
}
|
|
2897
|
+
} catch (err) {
|
|
2898
|
+
throw new MastraError(
|
|
2899
|
+
{
|
|
2900
|
+
id: "STORAGE_LANCE_VECTOR_CREATE_INDEX_FAILED_INVALID_ARGS",
|
|
2901
|
+
domain: ErrorDomain.STORAGE,
|
|
2902
|
+
category: ErrorCategory.USER,
|
|
2903
|
+
details: { tableName: tableName || "", indexName, dimension, metric }
|
|
2904
|
+
},
|
|
2905
|
+
err
|
|
2906
|
+
);
|
|
2907
|
+
}
|
|
2908
|
+
try {
|
|
2909
|
+
const tables = await this.lanceClient.tableNames();
|
|
2910
|
+
if (!tables.includes(tableName)) {
|
|
2911
|
+
throw new Error(
|
|
2912
|
+
`Table ${tableName} does not exist. Please create the table first by calling createTable() method.`
|
|
2913
|
+
);
|
|
2914
|
+
}
|
|
2915
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
2916
|
+
let metricType;
|
|
2917
|
+
if (metric === "euclidean") {
|
|
2918
|
+
metricType = "l2";
|
|
2919
|
+
} else if (metric === "dotproduct") {
|
|
2920
|
+
metricType = "dot";
|
|
2921
|
+
} else if (metric === "cosine") {
|
|
2922
|
+
metricType = "cosine";
|
|
2923
|
+
}
|
|
2924
|
+
if (indexConfig.type === "ivfflat") {
|
|
2925
|
+
await table.createIndex(indexName, {
|
|
2926
|
+
config: Index.ivfPq({
|
|
2927
|
+
numPartitions: indexConfig.numPartitions || 128,
|
|
2928
|
+
numSubVectors: indexConfig.numSubVectors || 16,
|
|
2929
|
+
distanceType: metricType
|
|
2930
|
+
})
|
|
2931
|
+
});
|
|
2932
|
+
} else {
|
|
2933
|
+
this.logger.debug("Creating HNSW PQ index with config:", indexConfig);
|
|
2934
|
+
await table.createIndex(indexName, {
|
|
2935
|
+
config: Index.hnswPq({
|
|
2936
|
+
m: indexConfig?.hnsw?.m || 16,
|
|
2937
|
+
efConstruction: indexConfig?.hnsw?.efConstruction || 100,
|
|
2938
|
+
distanceType: metricType
|
|
2939
|
+
})
|
|
2940
|
+
});
|
|
2941
|
+
}
|
|
2942
|
+
} catch (error) {
|
|
2943
|
+
throw new MastraError(
|
|
2944
|
+
{
|
|
2945
|
+
id: "STORAGE_LANCE_VECTOR_CREATE_INDEX_FAILED",
|
|
2946
|
+
domain: ErrorDomain.STORAGE,
|
|
2947
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
2948
|
+
details: { tableName: tableName || "", indexName, dimension }
|
|
2949
|
+
},
|
|
2950
|
+
error
|
|
2951
|
+
);
|
|
2952
|
+
}
|
|
2953
|
+
}
|
|
2954
|
+
async listIndexes() {
|
|
2955
|
+
if (!this.lanceClient) {
|
|
2956
|
+
throw new MastraError({
|
|
2957
|
+
id: "STORAGE_LANCE_VECTOR_LIST_INDEXES_FAILED_INVALID_ARGS",
|
|
2958
|
+
domain: ErrorDomain.STORAGE,
|
|
2959
|
+
category: ErrorCategory.USER,
|
|
2960
|
+
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance",
|
|
2961
|
+
details: { methodName: "listIndexes" }
|
|
2962
|
+
});
|
|
2963
|
+
}
|
|
2964
|
+
try {
|
|
2965
|
+
const tables = await this.lanceClient.tableNames();
|
|
2966
|
+
const allIndices = [];
|
|
2967
|
+
for (const tableName of tables) {
|
|
2968
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
2969
|
+
const tableIndices = await table.listIndices();
|
|
2970
|
+
allIndices.push(...tableIndices.map((index) => index.name));
|
|
2971
|
+
}
|
|
2972
|
+
return allIndices;
|
|
2973
|
+
} catch (error) {
|
|
2974
|
+
throw new MastraError(
|
|
2975
|
+
{
|
|
2976
|
+
id: "STORAGE_LANCE_VECTOR_LIST_INDEXES_FAILED",
|
|
2977
|
+
domain: ErrorDomain.STORAGE,
|
|
2978
|
+
category: ErrorCategory.THIRD_PARTY
|
|
2979
|
+
},
|
|
2980
|
+
error
|
|
2981
|
+
);
|
|
2982
|
+
}
|
|
2983
|
+
}
|
|
2984
|
+
async describeIndex({ indexName }) {
|
|
2985
|
+
try {
|
|
2986
|
+
if (!this.lanceClient) {
|
|
2987
|
+
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
2988
|
+
}
|
|
2989
|
+
if (!indexName) {
|
|
2990
|
+
throw new Error("indexName is required");
|
|
2991
|
+
}
|
|
2992
|
+
} catch (err) {
|
|
2993
|
+
throw new MastraError(
|
|
2994
|
+
{
|
|
2995
|
+
id: "STORAGE_LANCE_VECTOR_DESCRIBE_INDEX_FAILED_INVALID_ARGS",
|
|
2996
|
+
domain: ErrorDomain.STORAGE,
|
|
2997
|
+
category: ErrorCategory.USER,
|
|
2998
|
+
details: { indexName }
|
|
2999
|
+
},
|
|
3000
|
+
err
|
|
3001
|
+
);
|
|
3002
|
+
}
|
|
3003
|
+
try {
|
|
3004
|
+
const tables = await this.lanceClient.tableNames();
|
|
3005
|
+
for (const tableName of tables) {
|
|
3006
|
+
this.logger.debug("Checking table:" + tableName);
|
|
3007
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
3008
|
+
const tableIndices = await table.listIndices();
|
|
3009
|
+
const foundIndex = tableIndices.find((index) => index.name === indexName);
|
|
3010
|
+
if (foundIndex) {
|
|
3011
|
+
const stats = await table.indexStats(foundIndex.name);
|
|
3012
|
+
if (!stats) {
|
|
3013
|
+
throw new Error(`Index stats not found for index: ${indexName}`);
|
|
3014
|
+
}
|
|
3015
|
+
const schema = await table.schema();
|
|
3016
|
+
const vectorCol = foundIndex.columns[0] || "vector";
|
|
3017
|
+
const vectorField = schema.fields.find((field) => field.name === vectorCol);
|
|
3018
|
+
const dimension = vectorField?.type?.["listSize"] || 0;
|
|
3019
|
+
return {
|
|
3020
|
+
dimension,
|
|
3021
|
+
metric: stats.distanceType,
|
|
3022
|
+
count: stats.numIndexedRows
|
|
3023
|
+
};
|
|
3024
|
+
}
|
|
3025
|
+
}
|
|
3026
|
+
throw new Error(`IndexName: ${indexName} not found`);
|
|
3027
|
+
} catch (error) {
|
|
3028
|
+
throw new MastraError(
|
|
3029
|
+
{
|
|
3030
|
+
id: "STORAGE_LANCE_VECTOR_DESCRIBE_INDEX_FAILED",
|
|
3031
|
+
domain: ErrorDomain.STORAGE,
|
|
3032
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
3033
|
+
details: { indexName }
|
|
3034
|
+
},
|
|
3035
|
+
error
|
|
3036
|
+
);
|
|
3037
|
+
}
|
|
3038
|
+
}
|
|
3039
|
+
async deleteIndex({ indexName }) {
|
|
3040
|
+
try {
|
|
3041
|
+
if (!this.lanceClient) {
|
|
3042
|
+
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
3043
|
+
}
|
|
3044
|
+
if (!indexName) {
|
|
3045
|
+
throw new Error("indexName is required");
|
|
3046
|
+
}
|
|
3047
|
+
} catch (err) {
|
|
3048
|
+
throw new MastraError(
|
|
3049
|
+
{
|
|
3050
|
+
id: "STORAGE_LANCE_VECTOR_DELETE_INDEX_FAILED_INVALID_ARGS",
|
|
3051
|
+
domain: ErrorDomain.STORAGE,
|
|
3052
|
+
category: ErrorCategory.USER,
|
|
3053
|
+
details: { indexName }
|
|
3054
|
+
},
|
|
3055
|
+
err
|
|
3056
|
+
);
|
|
3057
|
+
}
|
|
3058
|
+
try {
|
|
3059
|
+
const tables = await this.lanceClient.tableNames();
|
|
3060
|
+
for (const tableName of tables) {
|
|
3061
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
3062
|
+
const tableIndices = await table.listIndices();
|
|
3063
|
+
const foundIndex = tableIndices.find((index) => index.name === indexName);
|
|
3064
|
+
if (foundIndex) {
|
|
3065
|
+
await table.dropIndex(indexName);
|
|
3066
|
+
return;
|
|
3067
|
+
}
|
|
3068
|
+
}
|
|
3069
|
+
throw new Error(`Index ${indexName} not found`);
|
|
3070
|
+
} catch (error) {
|
|
3071
|
+
throw new MastraError(
|
|
3072
|
+
{
|
|
3073
|
+
id: "STORAGE_LANCE_VECTOR_DELETE_INDEX_FAILED",
|
|
3074
|
+
domain: ErrorDomain.STORAGE,
|
|
3075
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
3076
|
+
details: { indexName }
|
|
3077
|
+
},
|
|
3078
|
+
error
|
|
3079
|
+
);
|
|
3080
|
+
}
|
|
3081
|
+
}
|
|
3082
|
+
/**
|
|
3083
|
+
* Deletes all tables in the database
|
|
3084
|
+
*/
|
|
3085
|
+
async deleteAllTables() {
|
|
3086
|
+
if (!this.lanceClient) {
|
|
3087
|
+
throw new MastraError({
|
|
3088
|
+
id: "STORAGE_LANCE_VECTOR_DELETE_ALL_TABLES_FAILED_INVALID_ARGS",
|
|
3089
|
+
domain: ErrorDomain.STORAGE,
|
|
3090
|
+
category: ErrorCategory.USER,
|
|
3091
|
+
details: { methodName: "deleteAllTables" },
|
|
3092
|
+
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance"
|
|
3093
|
+
});
|
|
3094
|
+
}
|
|
3095
|
+
try {
|
|
3096
|
+
await this.lanceClient.dropAllTables();
|
|
3097
|
+
} catch (error) {
|
|
3098
|
+
throw new MastraError(
|
|
3099
|
+
{
|
|
3100
|
+
id: "STORAGE_LANCE_VECTOR_DELETE_ALL_TABLES_FAILED",
|
|
3101
|
+
domain: ErrorDomain.STORAGE,
|
|
3102
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
3103
|
+
details: { methodName: "deleteAllTables" }
|
|
3104
|
+
},
|
|
3105
|
+
error
|
|
3106
|
+
);
|
|
3107
|
+
}
|
|
3108
|
+
}
|
|
3109
|
+
async deleteTable(tableName) {
|
|
3110
|
+
if (!this.lanceClient) {
|
|
3111
|
+
throw new MastraError({
|
|
3112
|
+
id: "STORAGE_LANCE_VECTOR_DELETE_TABLE_FAILED_INVALID_ARGS",
|
|
3113
|
+
domain: ErrorDomain.STORAGE,
|
|
3114
|
+
category: ErrorCategory.USER,
|
|
3115
|
+
details: { tableName },
|
|
3116
|
+
text: "LanceDB client not initialized. Use LanceVectorStore.create() to create an instance"
|
|
3117
|
+
});
|
|
3118
|
+
}
|
|
3119
|
+
try {
|
|
3120
|
+
await this.lanceClient.dropTable(tableName);
|
|
3121
|
+
} catch (error) {
|
|
3122
|
+
throw new MastraError(
|
|
3123
|
+
{
|
|
3124
|
+
id: "STORAGE_LANCE_VECTOR_DELETE_TABLE_FAILED",
|
|
3125
|
+
domain: ErrorDomain.STORAGE,
|
|
3126
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
3127
|
+
details: { tableName }
|
|
3128
|
+
},
|
|
3129
|
+
error
|
|
3130
|
+
);
|
|
3131
|
+
}
|
|
3132
|
+
}
|
|
3133
|
+
async updateVector({ indexName, id, update }) {
|
|
3134
|
+
try {
|
|
3135
|
+
if (!this.lanceClient) {
|
|
3136
|
+
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
3137
|
+
}
|
|
3138
|
+
if (!indexName) {
|
|
3139
|
+
throw new Error("indexName is required");
|
|
3140
|
+
}
|
|
3141
|
+
if (!id) {
|
|
3142
|
+
throw new Error("id is required");
|
|
3143
|
+
}
|
|
3144
|
+
} catch (err) {
|
|
3145
|
+
throw new MastraError(
|
|
3146
|
+
{
|
|
3147
|
+
id: "STORAGE_LANCE_VECTOR_UPDATE_VECTOR_FAILED_INVALID_ARGS",
|
|
3148
|
+
domain: ErrorDomain.STORAGE,
|
|
3149
|
+
category: ErrorCategory.USER,
|
|
3150
|
+
details: { indexName, id }
|
|
3151
|
+
},
|
|
3152
|
+
err
|
|
3153
|
+
);
|
|
3154
|
+
}
|
|
3155
|
+
try {
|
|
3156
|
+
const tables = await this.lanceClient.tableNames();
|
|
3157
|
+
for (const tableName of tables) {
|
|
3158
|
+
this.logger.debug("Checking table:" + tableName);
|
|
3159
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
3160
|
+
try {
|
|
3161
|
+
const schema = await table.schema();
|
|
3162
|
+
const hasColumn = schema.fields.some((field) => field.name === indexName);
|
|
3163
|
+
if (hasColumn) {
|
|
3164
|
+
this.logger.debug(`Found column ${indexName} in table ${tableName}`);
|
|
3165
|
+
const existingRecord = await table.query().where(`id = '${id}'`).select(schema.fields.map((field) => field.name)).limit(1).toArray();
|
|
3166
|
+
if (existingRecord.length === 0) {
|
|
3167
|
+
throw new Error(`Record with id '${id}' not found in table ${tableName}`);
|
|
3168
|
+
}
|
|
3169
|
+
const rowData = {
|
|
3170
|
+
id
|
|
3171
|
+
};
|
|
3172
|
+
Object.entries(existingRecord[0]).forEach(([key, value]) => {
|
|
3173
|
+
if (key !== "id" && key !== "_distance") {
|
|
3174
|
+
if (key === indexName) {
|
|
3175
|
+
if (!update.vector) {
|
|
3176
|
+
if (Array.isArray(value)) {
|
|
3177
|
+
rowData[key] = [...value];
|
|
3178
|
+
} else if (typeof value === "object" && value !== null) {
|
|
3179
|
+
rowData[key] = Array.from(value);
|
|
3180
|
+
} else {
|
|
3181
|
+
rowData[key] = value;
|
|
3182
|
+
}
|
|
3183
|
+
}
|
|
3184
|
+
} else {
|
|
3185
|
+
rowData[key] = value;
|
|
3186
|
+
}
|
|
3187
|
+
}
|
|
3188
|
+
});
|
|
3189
|
+
if (update.vector) {
|
|
3190
|
+
rowData[indexName] = update.vector;
|
|
3191
|
+
}
|
|
3192
|
+
if (update.metadata) {
|
|
3193
|
+
Object.entries(update.metadata).forEach(([key, value]) => {
|
|
3194
|
+
rowData[`metadata_${key}`] = value;
|
|
3195
|
+
});
|
|
3196
|
+
}
|
|
3197
|
+
await table.add([rowData], { mode: "overwrite" });
|
|
3198
|
+
return;
|
|
3199
|
+
}
|
|
3200
|
+
} catch (err) {
|
|
3201
|
+
this.logger.error(`Error checking schema for table ${tableName}:` + err);
|
|
3202
|
+
continue;
|
|
3203
|
+
}
|
|
3204
|
+
}
|
|
3205
|
+
throw new Error(`No table found with column/index '${indexName}'`);
|
|
3206
|
+
} catch (error) {
|
|
3207
|
+
throw new MastraError(
|
|
3208
|
+
{
|
|
3209
|
+
id: "STORAGE_LANCE_VECTOR_UPDATE_VECTOR_FAILED",
|
|
3210
|
+
domain: ErrorDomain.STORAGE,
|
|
3211
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
3212
|
+
details: { indexName, id, hasVector: !!update.vector, hasMetadata: !!update.metadata }
|
|
3213
|
+
},
|
|
3214
|
+
error
|
|
3215
|
+
);
|
|
3216
|
+
}
|
|
3217
|
+
}
|
|
3218
|
+
async deleteVector({ indexName, id }) {
|
|
3219
|
+
try {
|
|
3220
|
+
if (!this.lanceClient) {
|
|
3221
|
+
throw new Error("LanceDB client not initialized. Use LanceVectorStore.create() to create an instance");
|
|
3222
|
+
}
|
|
3223
|
+
if (!indexName) {
|
|
3224
|
+
throw new Error("indexName is required");
|
|
3225
|
+
}
|
|
3226
|
+
if (!id) {
|
|
3227
|
+
throw new Error("id is required");
|
|
3228
|
+
}
|
|
3229
|
+
} catch (err) {
|
|
3230
|
+
throw new MastraError(
|
|
3231
|
+
{
|
|
3232
|
+
id: "STORAGE_LANCE_VECTOR_DELETE_VECTOR_FAILED_INVALID_ARGS",
|
|
3233
|
+
domain: ErrorDomain.STORAGE,
|
|
3234
|
+
category: ErrorCategory.USER,
|
|
3235
|
+
details: { indexName, id }
|
|
3236
|
+
},
|
|
3237
|
+
err
|
|
3238
|
+
);
|
|
3239
|
+
}
|
|
3240
|
+
try {
|
|
3241
|
+
const tables = await this.lanceClient.tableNames();
|
|
3242
|
+
for (const tableName of tables) {
|
|
3243
|
+
this.logger.debug("Checking table:" + tableName);
|
|
3244
|
+
const table = await this.lanceClient.openTable(tableName);
|
|
3245
|
+
try {
|
|
3246
|
+
const schema = await table.schema();
|
|
3247
|
+
const hasColumn = schema.fields.some((field) => field.name === indexName);
|
|
3248
|
+
if (hasColumn) {
|
|
3249
|
+
this.logger.debug(`Found column ${indexName} in table ${tableName}`);
|
|
3250
|
+
await table.delete(`id = '${id}'`);
|
|
3251
|
+
return;
|
|
3252
|
+
}
|
|
3253
|
+
} catch (err) {
|
|
3254
|
+
this.logger.error(`Error checking schema for table ${tableName}:` + err);
|
|
3255
|
+
continue;
|
|
3256
|
+
}
|
|
3257
|
+
}
|
|
3258
|
+
throw new Error(`No table found with column/index '${indexName}'`);
|
|
3259
|
+
} catch (error) {
|
|
3260
|
+
throw new MastraError(
|
|
3261
|
+
{
|
|
3262
|
+
id: "STORAGE_LANCE_VECTOR_DELETE_VECTOR_FAILED",
|
|
3263
|
+
domain: ErrorDomain.STORAGE,
|
|
3264
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
3265
|
+
details: { indexName, id }
|
|
3266
|
+
},
|
|
3267
|
+
error
|
|
3268
|
+
);
|
|
3269
|
+
}
|
|
3270
|
+
}
|
|
3271
|
+
/**
|
|
3272
|
+
* Converts a flattened object with keys using underscore notation back to a nested object.
|
|
3273
|
+
* Example: { name: 'test', details_text: 'test' } → { name: 'test', details: { text: 'test' } }
|
|
3274
|
+
*/
|
|
3275
|
+
unflattenObject(obj) {
|
|
3276
|
+
const result = {};
|
|
3277
|
+
Object.keys(obj).forEach((key) => {
|
|
3278
|
+
const value = obj[key];
|
|
3279
|
+
const parts = key.split("_");
|
|
3280
|
+
let current = result;
|
|
3281
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
3282
|
+
const part = parts[i];
|
|
3283
|
+
if (!part) continue;
|
|
3284
|
+
if (!current[part] || typeof current[part] !== "object") {
|
|
3285
|
+
current[part] = {};
|
|
3286
|
+
}
|
|
3287
|
+
current = current[part];
|
|
3288
|
+
}
|
|
3289
|
+
const lastPart = parts[parts.length - 1];
|
|
3290
|
+
if (lastPart) {
|
|
3291
|
+
current[lastPart] = value;
|
|
3292
|
+
}
|
|
3293
|
+
});
|
|
3294
|
+
return result;
|
|
3295
|
+
}
|
|
3296
|
+
};
|
|
3297
|
+
|
|
3298
|
+
export { LanceStorage, LanceVectorStore };
|
|
3299
|
+
//# sourceMappingURL=index.js.map
|
|
3300
|
+
//# sourceMappingURL=index.js.map
|