@mastra/clickhouse 0.0.0-trigger-playground-ui-package-20250506151043 → 0.0.0-tsconfig-compile-20250703214351
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +426 -2
- package/dist/_tsup-dts-rollup.d.cts +61 -8
- package/dist/_tsup-dts-rollup.d.ts +61 -8
- package/dist/index.cjs +466 -114
- package/dist/index.js +449 -97
- package/package.json +14 -10
- package/src/storage/index.test.ts +417 -119
- package/src/storage/index.ts +520 -114
package/dist/index.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { createClient } from '@clickhouse/client';
|
|
2
|
+
import { MessageList } from '@mastra/core/agent';
|
|
3
|
+
import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
|
|
2
4
|
import { TABLE_EVALS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, MastraStorage, TABLE_SCHEMAS } from '@mastra/core/storage';
|
|
3
5
|
|
|
4
6
|
// src/storage/index.ts
|
|
@@ -63,7 +65,12 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
63
65
|
const resultValue = JSON.parse(row.result);
|
|
64
66
|
const testInfoValue = row.test_info ? JSON.parse(row.test_info) : void 0;
|
|
65
67
|
if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
|
|
66
|
-
throw new
|
|
68
|
+
throw new MastraError({
|
|
69
|
+
id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
|
|
70
|
+
text: `Invalid MetricResult format: ${JSON.stringify(resultValue)}`,
|
|
71
|
+
domain: ErrorDomain.STORAGE,
|
|
72
|
+
category: ErrorCategory.USER
|
|
73
|
+
});
|
|
67
74
|
}
|
|
68
75
|
return {
|
|
69
76
|
input: row.input,
|
|
@@ -78,6 +85,18 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
78
85
|
createdAt: row.created_at
|
|
79
86
|
};
|
|
80
87
|
}
|
|
88
|
+
escape(value) {
|
|
89
|
+
if (typeof value === "string") {
|
|
90
|
+
return `'${value.replace(/'/g, "''")}'`;
|
|
91
|
+
}
|
|
92
|
+
if (value instanceof Date) {
|
|
93
|
+
return `'${value.toISOString()}'`;
|
|
94
|
+
}
|
|
95
|
+
if (value === null || value === void 0) {
|
|
96
|
+
return "NULL";
|
|
97
|
+
}
|
|
98
|
+
return value.toString();
|
|
99
|
+
}
|
|
81
100
|
async getEvalsByAgentName(agentName, type) {
|
|
82
101
|
try {
|
|
83
102
|
const baseQuery = `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
@@ -98,11 +117,18 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
98
117
|
const rows = await result.json();
|
|
99
118
|
return rows.data.map((row) => this.transformEvalRow(row));
|
|
100
119
|
} catch (error) {
|
|
101
|
-
if (error
|
|
120
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
102
121
|
return [];
|
|
103
122
|
}
|
|
104
|
-
|
|
105
|
-
|
|
123
|
+
throw new MastraError(
|
|
124
|
+
{
|
|
125
|
+
id: "CLICKHOUSE_STORAGE_GET_EVALS_BY_AGENT_FAILED",
|
|
126
|
+
domain: ErrorDomain.STORAGE,
|
|
127
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
128
|
+
details: { agentName, type: type ?? null }
|
|
129
|
+
},
|
|
130
|
+
error
|
|
131
|
+
);
|
|
106
132
|
}
|
|
107
133
|
}
|
|
108
134
|
async batchInsert({ tableName, records }) {
|
|
@@ -126,8 +152,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
126
152
|
}
|
|
127
153
|
});
|
|
128
154
|
} catch (error) {
|
|
129
|
-
|
|
130
|
-
|
|
155
|
+
throw new MastraError(
|
|
156
|
+
{
|
|
157
|
+
id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
|
|
158
|
+
domain: ErrorDomain.STORAGE,
|
|
159
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
160
|
+
details: { tableName }
|
|
161
|
+
},
|
|
162
|
+
error
|
|
163
|
+
);
|
|
131
164
|
}
|
|
132
165
|
}
|
|
133
166
|
async getTraces({
|
|
@@ -175,48 +208,96 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
175
208
|
args.var_to_date = toDate.getTime() / 1e3;
|
|
176
209
|
}
|
|
177
210
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
211
|
+
try {
|
|
212
|
+
const result = await this.db.query({
|
|
213
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
214
|
+
query_params: args,
|
|
215
|
+
clickhouse_settings: {
|
|
216
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
217
|
+
date_time_input_format: "best_effort",
|
|
218
|
+
date_time_output_format: "iso",
|
|
219
|
+
use_client_time_zone: 1,
|
|
220
|
+
output_format_json_quote_64bit_integers: 0
|
|
221
|
+
}
|
|
222
|
+
});
|
|
223
|
+
if (!result) {
|
|
224
|
+
return [];
|
|
187
225
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
return
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
226
|
+
const resp = await result.json();
|
|
227
|
+
const rows = resp.data;
|
|
228
|
+
return rows.map((row) => ({
|
|
229
|
+
id: row.id,
|
|
230
|
+
parentSpanId: row.parentSpanId,
|
|
231
|
+
traceId: row.traceId,
|
|
232
|
+
name: row.name,
|
|
233
|
+
scope: row.scope,
|
|
234
|
+
kind: row.kind,
|
|
235
|
+
status: safelyParseJSON(row.status),
|
|
236
|
+
events: safelyParseJSON(row.events),
|
|
237
|
+
links: safelyParseJSON(row.links),
|
|
238
|
+
attributes: safelyParseJSON(row.attributes),
|
|
239
|
+
startTime: row.startTime,
|
|
240
|
+
endTime: row.endTime,
|
|
241
|
+
other: safelyParseJSON(row.other),
|
|
242
|
+
createdAt: row.createdAt
|
|
243
|
+
}));
|
|
244
|
+
} catch (error) {
|
|
245
|
+
if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
|
|
246
|
+
return [];
|
|
247
|
+
}
|
|
248
|
+
throw new MastraError(
|
|
249
|
+
{
|
|
250
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
|
|
251
|
+
domain: ErrorDomain.STORAGE,
|
|
252
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
253
|
+
details: {
|
|
254
|
+
name: name ?? null,
|
|
255
|
+
scope: scope ?? null,
|
|
256
|
+
page,
|
|
257
|
+
perPage,
|
|
258
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
259
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
260
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
261
|
+
toDate: toDate?.toISOString() ?? null
|
|
262
|
+
}
|
|
263
|
+
},
|
|
264
|
+
error
|
|
265
|
+
);
|
|
266
|
+
}
|
|
210
267
|
}
|
|
211
268
|
async optimizeTable({ tableName }) {
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
269
|
+
try {
|
|
270
|
+
await this.db.command({
|
|
271
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`
|
|
272
|
+
});
|
|
273
|
+
} catch (error) {
|
|
274
|
+
throw new MastraError(
|
|
275
|
+
{
|
|
276
|
+
id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
|
|
277
|
+
domain: ErrorDomain.STORAGE,
|
|
278
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
279
|
+
details: { tableName }
|
|
280
|
+
},
|
|
281
|
+
error
|
|
282
|
+
);
|
|
283
|
+
}
|
|
215
284
|
}
|
|
216
285
|
async materializeTtl({ tableName }) {
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
286
|
+
try {
|
|
287
|
+
await this.db.command({
|
|
288
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
|
|
289
|
+
});
|
|
290
|
+
} catch (error) {
|
|
291
|
+
throw new MastraError(
|
|
292
|
+
{
|
|
293
|
+
id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
|
|
294
|
+
domain: ErrorDomain.STORAGE,
|
|
295
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
296
|
+
details: { tableName }
|
|
297
|
+
},
|
|
298
|
+
error
|
|
299
|
+
);
|
|
300
|
+
}
|
|
220
301
|
}
|
|
221
302
|
async createTable({
|
|
222
303
|
tableName,
|
|
@@ -235,7 +316,6 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
235
316
|
${["id String"].concat(columns)}
|
|
236
317
|
)
|
|
237
318
|
ENGINE = ${TABLE_ENGINES[tableName]}
|
|
238
|
-
PARTITION BY "createdAt"
|
|
239
319
|
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
240
320
|
ORDER BY (createdAt, run_id, workflow_name)
|
|
241
321
|
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
|
|
@@ -245,7 +325,6 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
245
325
|
${columns}
|
|
246
326
|
)
|
|
247
327
|
ENGINE = ${TABLE_ENGINES[tableName]}
|
|
248
|
-
PARTITION BY "createdAt"
|
|
249
328
|
PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
250
329
|
ORDER BY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
|
|
251
330
|
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
|
|
@@ -262,8 +341,75 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
262
341
|
}
|
|
263
342
|
});
|
|
264
343
|
} catch (error) {
|
|
265
|
-
|
|
266
|
-
|
|
344
|
+
throw new MastraError(
|
|
345
|
+
{
|
|
346
|
+
id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
|
|
347
|
+
domain: ErrorDomain.STORAGE,
|
|
348
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
349
|
+
details: { tableName }
|
|
350
|
+
},
|
|
351
|
+
error
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
getSqlType(type) {
|
|
356
|
+
switch (type) {
|
|
357
|
+
case "text":
|
|
358
|
+
return "String";
|
|
359
|
+
case "timestamp":
|
|
360
|
+
return "DateTime64(3)";
|
|
361
|
+
case "integer":
|
|
362
|
+
case "bigint":
|
|
363
|
+
return "Int64";
|
|
364
|
+
case "jsonb":
|
|
365
|
+
return "String";
|
|
366
|
+
default:
|
|
367
|
+
return super.getSqlType(type);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* Alters table schema to add columns if they don't exist
|
|
372
|
+
* @param tableName Name of the table
|
|
373
|
+
* @param schema Schema of the table
|
|
374
|
+
* @param ifNotExists Array of column names to add if they don't exist
|
|
375
|
+
*/
|
|
376
|
+
async alterTable({
|
|
377
|
+
tableName,
|
|
378
|
+
schema,
|
|
379
|
+
ifNotExists
|
|
380
|
+
}) {
|
|
381
|
+
try {
|
|
382
|
+
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
383
|
+
const result = await this.db.query({
|
|
384
|
+
query: describeSql
|
|
385
|
+
});
|
|
386
|
+
const rows = await result.json();
|
|
387
|
+
const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
|
|
388
|
+
for (const columnName of ifNotExists) {
|
|
389
|
+
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
390
|
+
const columnDef = schema[columnName];
|
|
391
|
+
let sqlType = this.getSqlType(columnDef.type);
|
|
392
|
+
if (columnDef.nullable !== false) {
|
|
393
|
+
sqlType = `Nullable(${sqlType})`;
|
|
394
|
+
}
|
|
395
|
+
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
|
|
396
|
+
const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
397
|
+
await this.db.query({
|
|
398
|
+
query: alterSql
|
|
399
|
+
});
|
|
400
|
+
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
} catch (error) {
|
|
404
|
+
throw new MastraError(
|
|
405
|
+
{
|
|
406
|
+
id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
|
|
407
|
+
domain: ErrorDomain.STORAGE,
|
|
408
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
409
|
+
details: { tableName }
|
|
410
|
+
},
|
|
411
|
+
error
|
|
412
|
+
);
|
|
267
413
|
}
|
|
268
414
|
}
|
|
269
415
|
async clearTable({ tableName }) {
|
|
@@ -279,8 +425,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
279
425
|
}
|
|
280
426
|
});
|
|
281
427
|
} catch (error) {
|
|
282
|
-
|
|
283
|
-
|
|
428
|
+
throw new MastraError(
|
|
429
|
+
{
|
|
430
|
+
id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
|
|
431
|
+
domain: ErrorDomain.STORAGE,
|
|
432
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
433
|
+
details: { tableName }
|
|
434
|
+
},
|
|
435
|
+
error
|
|
436
|
+
);
|
|
284
437
|
}
|
|
285
438
|
}
|
|
286
439
|
async insert({ tableName, record }) {
|
|
@@ -303,11 +456,21 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
303
456
|
}
|
|
304
457
|
});
|
|
305
458
|
} catch (error) {
|
|
306
|
-
|
|
307
|
-
|
|
459
|
+
throw new MastraError(
|
|
460
|
+
{
|
|
461
|
+
id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
|
|
462
|
+
domain: ErrorDomain.STORAGE,
|
|
463
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
464
|
+
details: { tableName }
|
|
465
|
+
},
|
|
466
|
+
error
|
|
467
|
+
);
|
|
308
468
|
}
|
|
309
469
|
}
|
|
310
|
-
async load({
|
|
470
|
+
async load({
|
|
471
|
+
tableName,
|
|
472
|
+
keys
|
|
473
|
+
}) {
|
|
311
474
|
try {
|
|
312
475
|
const keyEntries = Object.entries(keys);
|
|
313
476
|
const conditions = keyEntries.map(
|
|
@@ -344,8 +507,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
344
507
|
const data = transformRow(rows.data[0]);
|
|
345
508
|
return data;
|
|
346
509
|
} catch (error) {
|
|
347
|
-
|
|
348
|
-
|
|
510
|
+
throw new MastraError(
|
|
511
|
+
{
|
|
512
|
+
id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
|
|
513
|
+
domain: ErrorDomain.STORAGE,
|
|
514
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
515
|
+
details: { tableName }
|
|
516
|
+
},
|
|
517
|
+
error
|
|
518
|
+
);
|
|
349
519
|
}
|
|
350
520
|
}
|
|
351
521
|
async getThreadById({ threadId }) {
|
|
@@ -382,8 +552,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
382
552
|
updatedAt: thread.updatedAt
|
|
383
553
|
};
|
|
384
554
|
} catch (error) {
|
|
385
|
-
|
|
386
|
-
|
|
555
|
+
throw new MastraError(
|
|
556
|
+
{
|
|
557
|
+
id: "CLICKHOUSE_STORAGE_GET_THREAD_BY_ID_FAILED",
|
|
558
|
+
domain: ErrorDomain.STORAGE,
|
|
559
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
560
|
+
details: { threadId }
|
|
561
|
+
},
|
|
562
|
+
error
|
|
563
|
+
);
|
|
387
564
|
}
|
|
388
565
|
}
|
|
389
566
|
async getThreadsByResourceId({ resourceId }) {
|
|
@@ -416,8 +593,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
416
593
|
updatedAt: thread.updatedAt
|
|
417
594
|
}));
|
|
418
595
|
} catch (error) {
|
|
419
|
-
|
|
420
|
-
|
|
596
|
+
throw new MastraError(
|
|
597
|
+
{
|
|
598
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_FAILED",
|
|
599
|
+
domain: ErrorDomain.STORAGE,
|
|
600
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
601
|
+
details: { resourceId }
|
|
602
|
+
},
|
|
603
|
+
error
|
|
604
|
+
);
|
|
421
605
|
}
|
|
422
606
|
}
|
|
423
607
|
async saveThread({ thread }) {
|
|
@@ -441,8 +625,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
441
625
|
});
|
|
442
626
|
return thread;
|
|
443
627
|
} catch (error) {
|
|
444
|
-
|
|
445
|
-
|
|
628
|
+
throw new MastraError(
|
|
629
|
+
{
|
|
630
|
+
id: "CLICKHOUSE_STORAGE_SAVE_THREAD_FAILED",
|
|
631
|
+
domain: ErrorDomain.STORAGE,
|
|
632
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
633
|
+
details: { threadId: thread.id }
|
|
634
|
+
},
|
|
635
|
+
error
|
|
636
|
+
);
|
|
446
637
|
}
|
|
447
638
|
}
|
|
448
639
|
async updateThread({
|
|
@@ -467,15 +658,18 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
467
658
|
};
|
|
468
659
|
await this.db.insert({
|
|
469
660
|
table: TABLE_THREADS,
|
|
661
|
+
format: "JSONEachRow",
|
|
470
662
|
values: [
|
|
471
663
|
{
|
|
472
|
-
|
|
664
|
+
id: updatedThread.id,
|
|
665
|
+
resourceId: updatedThread.resourceId,
|
|
666
|
+
title: updatedThread.title,
|
|
667
|
+
metadata: updatedThread.metadata,
|
|
668
|
+
createdAt: updatedThread.createdAt,
|
|
473
669
|
updatedAt: updatedThread.updatedAt.toISOString()
|
|
474
670
|
}
|
|
475
671
|
],
|
|
476
|
-
format: "JSONEachRow",
|
|
477
672
|
clickhouse_settings: {
|
|
478
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
479
673
|
date_time_input_format: "best_effort",
|
|
480
674
|
use_client_time_zone: 1,
|
|
481
675
|
output_format_json_quote_64bit_integers: 0
|
|
@@ -483,14 +677,21 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
483
677
|
});
|
|
484
678
|
return updatedThread;
|
|
485
679
|
} catch (error) {
|
|
486
|
-
|
|
487
|
-
|
|
680
|
+
throw new MastraError(
|
|
681
|
+
{
|
|
682
|
+
id: "CLICKHOUSE_STORAGE_UPDATE_THREAD_FAILED",
|
|
683
|
+
domain: ErrorDomain.STORAGE,
|
|
684
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
685
|
+
details: { threadId: id, title }
|
|
686
|
+
},
|
|
687
|
+
error
|
|
688
|
+
);
|
|
488
689
|
}
|
|
489
690
|
}
|
|
490
691
|
async deleteThread({ threadId }) {
|
|
491
692
|
try {
|
|
492
693
|
await this.db.command({
|
|
493
|
-
query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id =
|
|
694
|
+
query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
494
695
|
query_params: { var_thread_id: threadId },
|
|
495
696
|
clickhouse_settings: {
|
|
496
697
|
output_format_json_quote_64bit_integers: 0
|
|
@@ -504,14 +705,26 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
504
705
|
}
|
|
505
706
|
});
|
|
506
707
|
} catch (error) {
|
|
507
|
-
|
|
508
|
-
|
|
708
|
+
throw new MastraError(
|
|
709
|
+
{
|
|
710
|
+
id: "CLICKHOUSE_STORAGE_DELETE_THREAD_FAILED",
|
|
711
|
+
domain: ErrorDomain.STORAGE,
|
|
712
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
713
|
+
details: { threadId }
|
|
714
|
+
},
|
|
715
|
+
error
|
|
716
|
+
);
|
|
509
717
|
}
|
|
510
718
|
}
|
|
511
|
-
async getMessages({
|
|
719
|
+
async getMessages({
|
|
720
|
+
threadId,
|
|
721
|
+
resourceId,
|
|
722
|
+
selectBy,
|
|
723
|
+
format
|
|
724
|
+
}) {
|
|
512
725
|
try {
|
|
513
726
|
const messages = [];
|
|
514
|
-
const limit =
|
|
727
|
+
const limit = this.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
515
728
|
const include = selectBy?.include || [];
|
|
516
729
|
if (include.length) {
|
|
517
730
|
const includeResult = await this.db.query({
|
|
@@ -604,16 +817,27 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
604
817
|
}
|
|
605
818
|
}
|
|
606
819
|
});
|
|
607
|
-
|
|
820
|
+
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
821
|
+
if (format === `v2`) return list.get.all.v2();
|
|
822
|
+
return list.get.all.v1();
|
|
608
823
|
} catch (error) {
|
|
609
|
-
|
|
610
|
-
|
|
824
|
+
throw new MastraError(
|
|
825
|
+
{
|
|
826
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
|
|
827
|
+
domain: ErrorDomain.STORAGE,
|
|
828
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
829
|
+
details: { threadId, resourceId: resourceId ?? "" }
|
|
830
|
+
},
|
|
831
|
+
error
|
|
832
|
+
);
|
|
611
833
|
}
|
|
612
834
|
}
|
|
613
|
-
async saveMessages(
|
|
835
|
+
async saveMessages(args) {
|
|
836
|
+
const { messages, format = "v1" } = args;
|
|
614
837
|
if (messages.length === 0) return messages;
|
|
615
838
|
try {
|
|
616
839
|
const threadId = messages[0]?.threadId;
|
|
840
|
+
const resourceId = messages[0]?.resourceId;
|
|
617
841
|
if (!threadId) {
|
|
618
842
|
throw new Error("Thread ID is required");
|
|
619
843
|
}
|
|
@@ -621,28 +845,100 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
621
845
|
if (!thread) {
|
|
622
846
|
throw new Error(`Thread ${threadId} not found`);
|
|
623
847
|
}
|
|
624
|
-
await this.db.
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
thread_id: threadId,
|
|
630
|
-
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
631
|
-
createdAt: message.createdAt.toISOString(),
|
|
632
|
-
role: message.role,
|
|
633
|
-
type: message.type
|
|
634
|
-
})),
|
|
848
|
+
const existingResult = await this.db.query({
|
|
849
|
+
query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
850
|
+
query_params: {
|
|
851
|
+
ids: messages.map((m) => m.id)
|
|
852
|
+
},
|
|
635
853
|
clickhouse_settings: {
|
|
636
854
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
637
855
|
date_time_input_format: "best_effort",
|
|
856
|
+
date_time_output_format: "iso",
|
|
638
857
|
use_client_time_zone: 1,
|
|
639
858
|
output_format_json_quote_64bit_integers: 0
|
|
640
|
-
}
|
|
859
|
+
},
|
|
860
|
+
format: "JSONEachRow"
|
|
641
861
|
});
|
|
642
|
-
|
|
862
|
+
const existingRows = await existingResult.json();
|
|
863
|
+
const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
|
|
864
|
+
const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${threadId}`));
|
|
865
|
+
const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${threadId}`));
|
|
866
|
+
const updatePromises = toUpdate.map(
|
|
867
|
+
(message) => this.db.command({
|
|
868
|
+
query: `
|
|
869
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
870
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}
|
|
871
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
872
|
+
`,
|
|
873
|
+
query_params: {
|
|
874
|
+
var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
875
|
+
var_role: message.role,
|
|
876
|
+
var_type: message.type || "v2",
|
|
877
|
+
var_id: message.id,
|
|
878
|
+
var_thread_id: threadId
|
|
879
|
+
},
|
|
880
|
+
clickhouse_settings: {
|
|
881
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
882
|
+
date_time_input_format: "best_effort",
|
|
883
|
+
use_client_time_zone: 1,
|
|
884
|
+
output_format_json_quote_64bit_integers: 0
|
|
885
|
+
}
|
|
886
|
+
})
|
|
887
|
+
);
|
|
888
|
+
await Promise.all([
|
|
889
|
+
// Insert messages
|
|
890
|
+
this.db.insert({
|
|
891
|
+
table: TABLE_MESSAGES,
|
|
892
|
+
format: "JSONEachRow",
|
|
893
|
+
values: toInsert.map((message) => ({
|
|
894
|
+
id: message.id,
|
|
895
|
+
thread_id: threadId,
|
|
896
|
+
content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
|
|
897
|
+
createdAt: message.createdAt.toISOString(),
|
|
898
|
+
role: message.role,
|
|
899
|
+
type: message.type || "v2"
|
|
900
|
+
})),
|
|
901
|
+
clickhouse_settings: {
|
|
902
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
903
|
+
date_time_input_format: "best_effort",
|
|
904
|
+
use_client_time_zone: 1,
|
|
905
|
+
output_format_json_quote_64bit_integers: 0
|
|
906
|
+
}
|
|
907
|
+
}),
|
|
908
|
+
...updatePromises,
|
|
909
|
+
// Update thread's updatedAt timestamp
|
|
910
|
+
this.db.insert({
|
|
911
|
+
table: TABLE_THREADS,
|
|
912
|
+
format: "JSONEachRow",
|
|
913
|
+
values: [
|
|
914
|
+
{
|
|
915
|
+
id: thread.id,
|
|
916
|
+
resourceId: thread.resourceId,
|
|
917
|
+
title: thread.title,
|
|
918
|
+
metadata: thread.metadata,
|
|
919
|
+
createdAt: thread.createdAt,
|
|
920
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
921
|
+
}
|
|
922
|
+
],
|
|
923
|
+
clickhouse_settings: {
|
|
924
|
+
date_time_input_format: "best_effort",
|
|
925
|
+
use_client_time_zone: 1,
|
|
926
|
+
output_format_json_quote_64bit_integers: 0
|
|
927
|
+
}
|
|
928
|
+
})
|
|
929
|
+
]);
|
|
930
|
+
const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
|
|
931
|
+
if (format === `v2`) return list.get.all.v2();
|
|
932
|
+
return list.get.all.v1();
|
|
643
933
|
} catch (error) {
|
|
644
|
-
|
|
645
|
-
|
|
934
|
+
throw new MastraError(
|
|
935
|
+
{
|
|
936
|
+
id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
|
|
937
|
+
domain: ErrorDomain.STORAGE,
|
|
938
|
+
category: ErrorCategory.THIRD_PARTY
|
|
939
|
+
},
|
|
940
|
+
error
|
|
941
|
+
);
|
|
646
942
|
}
|
|
647
943
|
}
|
|
648
944
|
async persistWorkflowSnapshot({
|
|
@@ -679,8 +975,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
679
975
|
}
|
|
680
976
|
});
|
|
681
977
|
} catch (error) {
|
|
682
|
-
|
|
683
|
-
|
|
978
|
+
throw new MastraError(
|
|
979
|
+
{
|
|
980
|
+
id: "CLICKHOUSE_STORAGE_PERSIST_WORKFLOW_SNAPSHOT_FAILED",
|
|
981
|
+
domain: ErrorDomain.STORAGE,
|
|
982
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
983
|
+
details: { workflowName, runId }
|
|
984
|
+
},
|
|
985
|
+
error
|
|
986
|
+
);
|
|
684
987
|
}
|
|
685
988
|
}
|
|
686
989
|
async loadWorkflowSnapshot({
|
|
@@ -700,8 +1003,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
700
1003
|
}
|
|
701
1004
|
return result.snapshot;
|
|
702
1005
|
} catch (error) {
|
|
703
|
-
|
|
704
|
-
|
|
1006
|
+
throw new MastraError(
|
|
1007
|
+
{
|
|
1008
|
+
id: "CLICKHOUSE_STORAGE_LOAD_WORKFLOW_SNAPSHOT_FAILED",
|
|
1009
|
+
domain: ErrorDomain.STORAGE,
|
|
1010
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1011
|
+
details: { workflowName, runId }
|
|
1012
|
+
},
|
|
1013
|
+
error
|
|
1014
|
+
);
|
|
705
1015
|
}
|
|
706
1016
|
}
|
|
707
1017
|
parseWorkflowRun(row) {
|
|
@@ -792,8 +1102,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
792
1102
|
});
|
|
793
1103
|
return { runs, total: total || runs.length };
|
|
794
1104
|
} catch (error) {
|
|
795
|
-
|
|
796
|
-
|
|
1105
|
+
throw new MastraError(
|
|
1106
|
+
{
|
|
1107
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUNS_FAILED",
|
|
1108
|
+
domain: ErrorDomain.STORAGE,
|
|
1109
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1110
|
+
details: { workflowName: workflowName ?? "", resourceId: resourceId ?? "" }
|
|
1111
|
+
},
|
|
1112
|
+
error
|
|
1113
|
+
);
|
|
797
1114
|
}
|
|
798
1115
|
}
|
|
799
1116
|
async getWorkflowRunById({
|
|
@@ -833,8 +1150,15 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
833
1150
|
}
|
|
834
1151
|
return this.parseWorkflowRun(resultJson[0]);
|
|
835
1152
|
} catch (error) {
|
|
836
|
-
|
|
837
|
-
|
|
1153
|
+
throw new MastraError(
|
|
1154
|
+
{
|
|
1155
|
+
id: "CLICKHOUSE_STORAGE_GET_WORKFLOW_RUN_BY_ID_FAILED",
|
|
1156
|
+
domain: ErrorDomain.STORAGE,
|
|
1157
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1158
|
+
details: { runId: runId ?? "", workflowName: workflowName ?? "" }
|
|
1159
|
+
},
|
|
1160
|
+
error
|
|
1161
|
+
);
|
|
838
1162
|
}
|
|
839
1163
|
}
|
|
840
1164
|
async hasColumn(table, column) {
|
|
@@ -845,9 +1169,37 @@ var ClickhouseStore = class extends MastraStorage {
|
|
|
845
1169
|
const columns = await result.json();
|
|
846
1170
|
return columns.some((c) => c.name === column);
|
|
847
1171
|
}
|
|
1172
|
+
async getTracesPaginated(_args) {
|
|
1173
|
+
throw new MastraError({
|
|
1174
|
+
id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
|
|
1175
|
+
domain: ErrorDomain.STORAGE,
|
|
1176
|
+
category: ErrorCategory.USER,
|
|
1177
|
+
text: "Method not implemented."
|
|
1178
|
+
});
|
|
1179
|
+
}
|
|
1180
|
+
async getThreadsByResourceIdPaginated(_args) {
|
|
1181
|
+
throw new MastraError({
|
|
1182
|
+
id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
|
|
1183
|
+
domain: ErrorDomain.STORAGE,
|
|
1184
|
+
category: ErrorCategory.USER,
|
|
1185
|
+
text: "Method not implemented."
|
|
1186
|
+
});
|
|
1187
|
+
}
|
|
1188
|
+
async getMessagesPaginated(_args) {
|
|
1189
|
+
throw new MastraError({
|
|
1190
|
+
id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
|
|
1191
|
+
domain: ErrorDomain.STORAGE,
|
|
1192
|
+
category: ErrorCategory.USER,
|
|
1193
|
+
text: "Method not implemented."
|
|
1194
|
+
});
|
|
1195
|
+
}
|
|
848
1196
|
async close() {
|
|
849
1197
|
await this.db.close();
|
|
850
1198
|
}
|
|
1199
|
+
async updateMessages(_args) {
|
|
1200
|
+
this.logger.error("updateMessages is not yet implemented in ClickhouseStore");
|
|
1201
|
+
throw new Error("Method not implemented");
|
|
1202
|
+
}
|
|
851
1203
|
};
|
|
852
1204
|
|
|
853
1205
|
export { COLUMN_TYPES, ClickhouseStore, TABLE_ENGINES };
|