@mastra/clickhouse 0.11.1-alpha.0 → 0.11.1-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +7 -7
- package/CHANGELOG.md +26 -0
- package/dist/_tsup-dts-rollup.d.cts +8 -4
- package/dist/_tsup-dts-rollup.d.ts +8 -4
- package/dist/index.cjs +348 -100
- package/dist/index.js +330 -82
- package/package.json +3 -3
- package/src/storage/index.test.ts +78 -0
- package/src/storage/index.ts +362 -99
package/src/storage/index.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { ClickHouseClient } from '@clickhouse/client';
|
|
|
2
2
|
import { createClient } from '@clickhouse/client';
|
|
3
3
|
import { MessageList } from '@mastra/core/agent';
|
|
4
4
|
import type { MastraMessageContentV2 } from '@mastra/core/agent';
|
|
5
|
+
import { MastraError, ErrorDomain, ErrorCategory } from '@mastra/core/error';
|
|
5
6
|
import type { MetricResult, TestInfo } from '@mastra/core/eval';
|
|
6
7
|
import type { MastraMessageV1, MastraMessageV2, StorageThreadType } from '@mastra/core/memory';
|
|
7
8
|
import {
|
|
@@ -22,10 +23,13 @@ import type {
|
|
|
22
23
|
WorkflowRun,
|
|
23
24
|
WorkflowRuns,
|
|
24
25
|
StorageGetTracesArg,
|
|
26
|
+
TABLE_RESOURCES,
|
|
25
27
|
} from '@mastra/core/storage';
|
|
26
28
|
import type { Trace } from '@mastra/core/telemetry';
|
|
27
29
|
import type { WorkflowRunState } from '@mastra/core/workflows';
|
|
28
30
|
|
|
31
|
+
type SUPPORTED_TABLE_NAMES = Exclude<TABLE_NAMES, typeof TABLE_RESOURCES>;
|
|
32
|
+
|
|
29
33
|
function safelyParseJSON(jsonString: string): any {
|
|
30
34
|
try {
|
|
31
35
|
return JSON.parse(jsonString);
|
|
@@ -65,7 +69,7 @@ export type ClickhouseConfig = {
|
|
|
65
69
|
};
|
|
66
70
|
};
|
|
67
71
|
|
|
68
|
-
export const TABLE_ENGINES: Record<
|
|
72
|
+
export const TABLE_ENGINES: Record<SUPPORTED_TABLE_NAMES, string> = {
|
|
69
73
|
[TABLE_MESSAGES]: `MergeTree()`,
|
|
70
74
|
[TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
71
75
|
[TABLE_TRACES]: `MergeTree()`,
|
|
@@ -126,7 +130,12 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
126
130
|
const testInfoValue = row.test_info ? JSON.parse(row.test_info as string) : undefined;
|
|
127
131
|
|
|
128
132
|
if (!resultValue || typeof resultValue !== 'object' || !('score' in resultValue)) {
|
|
129
|
-
throw new
|
|
133
|
+
throw new MastraError({
|
|
134
|
+
id: 'CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT',
|
|
135
|
+
text: `Invalid MetricResult format: ${JSON.stringify(resultValue)}`,
|
|
136
|
+
domain: ErrorDomain.STORAGE,
|
|
137
|
+
category: ErrorCategory.USER,
|
|
138
|
+
});
|
|
130
139
|
}
|
|
131
140
|
|
|
132
141
|
return {
|
|
@@ -143,6 +152,19 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
143
152
|
};
|
|
144
153
|
}
|
|
145
154
|
|
|
155
|
+
private escape(value: any): string {
|
|
156
|
+
if (typeof value === 'string') {
|
|
157
|
+
return `'${value.replace(/'/g, "''")}'`;
|
|
158
|
+
}
|
|
159
|
+
if (value instanceof Date) {
|
|
160
|
+
return `'${value.toISOString()}'`;
|
|
161
|
+
}
|
|
162
|
+
if (value === null || value === undefined) {
|
|
163
|
+
return 'NULL';
|
|
164
|
+
}
|
|
165
|
+
return value.toString();
|
|
166
|
+
}
|
|
167
|
+
|
|
146
168
|
async getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]> {
|
|
147
169
|
try {
|
|
148
170
|
const baseQuery = `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
@@ -170,13 +192,19 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
170
192
|
|
|
171
193
|
const rows = await result.json();
|
|
172
194
|
return rows.data.map((row: any) => this.transformEvalRow(row));
|
|
173
|
-
} catch (error) {
|
|
174
|
-
|
|
175
|
-
if (error instanceof Error && error.message.includes('no such table')) {
|
|
195
|
+
} catch (error: any) {
|
|
196
|
+
if (error?.message?.includes('no such table') || error?.message?.includes('does not exist')) {
|
|
176
197
|
return [];
|
|
177
198
|
}
|
|
178
|
-
|
|
179
|
-
|
|
199
|
+
throw new MastraError(
|
|
200
|
+
{
|
|
201
|
+
id: 'CLICKHOUSE_STORAGE_GET_EVALS_BY_AGENT_FAILED',
|
|
202
|
+
domain: ErrorDomain.STORAGE,
|
|
203
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
204
|
+
details: { agentName, type: type ?? null },
|
|
205
|
+
},
|
|
206
|
+
error,
|
|
207
|
+
);
|
|
180
208
|
}
|
|
181
209
|
}
|
|
182
210
|
|
|
@@ -202,9 +230,16 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
202
230
|
output_format_json_quote_64bit_integers: 0,
|
|
203
231
|
},
|
|
204
232
|
});
|
|
205
|
-
} catch (error) {
|
|
206
|
-
|
|
207
|
-
|
|
233
|
+
} catch (error: any) {
|
|
234
|
+
throw new MastraError(
|
|
235
|
+
{
|
|
236
|
+
id: 'CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED',
|
|
237
|
+
domain: ErrorDomain.STORAGE,
|
|
238
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
239
|
+
details: { tableName },
|
|
240
|
+
},
|
|
241
|
+
error,
|
|
242
|
+
);
|
|
208
243
|
}
|
|
209
244
|
}
|
|
210
245
|
|
|
@@ -269,59 +304,107 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
269
304
|
|
|
270
305
|
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
271
306
|
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
307
|
+
try {
|
|
308
|
+
const result = await this.db.query({
|
|
309
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
310
|
+
query_params: args,
|
|
311
|
+
clickhouse_settings: {
|
|
312
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
313
|
+
date_time_input_format: 'best_effort',
|
|
314
|
+
date_time_output_format: 'iso',
|
|
315
|
+
use_client_time_zone: 1,
|
|
316
|
+
output_format_json_quote_64bit_integers: 0,
|
|
317
|
+
},
|
|
318
|
+
});
|
|
283
319
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
320
|
+
if (!result) {
|
|
321
|
+
return [];
|
|
322
|
+
}
|
|
287
323
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
324
|
+
const resp = await result.json();
|
|
325
|
+
const rows: any[] = resp.data;
|
|
326
|
+
return rows.map(row => ({
|
|
327
|
+
id: row.id,
|
|
328
|
+
parentSpanId: row.parentSpanId,
|
|
329
|
+
traceId: row.traceId,
|
|
330
|
+
name: row.name,
|
|
331
|
+
scope: row.scope,
|
|
332
|
+
kind: row.kind,
|
|
333
|
+
status: safelyParseJSON(row.status as string),
|
|
334
|
+
events: safelyParseJSON(row.events as string),
|
|
335
|
+
links: safelyParseJSON(row.links as string),
|
|
336
|
+
attributes: safelyParseJSON(row.attributes as string),
|
|
337
|
+
startTime: row.startTime,
|
|
338
|
+
endTime: row.endTime,
|
|
339
|
+
other: safelyParseJSON(row.other as string),
|
|
340
|
+
createdAt: row.createdAt,
|
|
341
|
+
}));
|
|
342
|
+
} catch (error: any) {
|
|
343
|
+
if (error?.message?.includes('no such table') || error?.message?.includes('does not exist')) {
|
|
344
|
+
return [];
|
|
345
|
+
}
|
|
346
|
+
throw new MastraError(
|
|
347
|
+
{
|
|
348
|
+
id: 'CLICKHOUSE_STORAGE_GET_TRACES_FAILED',
|
|
349
|
+
domain: ErrorDomain.STORAGE,
|
|
350
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
351
|
+
details: {
|
|
352
|
+
name: name ?? null,
|
|
353
|
+
scope: scope ?? null,
|
|
354
|
+
page,
|
|
355
|
+
perPage,
|
|
356
|
+
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
357
|
+
filters: filters ? JSON.stringify(filters) : null,
|
|
358
|
+
fromDate: fromDate?.toISOString() ?? null,
|
|
359
|
+
toDate: toDate?.toISOString() ?? null,
|
|
360
|
+
},
|
|
361
|
+
},
|
|
362
|
+
error,
|
|
363
|
+
);
|
|
364
|
+
}
|
|
306
365
|
}
|
|
307
366
|
|
|
308
367
|
async optimizeTable({ tableName }: { tableName: TABLE_NAMES }): Promise<void> {
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
368
|
+
try {
|
|
369
|
+
await this.db.command({
|
|
370
|
+
query: `OPTIMIZE TABLE ${tableName} FINAL`,
|
|
371
|
+
});
|
|
372
|
+
} catch (error: any) {
|
|
373
|
+
throw new MastraError(
|
|
374
|
+
{
|
|
375
|
+
id: 'CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED',
|
|
376
|
+
domain: ErrorDomain.STORAGE,
|
|
377
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
378
|
+
details: { tableName },
|
|
379
|
+
},
|
|
380
|
+
error,
|
|
381
|
+
);
|
|
382
|
+
}
|
|
312
383
|
}
|
|
313
384
|
|
|
314
385
|
async materializeTtl({ tableName }: { tableName: TABLE_NAMES }): Promise<void> {
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
386
|
+
try {
|
|
387
|
+
await this.db.command({
|
|
388
|
+
query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`,
|
|
389
|
+
});
|
|
390
|
+
} catch (error: any) {
|
|
391
|
+
throw new MastraError(
|
|
392
|
+
{
|
|
393
|
+
id: 'CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED',
|
|
394
|
+
domain: ErrorDomain.STORAGE,
|
|
395
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
396
|
+
details: { tableName },
|
|
397
|
+
},
|
|
398
|
+
error,
|
|
399
|
+
);
|
|
400
|
+
}
|
|
318
401
|
}
|
|
319
402
|
|
|
320
403
|
async createTable({
|
|
321
404
|
tableName,
|
|
322
405
|
schema,
|
|
323
406
|
}: {
|
|
324
|
-
tableName:
|
|
407
|
+
tableName: SUPPORTED_TABLE_NAMES;
|
|
325
408
|
schema: Record<string, StorageColumn>;
|
|
326
409
|
}): Promise<void> {
|
|
327
410
|
try {
|
|
@@ -368,9 +451,16 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
368
451
|
output_format_json_quote_64bit_integers: 0,
|
|
369
452
|
},
|
|
370
453
|
});
|
|
371
|
-
} catch (error) {
|
|
372
|
-
|
|
373
|
-
|
|
454
|
+
} catch (error: any) {
|
|
455
|
+
throw new MastraError(
|
|
456
|
+
{
|
|
457
|
+
id: 'CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED',
|
|
458
|
+
domain: ErrorDomain.STORAGE,
|
|
459
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
460
|
+
details: { tableName },
|
|
461
|
+
},
|
|
462
|
+
error,
|
|
463
|
+
);
|
|
374
464
|
}
|
|
375
465
|
}
|
|
376
466
|
|
|
@@ -433,11 +523,16 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
433
523
|
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
434
524
|
}
|
|
435
525
|
}
|
|
436
|
-
} catch (error) {
|
|
437
|
-
|
|
438
|
-
|
|
526
|
+
} catch (error: any) {
|
|
527
|
+
throw new MastraError(
|
|
528
|
+
{
|
|
529
|
+
id: 'CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED',
|
|
530
|
+
domain: ErrorDomain.STORAGE,
|
|
531
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
532
|
+
details: { tableName },
|
|
533
|
+
},
|
|
534
|
+
error,
|
|
439
535
|
);
|
|
440
|
-
throw new Error(`Failed to alter table ${tableName}: ${error}`);
|
|
441
536
|
}
|
|
442
537
|
}
|
|
443
538
|
|
|
@@ -453,9 +548,16 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
453
548
|
output_format_json_quote_64bit_integers: 0,
|
|
454
549
|
},
|
|
455
550
|
});
|
|
456
|
-
} catch (error) {
|
|
457
|
-
|
|
458
|
-
|
|
551
|
+
} catch (error: any) {
|
|
552
|
+
throw new MastraError(
|
|
553
|
+
{
|
|
554
|
+
id: 'CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED',
|
|
555
|
+
domain: ErrorDomain.STORAGE,
|
|
556
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
557
|
+
details: { tableName },
|
|
558
|
+
},
|
|
559
|
+
error,
|
|
560
|
+
);
|
|
459
561
|
}
|
|
460
562
|
}
|
|
461
563
|
|
|
@@ -478,13 +580,26 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
478
580
|
use_client_time_zone: 1,
|
|
479
581
|
},
|
|
480
582
|
});
|
|
481
|
-
} catch (error) {
|
|
482
|
-
|
|
483
|
-
|
|
583
|
+
} catch (error: any) {
|
|
584
|
+
throw new MastraError(
|
|
585
|
+
{
|
|
586
|
+
id: 'CLICKHOUSE_STORAGE_INSERT_FAILED',
|
|
587
|
+
domain: ErrorDomain.STORAGE,
|
|
588
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
589
|
+
details: { tableName },
|
|
590
|
+
},
|
|
591
|
+
error,
|
|
592
|
+
);
|
|
484
593
|
}
|
|
485
594
|
}
|
|
486
595
|
|
|
487
|
-
async load<R>({
|
|
596
|
+
async load<R>({
|
|
597
|
+
tableName,
|
|
598
|
+
keys,
|
|
599
|
+
}: {
|
|
600
|
+
tableName: SUPPORTED_TABLE_NAMES;
|
|
601
|
+
keys: Record<string, string>;
|
|
602
|
+
}): Promise<R | null> {
|
|
488
603
|
try {
|
|
489
604
|
const keyEntries = Object.entries(keys);
|
|
490
605
|
const conditions = keyEntries
|
|
@@ -498,7 +613,7 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
498
613
|
}, {});
|
|
499
614
|
|
|
500
615
|
const result = await this.db.query({
|
|
501
|
-
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt, toDateTime64(updatedAt, 3) as updatedAt FROM ${tableName} ${TABLE_ENGINES[tableName as
|
|
616
|
+
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt, toDateTime64(updatedAt, 3) as updatedAt FROM ${tableName} ${TABLE_ENGINES[tableName as SUPPORTED_TABLE_NAMES].startsWith('ReplacingMergeTree') ? 'FINAL' : ''} WHERE ${conditions}`,
|
|
502
617
|
query_params: values,
|
|
503
618
|
clickhouse_settings: {
|
|
504
619
|
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
@@ -529,8 +644,15 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
529
644
|
const data: R = transformRow(rows.data[0]);
|
|
530
645
|
return data;
|
|
531
646
|
} catch (error) {
|
|
532
|
-
|
|
533
|
-
|
|
647
|
+
throw new MastraError(
|
|
648
|
+
{
|
|
649
|
+
id: 'CLICKHOUSE_STORAGE_LOAD_FAILED',
|
|
650
|
+
domain: ErrorDomain.STORAGE,
|
|
651
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
652
|
+
details: { tableName },
|
|
653
|
+
},
|
|
654
|
+
error,
|
|
655
|
+
);
|
|
534
656
|
}
|
|
535
657
|
}
|
|
536
658
|
|
|
@@ -570,9 +692,16 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
570
692
|
createdAt: thread.createdAt,
|
|
571
693
|
updatedAt: thread.updatedAt,
|
|
572
694
|
};
|
|
573
|
-
} catch (error) {
|
|
574
|
-
|
|
575
|
-
|
|
695
|
+
} catch (error: any) {
|
|
696
|
+
throw new MastraError(
|
|
697
|
+
{
|
|
698
|
+
id: 'CLICKHOUSE_STORAGE_GET_THREAD_BY_ID_FAILED',
|
|
699
|
+
domain: ErrorDomain.STORAGE,
|
|
700
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
701
|
+
details: { threadId },
|
|
702
|
+
},
|
|
703
|
+
error,
|
|
704
|
+
);
|
|
576
705
|
}
|
|
577
706
|
}
|
|
578
707
|
|
|
@@ -608,8 +737,15 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
608
737
|
updatedAt: thread.updatedAt,
|
|
609
738
|
}));
|
|
610
739
|
} catch (error) {
|
|
611
|
-
|
|
612
|
-
|
|
740
|
+
throw new MastraError(
|
|
741
|
+
{
|
|
742
|
+
id: 'CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_FAILED',
|
|
743
|
+
domain: ErrorDomain.STORAGE,
|
|
744
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
745
|
+
details: { resourceId },
|
|
746
|
+
},
|
|
747
|
+
error,
|
|
748
|
+
);
|
|
613
749
|
}
|
|
614
750
|
}
|
|
615
751
|
|
|
@@ -635,8 +771,15 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
635
771
|
|
|
636
772
|
return thread;
|
|
637
773
|
} catch (error) {
|
|
638
|
-
|
|
639
|
-
|
|
774
|
+
throw new MastraError(
|
|
775
|
+
{
|
|
776
|
+
id: 'CLICKHOUSE_STORAGE_SAVE_THREAD_FAILED',
|
|
777
|
+
domain: ErrorDomain.STORAGE,
|
|
778
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
779
|
+
details: { threadId: thread.id },
|
|
780
|
+
},
|
|
781
|
+
error,
|
|
782
|
+
);
|
|
640
783
|
}
|
|
641
784
|
}
|
|
642
785
|
|
|
@@ -691,8 +834,15 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
691
834
|
|
|
692
835
|
return updatedThread;
|
|
693
836
|
} catch (error) {
|
|
694
|
-
|
|
695
|
-
|
|
837
|
+
throw new MastraError(
|
|
838
|
+
{
|
|
839
|
+
id: 'CLICKHOUSE_STORAGE_UPDATE_THREAD_FAILED',
|
|
840
|
+
domain: ErrorDomain.STORAGE,
|
|
841
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
842
|
+
details: { threadId: id, title },
|
|
843
|
+
},
|
|
844
|
+
error,
|
|
845
|
+
);
|
|
696
846
|
}
|
|
697
847
|
}
|
|
698
848
|
|
|
@@ -716,8 +866,15 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
716
866
|
},
|
|
717
867
|
});
|
|
718
868
|
} catch (error) {
|
|
719
|
-
|
|
720
|
-
|
|
869
|
+
throw new MastraError(
|
|
870
|
+
{
|
|
871
|
+
id: 'CLICKHOUSE_STORAGE_DELETE_THREAD_FAILED',
|
|
872
|
+
domain: ErrorDomain.STORAGE,
|
|
873
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
874
|
+
details: { threadId },
|
|
875
|
+
},
|
|
876
|
+
error,
|
|
877
|
+
);
|
|
721
878
|
}
|
|
722
879
|
}
|
|
723
880
|
|
|
@@ -731,7 +888,7 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
731
888
|
}: StorageGetMessagesArg & { format?: 'v1' | 'v2' }): Promise<MastraMessageV1[] | MastraMessageV2[]> {
|
|
732
889
|
try {
|
|
733
890
|
const messages: any[] = [];
|
|
734
|
-
const limit =
|
|
891
|
+
const limit = this.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
735
892
|
const include = selectBy?.include || [];
|
|
736
893
|
|
|
737
894
|
if (include.length) {
|
|
@@ -839,8 +996,15 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
839
996
|
if (format === `v2`) return list.get.all.v2();
|
|
840
997
|
return list.get.all.v1();
|
|
841
998
|
} catch (error) {
|
|
842
|
-
|
|
843
|
-
|
|
999
|
+
throw new MastraError(
|
|
1000
|
+
{
|
|
1001
|
+
id: 'CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED',
|
|
1002
|
+
domain: ErrorDomain.STORAGE,
|
|
1003
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1004
|
+
details: { threadId, resourceId: resourceId ?? '' },
|
|
1005
|
+
},
|
|
1006
|
+
error,
|
|
1007
|
+
);
|
|
844
1008
|
}
|
|
845
1009
|
}
|
|
846
1010
|
|
|
@@ -865,13 +1029,62 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
865
1029
|
throw new Error(`Thread ${threadId} not found`);
|
|
866
1030
|
}
|
|
867
1031
|
|
|
1032
|
+
// Clickhouse's MergeTree engine does not support native upserts or unique constraints on (id, thread_id).
|
|
1033
|
+
// Note: We cannot switch to ReplacingMergeTree without a schema migration,
|
|
1034
|
+
// as it would require altering the table engine.
|
|
1035
|
+
// To ensure correct upsert behavior, we first fetch existing (id, thread_id) pairs for the incoming messages.
|
|
1036
|
+
const existingResult = await this.db.query({
|
|
1037
|
+
query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
1038
|
+
query_params: {
|
|
1039
|
+
ids: messages.map(m => m.id),
|
|
1040
|
+
},
|
|
1041
|
+
clickhouse_settings: {
|
|
1042
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1043
|
+
date_time_input_format: 'best_effort',
|
|
1044
|
+
date_time_output_format: 'iso',
|
|
1045
|
+
use_client_time_zone: 1,
|
|
1046
|
+
output_format_json_quote_64bit_integers: 0,
|
|
1047
|
+
},
|
|
1048
|
+
format: 'JSONEachRow',
|
|
1049
|
+
});
|
|
1050
|
+
const existingRows: Array<{ id: string; thread_id: string }> = await existingResult.json();
|
|
1051
|
+
|
|
1052
|
+
const existingSet = new Set(existingRows.map(row => `${row.id}::${row.thread_id}`));
|
|
1053
|
+
// Partition the batch into new inserts and updates:
|
|
1054
|
+
// New messages are inserted in bulk.
|
|
1055
|
+
const toInsert = messages.filter(m => !existingSet.has(`${m.id}::${threadId}`));
|
|
1056
|
+
// Existing messages are updated via ALTER TABLE ... UPDATE.
|
|
1057
|
+
const toUpdate = messages.filter(m => existingSet.has(`${m.id}::${threadId}`));
|
|
1058
|
+
const updatePromises = toUpdate.map(message =>
|
|
1059
|
+
this.db.command({
|
|
1060
|
+
query: `
|
|
1061
|
+
ALTER TABLE ${TABLE_MESSAGES}
|
|
1062
|
+
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}
|
|
1063
|
+
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
1064
|
+
`,
|
|
1065
|
+
query_params: {
|
|
1066
|
+
var_content: typeof message.content === 'string' ? message.content : JSON.stringify(message.content),
|
|
1067
|
+
var_role: message.role,
|
|
1068
|
+
var_type: message.type || 'v2',
|
|
1069
|
+
var_id: message.id,
|
|
1070
|
+
var_thread_id: threadId,
|
|
1071
|
+
},
|
|
1072
|
+
clickhouse_settings: {
|
|
1073
|
+
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1074
|
+
date_time_input_format: 'best_effort',
|
|
1075
|
+
use_client_time_zone: 1,
|
|
1076
|
+
output_format_json_quote_64bit_integers: 0,
|
|
1077
|
+
},
|
|
1078
|
+
}),
|
|
1079
|
+
);
|
|
1080
|
+
|
|
868
1081
|
// Execute message inserts and thread update in parallel for better performance
|
|
869
1082
|
await Promise.all([
|
|
870
1083
|
// Insert messages
|
|
871
1084
|
this.db.insert({
|
|
872
1085
|
table: TABLE_MESSAGES,
|
|
873
1086
|
format: 'JSONEachRow',
|
|
874
|
-
values:
|
|
1087
|
+
values: toInsert.map(message => ({
|
|
875
1088
|
id: message.id,
|
|
876
1089
|
thread_id: threadId,
|
|
877
1090
|
content: typeof message.content === 'string' ? message.content : JSON.stringify(message.content),
|
|
@@ -886,6 +1099,7 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
886
1099
|
output_format_json_quote_64bit_integers: 0,
|
|
887
1100
|
},
|
|
888
1101
|
}),
|
|
1102
|
+
...updatePromises,
|
|
889
1103
|
// Update thread's updatedAt timestamp
|
|
890
1104
|
this.db.insert({
|
|
891
1105
|
table: TABLE_THREADS,
|
|
@@ -911,9 +1125,15 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
911
1125
|
const list = new MessageList({ threadId, resourceId }).add(messages, 'memory');
|
|
912
1126
|
if (format === `v2`) return list.get.all.v2();
|
|
913
1127
|
return list.get.all.v1();
|
|
914
|
-
} catch (error) {
|
|
915
|
-
|
|
916
|
-
|
|
1128
|
+
} catch (error: any) {
|
|
1129
|
+
throw new MastraError(
|
|
1130
|
+
{
|
|
1131
|
+
id: 'CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED',
|
|
1132
|
+
domain: ErrorDomain.STORAGE,
|
|
1133
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1134
|
+
},
|
|
1135
|
+
error,
|
|
1136
|
+
);
|
|
917
1137
|
}
|
|
918
1138
|
}
|
|
919
1139
|
|
|
@@ -958,9 +1178,16 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
958
1178
|
output_format_json_quote_64bit_integers: 0,
|
|
959
1179
|
},
|
|
960
1180
|
});
|
|
961
|
-
} catch (error) {
|
|
962
|
-
|
|
963
|
-
|
|
1181
|
+
} catch (error: any) {
|
|
1182
|
+
throw new MastraError(
|
|
1183
|
+
{
|
|
1184
|
+
id: 'CLICKHOUSE_STORAGE_PERSIST_WORKFLOW_SNAPSHOT_FAILED',
|
|
1185
|
+
domain: ErrorDomain.STORAGE,
|
|
1186
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1187
|
+
details: { workflowName, runId },
|
|
1188
|
+
},
|
|
1189
|
+
error,
|
|
1190
|
+
);
|
|
964
1191
|
}
|
|
965
1192
|
}
|
|
966
1193
|
|
|
@@ -985,9 +1212,16 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
985
1212
|
}
|
|
986
1213
|
|
|
987
1214
|
return (result as any).snapshot;
|
|
988
|
-
} catch (error) {
|
|
989
|
-
|
|
990
|
-
|
|
1215
|
+
} catch (error: any) {
|
|
1216
|
+
throw new MastraError(
|
|
1217
|
+
{
|
|
1218
|
+
id: 'CLICKHOUSE_STORAGE_LOAD_WORKFLOW_SNAPSHOT_FAILED',
|
|
1219
|
+
domain: ErrorDomain.STORAGE,
|
|
1220
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1221
|
+
details: { workflowName, runId },
|
|
1222
|
+
},
|
|
1223
|
+
error,
|
|
1224
|
+
);
|
|
991
1225
|
}
|
|
992
1226
|
}
|
|
993
1227
|
|
|
@@ -1100,9 +1334,16 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
1100
1334
|
|
|
1101
1335
|
// Use runs.length as total when not paginating
|
|
1102
1336
|
return { runs, total: total || runs.length };
|
|
1103
|
-
} catch (error) {
|
|
1104
|
-
|
|
1105
|
-
|
|
1337
|
+
} catch (error: any) {
|
|
1338
|
+
throw new MastraError(
|
|
1339
|
+
{
|
|
1340
|
+
id: 'CLICKHOUSE_STORAGE_GET_WORKFLOW_RUNS_FAILED',
|
|
1341
|
+
domain: ErrorDomain.STORAGE,
|
|
1342
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1343
|
+
details: { workflowName: workflowName ?? '', resourceId: resourceId ?? '' },
|
|
1344
|
+
},
|
|
1345
|
+
error,
|
|
1346
|
+
);
|
|
1106
1347
|
}
|
|
1107
1348
|
}
|
|
1108
1349
|
|
|
@@ -1151,9 +1392,16 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
1151
1392
|
return null;
|
|
1152
1393
|
}
|
|
1153
1394
|
return this.parseWorkflowRun(resultJson[0]);
|
|
1154
|
-
} catch (error) {
|
|
1155
|
-
|
|
1156
|
-
|
|
1395
|
+
} catch (error: any) {
|
|
1396
|
+
throw new MastraError(
|
|
1397
|
+
{
|
|
1398
|
+
id: 'CLICKHOUSE_STORAGE_GET_WORKFLOW_RUN_BY_ID_FAILED',
|
|
1399
|
+
domain: ErrorDomain.STORAGE,
|
|
1400
|
+
category: ErrorCategory.THIRD_PARTY,
|
|
1401
|
+
details: { runId: runId ?? '', workflowName: workflowName ?? '' },
|
|
1402
|
+
},
|
|
1403
|
+
error,
|
|
1404
|
+
);
|
|
1157
1405
|
}
|
|
1158
1406
|
}
|
|
1159
1407
|
|
|
@@ -1167,7 +1415,12 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
1167
1415
|
}
|
|
1168
1416
|
|
|
1169
1417
|
async getTracesPaginated(_args: StorageGetTracesArg): Promise<PaginationInfo & { traces: Trace[] }> {
|
|
1170
|
-
throw new
|
|
1418
|
+
throw new MastraError({
|
|
1419
|
+
id: 'CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED',
|
|
1420
|
+
domain: ErrorDomain.STORAGE,
|
|
1421
|
+
category: ErrorCategory.USER,
|
|
1422
|
+
text: 'Method not implemented.',
|
|
1423
|
+
});
|
|
1171
1424
|
}
|
|
1172
1425
|
|
|
1173
1426
|
async getThreadsByResourceIdPaginated(_args: {
|
|
@@ -1175,13 +1428,23 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
1175
1428
|
page?: number;
|
|
1176
1429
|
perPage?: number;
|
|
1177
1430
|
}): Promise<PaginationInfo & { threads: StorageThreadType[] }> {
|
|
1178
|
-
throw new
|
|
1431
|
+
throw new MastraError({
|
|
1432
|
+
id: 'CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED',
|
|
1433
|
+
domain: ErrorDomain.STORAGE,
|
|
1434
|
+
category: ErrorCategory.USER,
|
|
1435
|
+
text: 'Method not implemented.',
|
|
1436
|
+
});
|
|
1179
1437
|
}
|
|
1180
1438
|
|
|
1181
1439
|
async getMessagesPaginated(
|
|
1182
1440
|
_args: StorageGetMessagesArg,
|
|
1183
1441
|
): Promise<PaginationInfo & { messages: MastraMessageV1[] | MastraMessageV2[] }> {
|
|
1184
|
-
throw new
|
|
1442
|
+
throw new MastraError({
|
|
1443
|
+
id: 'CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED',
|
|
1444
|
+
domain: ErrorDomain.STORAGE,
|
|
1445
|
+
category: ErrorCategory.USER,
|
|
1446
|
+
text: 'Method not implemented.',
|
|
1447
|
+
});
|
|
1185
1448
|
}
|
|
1186
1449
|
|
|
1187
1450
|
async close(): Promise<void> {
|