@mastra/clickhouse 0.14.2 → 0.14.3-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1473 +0,0 @@
1
- import type { ClickHouseClient } from '@clickhouse/client';
2
- import { MessageList } from '@mastra/core/agent';
3
- import type { MastraMessageContentV2 } from '@mastra/core/agent';
4
- import { ErrorCategory, ErrorDomain, MastraError } from '@mastra/core/error';
5
- import type { MastraMessageV1, MastraMessageV2, StorageThreadType } from '@mastra/core/memory';
6
- import type { PaginationInfo, StorageGetMessagesArg, StorageResourceType } from '@mastra/core/storage';
7
- import {
8
- MemoryStorage,
9
- resolveMessageLimit,
10
- TABLE_MESSAGES,
11
- TABLE_RESOURCES,
12
- TABLE_THREADS,
13
- } from '@mastra/core/storage';
14
- import type { StoreOperationsClickhouse } from '../operations';
15
- import { transformRow, transformRows } from '../utils';
16
-
17
- export class MemoryStorageClickhouse extends MemoryStorage {
18
- protected client: ClickHouseClient;
19
- protected operations: StoreOperationsClickhouse;
20
- constructor({ client, operations }: { client: ClickHouseClient; operations: StoreOperationsClickhouse }) {
21
- super();
22
- this.client = client;
23
- this.operations = operations;
24
- }
25
-
26
- public async getMessages(args: StorageGetMessagesArg & { format?: 'v1' }): Promise<MastraMessageV1[]>;
27
- public async getMessages(args: StorageGetMessagesArg & { format: 'v2' }): Promise<MastraMessageV2[]>;
28
- public async getMessages({
29
- threadId,
30
- resourceId,
31
- selectBy,
32
- format,
33
- }: StorageGetMessagesArg & { format?: 'v1' | 'v2' }): Promise<MastraMessageV1[] | MastraMessageV2[]> {
34
- try {
35
- const messages: any[] = [];
36
- const limit = resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
37
- const include = selectBy?.include || [];
38
-
39
- if (include.length) {
40
- const unionQueries: string[] = [];
41
- const params: any[] = [];
42
- let paramIdx = 1;
43
-
44
- for (const inc of include) {
45
- const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
46
- // if threadId is provided, use it, otherwise use threadId from args
47
- const searchId = inc.threadId || threadId;
48
-
49
- unionQueries.push(`
50
- SELECT * FROM (
51
- WITH numbered_messages AS (
52
- SELECT
53
- id, content, role, type, "createdAt", thread_id, "resourceId",
54
- ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
55
- FROM "${TABLE_MESSAGES}"
56
- WHERE thread_id = {var_thread_id_${paramIdx}:String}
57
- ),
58
- target_positions AS (
59
- SELECT row_num as target_pos
60
- FROM numbered_messages
61
- WHERE id = {var_include_id_${paramIdx}:String}
62
- )
63
- SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
64
- FROM numbered_messages m
65
- CROSS JOIN target_positions t
66
- WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
67
- ) AS query_${paramIdx}
68
- `);
69
-
70
- params.push(
71
- { [`var_thread_id_${paramIdx}`]: searchId },
72
- { [`var_include_id_${paramIdx}`]: id },
73
- { [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
74
- { [`var_withNextMessages_${paramIdx}`]: withNextMessages },
75
- );
76
- paramIdx++;
77
- }
78
-
79
- const finalQuery = unionQueries.join(' UNION ALL ') + ' ORDER BY "createdAt" DESC';
80
-
81
- // Merge all parameter objects
82
- const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
83
-
84
- const includeResult = await this.client.query({
85
- query: finalQuery,
86
- query_params: mergedParams,
87
- clickhouse_settings: {
88
- date_time_input_format: 'best_effort',
89
- date_time_output_format: 'iso',
90
- use_client_time_zone: 1,
91
- output_format_json_quote_64bit_integers: 0,
92
- },
93
- });
94
-
95
- const rows = await includeResult.json();
96
- const includedMessages = transformRows(rows.data);
97
-
98
- // Deduplicate messages
99
- const seen = new Set<string>();
100
- const dedupedMessages = includedMessages.filter((message: any) => {
101
- if (seen.has(message.id)) return false;
102
- seen.add(message.id);
103
- return true;
104
- });
105
-
106
- messages.push(...dedupedMessages);
107
- }
108
-
109
- // Then get the remaining messages, excluding the ids we just fetched
110
- const result = await this.client.query({
111
- query: `
112
- SELECT
113
- id,
114
- content,
115
- role,
116
- type,
117
- toDateTime64(createdAt, 3) as createdAt,
118
- thread_id AS "threadId"
119
- FROM "${TABLE_MESSAGES}"
120
- WHERE thread_id = {threadId:String}
121
- AND id NOT IN ({exclude:Array(String)})
122
- ORDER BY "createdAt" DESC
123
- LIMIT {limit:Int64}
124
- `,
125
- query_params: {
126
- threadId,
127
- exclude: messages.map(m => m.id),
128
- limit,
129
- },
130
- clickhouse_settings: {
131
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
132
- date_time_input_format: 'best_effort',
133
- date_time_output_format: 'iso',
134
- use_client_time_zone: 1,
135
- output_format_json_quote_64bit_integers: 0,
136
- },
137
- });
138
-
139
- const rows = await result.json();
140
- messages.push(...transformRows(rows.data));
141
-
142
- // Sort all messages by creation date
143
- messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
144
-
145
- // Parse message content
146
- messages.forEach(message => {
147
- if (typeof message.content === 'string') {
148
- try {
149
- message.content = JSON.parse(message.content);
150
- } catch {
151
- // If parsing fails, leave as string
152
- }
153
- }
154
- });
155
-
156
- const list = new MessageList({ threadId, resourceId }).add(messages, 'memory');
157
- if (format === `v2`) return list.get.all.v2();
158
- return list.get.all.v1();
159
- } catch (error) {
160
- throw new MastraError(
161
- {
162
- id: 'CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED',
163
- domain: ErrorDomain.STORAGE,
164
- category: ErrorCategory.THIRD_PARTY,
165
- details: { threadId, resourceId: resourceId ?? '' },
166
- },
167
- error,
168
- );
169
- }
170
- }
171
-
172
- public async getMessagesById({
173
- messageIds,
174
- format,
175
- }: {
176
- messageIds: string[];
177
- format: 'v1';
178
- }): Promise<MastraMessageV1[]>;
179
- public async getMessagesById({
180
- messageIds,
181
- format,
182
- }: {
183
- messageIds: string[];
184
- format?: 'v2';
185
- }): Promise<MastraMessageV2[]>;
186
- public async getMessagesById({
187
- messageIds,
188
- format,
189
- }: {
190
- messageIds: string[];
191
- format?: 'v1' | 'v2';
192
- }): Promise<MastraMessageV1[] | MastraMessageV2[]> {
193
- if (messageIds.length === 0) return [];
194
-
195
- try {
196
- const result = await this.client.query({
197
- query: `
198
- SELECT
199
- id,
200
- content,
201
- role,
202
- type,
203
- toDateTime64(createdAt, 3) as createdAt,
204
- thread_id AS "threadId",
205
- "resourceId"
206
- FROM "${TABLE_MESSAGES}"
207
- WHERE id IN {messageIds:Array(String)}
208
- ORDER BY "createdAt" DESC
209
- `,
210
- query_params: {
211
- messageIds,
212
- },
213
- clickhouse_settings: {
214
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
215
- date_time_input_format: 'best_effort',
216
- date_time_output_format: 'iso',
217
- use_client_time_zone: 1,
218
- output_format_json_quote_64bit_integers: 0,
219
- },
220
- });
221
-
222
- const rows = await result.json();
223
- const messages: any[] = transformRows(rows.data);
224
-
225
- // Parse message content
226
- messages.forEach(message => {
227
- if (typeof message.content === 'string') {
228
- try {
229
- message.content = JSON.parse(message.content);
230
- } catch {
231
- // If parsing fails, leave as string
232
- }
233
- }
234
- });
235
-
236
- const list = new MessageList().add(messages, 'memory');
237
- if (format === `v1`) return list.get.all.v1();
238
- return list.get.all.v2();
239
- } catch (error) {
240
- throw new MastraError(
241
- {
242
- id: 'CLICKHOUSE_STORAGE_GET_MESSAGES_BY_ID_FAILED',
243
- domain: ErrorDomain.STORAGE,
244
- category: ErrorCategory.THIRD_PARTY,
245
- details: { messageIds: JSON.stringify(messageIds) },
246
- },
247
- error,
248
- );
249
- }
250
- }
251
-
252
- async saveMessages(args: { messages: MastraMessageV1[]; format?: undefined | 'v1' }): Promise<MastraMessageV1[]>;
253
- async saveMessages(args: { messages: MastraMessageV2[]; format: 'v2' }): Promise<MastraMessageV2[]>;
254
- async saveMessages(
255
- args: { messages: MastraMessageV1[]; format?: undefined | 'v1' } | { messages: MastraMessageV2[]; format: 'v2' },
256
- ): Promise<MastraMessageV2[] | MastraMessageV1[]> {
257
- const { messages, format = 'v1' } = args;
258
- if (messages.length === 0) return messages;
259
-
260
- for (const message of messages) {
261
- const resourceId = message.resourceId;
262
- if (!resourceId) {
263
- throw new Error('Resource ID is required');
264
- }
265
-
266
- if (!message.threadId) {
267
- throw new Error('Thread ID is required');
268
- }
269
-
270
- // Check if thread exists
271
- const thread = await this.getThreadById({ threadId: message.threadId });
272
- if (!thread) {
273
- throw new Error(`Thread ${message.threadId} not found`);
274
- }
275
- }
276
-
277
- const threadIdSet = new Map();
278
-
279
- await Promise.all(
280
- messages.map(async m => {
281
- const resourceId = m.resourceId;
282
- if (!resourceId) {
283
- throw new Error('Resource ID is required');
284
- }
285
-
286
- if (!m.threadId) {
287
- throw new Error('Thread ID is required');
288
- }
289
-
290
- // Check if thread exists
291
- const thread = await this.getThreadById({ threadId: m.threadId });
292
- if (!thread) {
293
- throw new Error(`Thread ${m.threadId} not found`);
294
- }
295
-
296
- threadIdSet.set(m.threadId, thread);
297
- }),
298
- );
299
-
300
- try {
301
- // Clickhouse's MergeTree engine does not support native upserts or unique constraints on (id, thread_id).
302
- // Note: We cannot switch to ReplacingMergeTree without a schema migration,
303
- // as it would require altering the table engine.
304
- // To ensure correct upsert behavior, we first fetch existing (id, thread_id) pairs for the incoming messages.
305
- const existingResult = await this.client.query({
306
- query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
307
- query_params: {
308
- ids: messages.map(m => m.id),
309
- },
310
- clickhouse_settings: {
311
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
312
- date_time_input_format: 'best_effort',
313
- date_time_output_format: 'iso',
314
- use_client_time_zone: 1,
315
- output_format_json_quote_64bit_integers: 0,
316
- },
317
- format: 'JSONEachRow',
318
- });
319
- const existingRows: Array<{ id: string; thread_id: string }> = await existingResult.json();
320
-
321
- const existingSet = new Set(existingRows.map(row => `${row.id}::${row.thread_id}`));
322
-
323
- // Partition the batch into different operations:
324
- // 1. New messages (insert)
325
- // 2. Existing messages with same (id, threadId) (update)
326
- // 3. Messages with same id but different threadId (delete old + insert new)
327
- const toInsert = messages.filter(m => !existingSet.has(`${m.id}::${m.threadId}`));
328
- const toUpdate = messages.filter(m => existingSet.has(`${m.id}::${m.threadId}`));
329
-
330
- // Find messages that need to be moved (same id, different threadId)
331
- const toMove = messages.filter(m => {
332
- const existingRow = existingRows.find(row => row.id === m.id);
333
- return existingRow && existingRow.thread_id !== m.threadId;
334
- });
335
-
336
- // Delete old messages that are being moved
337
- const deletePromises = toMove.map(message => {
338
- const existingRow = existingRows.find(row => row.id === message.id);
339
- if (!existingRow) return Promise.resolve();
340
-
341
- return this.client.command({
342
- query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
343
- query_params: {
344
- var_id: message.id,
345
- var_old_thread_id: existingRow.thread_id,
346
- },
347
- clickhouse_settings: {
348
- date_time_input_format: 'best_effort',
349
- use_client_time_zone: 1,
350
- output_format_json_quote_64bit_integers: 0,
351
- },
352
- });
353
- });
354
-
355
- const updatePromises = toUpdate.map(message =>
356
- this.client.command({
357
- query: `
358
- ALTER TABLE ${TABLE_MESSAGES}
359
- UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
360
- WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
361
- `,
362
- query_params: {
363
- var_content: typeof message.content === 'string' ? message.content : JSON.stringify(message.content),
364
- var_role: message.role,
365
- var_type: message.type || 'v2',
366
- var_resourceId: message.resourceId,
367
- var_id: message.id,
368
- var_thread_id: message.threadId,
369
- },
370
- clickhouse_settings: {
371
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
372
- date_time_input_format: 'best_effort',
373
- use_client_time_zone: 1,
374
- output_format_json_quote_64bit_integers: 0,
375
- },
376
- }),
377
- );
378
-
379
- // Execute message operations and thread update in parallel for better performance
380
- await Promise.all([
381
- // Insert new messages (including moved messages)
382
- this.client.insert({
383
- table: TABLE_MESSAGES,
384
- format: 'JSONEachRow',
385
- values: toInsert.map(message => ({
386
- id: message.id,
387
- thread_id: message.threadId,
388
- resourceId: message.resourceId,
389
- content: typeof message.content === 'string' ? message.content : JSON.stringify(message.content),
390
- createdAt: message.createdAt.toISOString(),
391
- role: message.role,
392
- type: message.type || 'v2',
393
- })),
394
- clickhouse_settings: {
395
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
396
- date_time_input_format: 'best_effort',
397
- use_client_time_zone: 1,
398
- output_format_json_quote_64bit_integers: 0,
399
- },
400
- }),
401
- ...updatePromises,
402
- ...deletePromises,
403
- // Update thread's updatedAt timestamp
404
- this.client.insert({
405
- table: TABLE_THREADS,
406
- format: 'JSONEachRow',
407
- values: Array.from(threadIdSet.values()).map(thread => ({
408
- id: thread.id,
409
- resourceId: thread.resourceId,
410
- title: thread.title,
411
- metadata: thread.metadata,
412
- createdAt: thread.createdAt,
413
- updatedAt: new Date().toISOString(),
414
- })),
415
- clickhouse_settings: {
416
- date_time_input_format: 'best_effort',
417
- use_client_time_zone: 1,
418
- output_format_json_quote_64bit_integers: 0,
419
- },
420
- }),
421
- ]);
422
-
423
- const list = new MessageList().add(messages, 'memory');
424
-
425
- if (format === `v2`) return list.get.all.v2();
426
- return list.get.all.v1();
427
- } catch (error: any) {
428
- throw new MastraError(
429
- {
430
- id: 'CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED',
431
- domain: ErrorDomain.STORAGE,
432
- category: ErrorCategory.THIRD_PARTY,
433
- },
434
- error,
435
- );
436
- }
437
- }
438
-
439
- async getThreadById({ threadId }: { threadId: string }): Promise<StorageThreadType | null> {
440
- try {
441
- const result = await this.client.query({
442
- query: `SELECT
443
- id,
444
- "resourceId",
445
- title,
446
- metadata,
447
- toDateTime64(createdAt, 3) as createdAt,
448
- toDateTime64(updatedAt, 3) as updatedAt
449
- FROM "${TABLE_THREADS}"
450
- FINAL
451
- WHERE id = {var_id:String}`,
452
- query_params: { var_id: threadId },
453
- clickhouse_settings: {
454
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
455
- date_time_input_format: 'best_effort',
456
- date_time_output_format: 'iso',
457
- use_client_time_zone: 1,
458
- output_format_json_quote_64bit_integers: 0,
459
- },
460
- });
461
-
462
- const rows = await result.json();
463
- const thread = transformRow(rows.data[0]) as StorageThreadType;
464
-
465
- if (!thread) {
466
- return null;
467
- }
468
-
469
- return {
470
- ...thread,
471
- metadata: typeof thread.metadata === 'string' ? JSON.parse(thread.metadata) : thread.metadata,
472
- createdAt: thread.createdAt,
473
- updatedAt: thread.updatedAt,
474
- };
475
- } catch (error: any) {
476
- throw new MastraError(
477
- {
478
- id: 'CLICKHOUSE_STORAGE_GET_THREAD_BY_ID_FAILED',
479
- domain: ErrorDomain.STORAGE,
480
- category: ErrorCategory.THIRD_PARTY,
481
- details: { threadId },
482
- },
483
- error,
484
- );
485
- }
486
- }
487
-
488
- async getThreadsByResourceId({ resourceId }: { resourceId: string }): Promise<StorageThreadType[]> {
489
- try {
490
- const result = await this.client.query({
491
- query: `SELECT
492
- id,
493
- "resourceId",
494
- title,
495
- metadata,
496
- toDateTime64(createdAt, 3) as createdAt,
497
- toDateTime64(updatedAt, 3) as updatedAt
498
- FROM "${TABLE_THREADS}"
499
- WHERE "resourceId" = {var_resourceId:String}`,
500
- query_params: { var_resourceId: resourceId },
501
- clickhouse_settings: {
502
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
503
- date_time_input_format: 'best_effort',
504
- date_time_output_format: 'iso',
505
- use_client_time_zone: 1,
506
- output_format_json_quote_64bit_integers: 0,
507
- },
508
- });
509
-
510
- const rows = await result.json();
511
- const threads = transformRows(rows.data) as StorageThreadType[];
512
-
513
- return threads.map((thread: StorageThreadType) => ({
514
- ...thread,
515
- metadata: typeof thread.metadata === 'string' ? JSON.parse(thread.metadata) : thread.metadata,
516
- createdAt: thread.createdAt,
517
- updatedAt: thread.updatedAt,
518
- }));
519
- } catch (error) {
520
- throw new MastraError(
521
- {
522
- id: 'CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_FAILED',
523
- domain: ErrorDomain.STORAGE,
524
- category: ErrorCategory.THIRD_PARTY,
525
- details: { resourceId },
526
- },
527
- error,
528
- );
529
- }
530
- }
531
-
532
- async saveThread({ thread }: { thread: StorageThreadType }): Promise<StorageThreadType> {
533
- try {
534
- await this.client.insert({
535
- table: TABLE_THREADS,
536
- values: [
537
- {
538
- ...thread,
539
- createdAt: thread.createdAt.toISOString(),
540
- updatedAt: thread.updatedAt.toISOString(),
541
- },
542
- ],
543
- format: 'JSONEachRow',
544
- clickhouse_settings: {
545
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
546
- date_time_input_format: 'best_effort',
547
- use_client_time_zone: 1,
548
- output_format_json_quote_64bit_integers: 0,
549
- },
550
- });
551
-
552
- return thread;
553
- } catch (error) {
554
- throw new MastraError(
555
- {
556
- id: 'CLICKHOUSE_STORAGE_SAVE_THREAD_FAILED',
557
- domain: ErrorDomain.STORAGE,
558
- category: ErrorCategory.THIRD_PARTY,
559
- details: { threadId: thread.id },
560
- },
561
- error,
562
- );
563
- }
564
- }
565
-
566
- async updateThread({
567
- id,
568
- title,
569
- metadata,
570
- }: {
571
- id: string;
572
- title: string;
573
- metadata: Record<string, unknown>;
574
- }): Promise<StorageThreadType> {
575
- try {
576
- // First get the existing thread to merge metadata
577
- const existingThread = await this.getThreadById({ threadId: id });
578
- if (!existingThread) {
579
- throw new Error(`Thread ${id} not found`);
580
- }
581
-
582
- // Merge the existing metadata with the new metadata
583
- const mergedMetadata = {
584
- ...existingThread.metadata,
585
- ...metadata,
586
- };
587
-
588
- const updatedThread = {
589
- ...existingThread,
590
- title,
591
- metadata: mergedMetadata,
592
- updatedAt: new Date(),
593
- };
594
-
595
- await this.client.insert({
596
- table: TABLE_THREADS,
597
- format: 'JSONEachRow',
598
- values: [
599
- {
600
- id: updatedThread.id,
601
- resourceId: updatedThread.resourceId,
602
- title: updatedThread.title,
603
- metadata: updatedThread.metadata,
604
- createdAt: updatedThread.createdAt,
605
- updatedAt: updatedThread.updatedAt.toISOString(),
606
- },
607
- ],
608
- clickhouse_settings: {
609
- date_time_input_format: 'best_effort',
610
- use_client_time_zone: 1,
611
- output_format_json_quote_64bit_integers: 0,
612
- },
613
- });
614
-
615
- return updatedThread;
616
- } catch (error) {
617
- throw new MastraError(
618
- {
619
- id: 'CLICKHOUSE_STORAGE_UPDATE_THREAD_FAILED',
620
- domain: ErrorDomain.STORAGE,
621
- category: ErrorCategory.THIRD_PARTY,
622
- details: { threadId: id, title },
623
- },
624
- error,
625
- );
626
- }
627
- }
628
-
629
- async deleteThread({ threadId }: { threadId: string }): Promise<void> {
630
- try {
631
- // First delete all messages associated with this thread
632
- await this.client.command({
633
- query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
634
- query_params: { var_thread_id: threadId },
635
- clickhouse_settings: {
636
- output_format_json_quote_64bit_integers: 0,
637
- },
638
- });
639
-
640
- // Then delete the thread
641
- await this.client.command({
642
- query: `DELETE FROM "${TABLE_THREADS}" WHERE id = {var_id:String};`,
643
- query_params: { var_id: threadId },
644
- clickhouse_settings: {
645
- output_format_json_quote_64bit_integers: 0,
646
- },
647
- });
648
- } catch (error) {
649
- throw new MastraError(
650
- {
651
- id: 'CLICKHOUSE_STORAGE_DELETE_THREAD_FAILED',
652
- domain: ErrorDomain.STORAGE,
653
- category: ErrorCategory.THIRD_PARTY,
654
- details: { threadId },
655
- },
656
- error,
657
- );
658
- }
659
- }
660
-
661
- async getThreadsByResourceIdPaginated(args: {
662
- resourceId: string;
663
- page?: number;
664
- perPage?: number;
665
- }): Promise<PaginationInfo & { threads: StorageThreadType[] }> {
666
- const { resourceId, page = 0, perPage = 100 } = args;
667
-
668
- try {
669
- const currentOffset = page * perPage;
670
-
671
- // Get total count
672
- const countResult = await this.client.query({
673
- query: `SELECT count() as total FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
674
- query_params: { resourceId },
675
- clickhouse_settings: {
676
- date_time_input_format: 'best_effort',
677
- date_time_output_format: 'iso',
678
- use_client_time_zone: 1,
679
- output_format_json_quote_64bit_integers: 0,
680
- },
681
- });
682
- const countData = await countResult.json();
683
- const total = (countData as any).data[0].total;
684
-
685
- if (total === 0) {
686
- return {
687
- threads: [],
688
- total: 0,
689
- page,
690
- perPage,
691
- hasMore: false,
692
- };
693
- }
694
-
695
- // Get paginated threads
696
- const dataResult = await this.client.query({
697
- query: `
698
- SELECT
699
- id,
700
- resourceId,
701
- title,
702
- metadata,
703
- toDateTime64(createdAt, 3) as createdAt,
704
- toDateTime64(updatedAt, 3) as updatedAt
705
- FROM ${TABLE_THREADS}
706
- WHERE resourceId = {resourceId:String}
707
- ORDER BY createdAt DESC
708
- LIMIT {limit:Int64} OFFSET {offset:Int64}
709
- `,
710
- query_params: {
711
- resourceId,
712
- limit: perPage,
713
- offset: currentOffset,
714
- },
715
- clickhouse_settings: {
716
- date_time_input_format: 'best_effort',
717
- date_time_output_format: 'iso',
718
- use_client_time_zone: 1,
719
- output_format_json_quote_64bit_integers: 0,
720
- },
721
- });
722
-
723
- const rows = await dataResult.json();
724
- const threads = transformRows<StorageThreadType>(rows.data);
725
-
726
- return {
727
- threads,
728
- total,
729
- page,
730
- perPage,
731
- hasMore: currentOffset + threads.length < total,
732
- };
733
- } catch (error) {
734
- throw new MastraError(
735
- {
736
- id: 'CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED',
737
- domain: ErrorDomain.STORAGE,
738
- category: ErrorCategory.THIRD_PARTY,
739
- details: { resourceId, page },
740
- },
741
- error,
742
- );
743
- }
744
- }
745
-
746
- async getMessagesPaginated(
747
- args: StorageGetMessagesArg & { format?: 'v1' | 'v2' },
748
- ): Promise<PaginationInfo & { messages: MastraMessageV1[] | MastraMessageV2[] }> {
749
- try {
750
- const { threadId, selectBy, format = 'v1' } = args;
751
- const page = selectBy?.pagination?.page || 0;
752
- const perPageInput = selectBy?.pagination?.perPage;
753
- const perPage =
754
- perPageInput !== undefined ? perPageInput : resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
755
- const offset = page * perPage;
756
- const dateRange = selectBy?.pagination?.dateRange;
757
- const fromDate = dateRange?.start;
758
- const toDate = dateRange?.end;
759
-
760
- const messages: MastraMessageV2[] = [];
761
-
762
- // Get include messages first (like libsql)
763
- if (selectBy?.include?.length) {
764
- const include = selectBy.include;
765
- const unionQueries: string[] = [];
766
- const params: any[] = [];
767
- let paramIdx = 1;
768
-
769
- for (const inc of include) {
770
- const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
771
- const searchId = inc.threadId || threadId;
772
-
773
- unionQueries.push(`
774
- SELECT * FROM (
775
- WITH numbered_messages AS (
776
- SELECT
777
- id, content, role, type, "createdAt", thread_id, "resourceId",
778
- ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
779
- FROM "${TABLE_MESSAGES}"
780
- WHERE thread_id = {var_thread_id_${paramIdx}:String}
781
- ),
782
- target_positions AS (
783
- SELECT row_num as target_pos
784
- FROM numbered_messages
785
- WHERE id = {var_include_id_${paramIdx}:String}
786
- )
787
- SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
788
- FROM numbered_messages m
789
- CROSS JOIN target_positions t
790
- WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
791
- ) AS query_${paramIdx}
792
- `);
793
-
794
- params.push(
795
- { [`var_thread_id_${paramIdx}`]: searchId },
796
- { [`var_include_id_${paramIdx}`]: id },
797
- { [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
798
- { [`var_withNextMessages_${paramIdx}`]: withNextMessages },
799
- );
800
- paramIdx++;
801
- }
802
-
803
- const finalQuery = unionQueries.join(' UNION ALL ') + ' ORDER BY "createdAt" DESC';
804
- const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
805
-
806
- const includeResult = await this.client.query({
807
- query: finalQuery,
808
- query_params: mergedParams,
809
- clickhouse_settings: {
810
- date_time_input_format: 'best_effort',
811
- date_time_output_format: 'iso',
812
- use_client_time_zone: 1,
813
- output_format_json_quote_64bit_integers: 0,
814
- },
815
- });
816
-
817
- const rows = await includeResult.json();
818
- const includedMessages = transformRows<MastraMessageV2>(rows.data);
819
-
820
- // Deduplicate messages
821
- const seen = new Set<string>();
822
- const dedupedMessages = includedMessages.filter((message: MastraMessageV2) => {
823
- if (seen.has(message.id)) return false;
824
- seen.add(message.id);
825
- return true;
826
- });
827
-
828
- messages.push(...dedupedMessages);
829
- }
830
-
831
- // Get total count
832
- let countQuery = `SELECT count() as total FROM ${TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
833
- const countParams: any = { threadId };
834
-
835
- if (fromDate) {
836
- countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
837
- countParams.fromDate = fromDate.toISOString();
838
- }
839
- if (toDate) {
840
- countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
841
- countParams.toDate = toDate.toISOString();
842
- }
843
-
844
- const countResult = await this.client.query({
845
- query: countQuery,
846
- query_params: countParams,
847
- clickhouse_settings: {
848
- date_time_input_format: 'best_effort',
849
- date_time_output_format: 'iso',
850
- use_client_time_zone: 1,
851
- output_format_json_quote_64bit_integers: 0,
852
- },
853
- });
854
- const countData = await countResult.json();
855
- const total = (countData as any).data[0].total;
856
-
857
- if (total === 0 && messages.length === 0) {
858
- return {
859
- messages: [],
860
- total: 0,
861
- page,
862
- perPage,
863
- hasMore: false,
864
- };
865
- }
866
-
867
- // Get regular paginated messages, excluding include message IDs
868
- const excludeIds = messages.map(m => m.id);
869
- let dataQuery = `
870
- SELECT
871
- id,
872
- content,
873
- role,
874
- type,
875
- toDateTime64(createdAt, 3) as createdAt,
876
- thread_id AS "threadId",
877
- resourceId
878
- FROM ${TABLE_MESSAGES}
879
- WHERE thread_id = {threadId:String}
880
- `;
881
- const dataParams: any = { threadId };
882
-
883
- if (fromDate) {
884
- dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
885
- dataParams.fromDate = fromDate.toISOString();
886
- }
887
- if (toDate) {
888
- dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
889
- dataParams.toDate = toDate.toISOString();
890
- }
891
-
892
- // Exclude include message IDs
893
- if (excludeIds.length > 0) {
894
- dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
895
- dataParams.excludeIds = excludeIds;
896
- }
897
-
898
- // For last N functionality, we need to get the most recent messages first, then sort them chronologically
899
- if (selectBy?.last) {
900
- dataQuery += `
901
- ORDER BY createdAt DESC
902
- LIMIT {limit:Int64}
903
- `;
904
- dataParams.limit = perPage;
905
- } else {
906
- dataQuery += `
907
- ORDER BY createdAt ASC
908
- LIMIT {limit:Int64} OFFSET {offset:Int64}
909
- `;
910
- dataParams.limit = perPage;
911
- dataParams.offset = offset;
912
- }
913
-
914
- const result = await this.client.query({
915
- query: dataQuery,
916
- query_params: dataParams,
917
- clickhouse_settings: {
918
- date_time_input_format: 'best_effort',
919
- date_time_output_format: 'iso',
920
- use_client_time_zone: 1,
921
- output_format_json_quote_64bit_integers: 0,
922
- },
923
- });
924
-
925
- const rows = await result.json();
926
- const paginatedMessages = transformRows<MastraMessageV2>(rows.data);
927
- messages.push(...paginatedMessages);
928
-
929
- // For last N functionality, sort messages chronologically
930
- if (selectBy?.last) {
931
- messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
932
- }
933
-
934
- return {
935
- messages: format === 'v2' ? messages : (messages as unknown as MastraMessageV1[]),
936
- total,
937
- page,
938
- perPage,
939
- hasMore: offset + perPage < total,
940
- };
941
- } catch (error: any) {
942
- throw new MastraError(
943
- {
944
- id: 'CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED',
945
- domain: ErrorDomain.STORAGE,
946
- category: ErrorCategory.THIRD_PARTY,
947
- },
948
- error,
949
- );
950
- }
951
- }
952
-
953
- async updateMessages(args: {
954
- messages: (Partial<Omit<MastraMessageV2, 'createdAt'>> & {
955
- id: string;
956
- threadId?: string;
957
- content?: { metadata?: MastraMessageContentV2['metadata']; content?: MastraMessageContentV2['content'] };
958
- })[];
959
- }): Promise<MastraMessageV2[]> {
960
- const { messages } = args;
961
-
962
- if (messages.length === 0) {
963
- return [];
964
- }
965
-
966
- try {
967
- const messageIds = messages.map(m => m.id);
968
-
969
- // Get existing messages
970
- const existingResult = await this.client.query({
971
- query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(',')})`,
972
- query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
973
- clickhouse_settings: {
974
- date_time_input_format: 'best_effort',
975
- date_time_output_format: 'iso',
976
- use_client_time_zone: 1,
977
- output_format_json_quote_64bit_integers: 0,
978
- },
979
- });
980
-
981
- const existingRows = await existingResult.json();
982
- const existingMessages = transformRows<MastraMessageV2>(existingRows.data);
983
-
984
- if (existingMessages.length === 0) {
985
- return [];
986
- }
987
-
988
- // Parse content from string to object for merging
989
- const parsedExistingMessages = existingMessages.map(msg => {
990
- if (typeof msg.content === 'string') {
991
- try {
992
- msg.content = JSON.parse(msg.content);
993
- } catch {
994
- // ignore if not valid json
995
- }
996
- }
997
- return msg;
998
- });
999
-
1000
- const threadIdsToUpdate = new Set<string>();
1001
- const updatePromises: Promise<any>[] = [];
1002
-
1003
- for (const existingMessage of parsedExistingMessages) {
1004
- const updatePayload = messages.find(m => m.id === existingMessage.id);
1005
- if (!updatePayload) continue;
1006
-
1007
- const { id, ...fieldsToUpdate } = updatePayload;
1008
- if (Object.keys(fieldsToUpdate).length === 0) continue;
1009
-
1010
- threadIdsToUpdate.add(existingMessage.threadId!);
1011
- if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
1012
- threadIdsToUpdate.add(updatePayload.threadId);
1013
- }
1014
-
1015
- const setClauses: string[] = [];
1016
- const values: any = {};
1017
- let paramIdx = 1;
1018
- let newContent: any = null;
1019
-
1020
- const updatableFields = { ...fieldsToUpdate };
1021
-
1022
- // Special handling for content: merge in code, then update the whole field
1023
- if (updatableFields.content) {
1024
- const existingContent = existingMessage.content || {};
1025
- const existingMetadata = existingContent.metadata || {};
1026
- const updateMetadata = updatableFields.content.metadata || {};
1027
-
1028
- newContent = {
1029
- ...existingContent,
1030
- ...updatableFields.content,
1031
- // Deep merge metadata
1032
- metadata: {
1033
- ...existingMetadata,
1034
- ...updateMetadata,
1035
- },
1036
- };
1037
-
1038
- // Ensure we're updating the content field
1039
- setClauses.push(`content = {var_content_${paramIdx}:String}`);
1040
- values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
1041
- paramIdx++;
1042
- delete updatableFields.content;
1043
- }
1044
-
1045
- // Handle other fields
1046
- for (const key in updatableFields) {
1047
- if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
1048
- const dbColumn = key === 'threadId' ? 'thread_id' : key;
1049
- setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
1050
- values[`var_${key}_${paramIdx}`] = updatableFields[key as keyof typeof updatableFields];
1051
- paramIdx++;
1052
- }
1053
- }
1054
-
1055
- if (setClauses.length > 0) {
1056
- values[`var_id_${paramIdx}`] = id;
1057
-
1058
- // Use ALTER TABLE UPDATE for ClickHouse
1059
- const updateQuery = `
1060
- ALTER TABLE ${TABLE_MESSAGES}
1061
- UPDATE ${setClauses.join(', ')}
1062
- WHERE id = {var_id_${paramIdx}:String}
1063
- `;
1064
-
1065
- console.log('Updating message:', id, 'with query:', updateQuery, 'values:', values);
1066
-
1067
- updatePromises.push(
1068
- this.client.command({
1069
- query: updateQuery,
1070
- query_params: values,
1071
- clickhouse_settings: {
1072
- date_time_input_format: 'best_effort',
1073
- use_client_time_zone: 1,
1074
- output_format_json_quote_64bit_integers: 0,
1075
- },
1076
- }),
1077
- );
1078
- }
1079
- }
1080
-
1081
- // Execute all updates
1082
- if (updatePromises.length > 0) {
1083
- await Promise.all(updatePromises);
1084
- }
1085
-
1086
- // Optimize table to apply changes immediately
1087
- await this.client.command({
1088
- query: `OPTIMIZE TABLE ${TABLE_MESSAGES} FINAL`,
1089
- clickhouse_settings: {
1090
- date_time_input_format: 'best_effort',
1091
- use_client_time_zone: 1,
1092
- output_format_json_quote_64bit_integers: 0,
1093
- },
1094
- });
1095
-
1096
- // Verify updates were applied and retry if needed
1097
- for (const existingMessage of parsedExistingMessages) {
1098
- const updatePayload = messages.find(m => m.id === existingMessage.id);
1099
- if (!updatePayload) continue;
1100
-
1101
- const { id, ...fieldsToUpdate } = updatePayload;
1102
- if (Object.keys(fieldsToUpdate).length === 0) continue;
1103
-
1104
- // Check if the update was actually applied
1105
- const verifyResult = await this.client.query({
1106
- query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
1107
- query_params: { messageId: id },
1108
- clickhouse_settings: {
1109
- date_time_input_format: 'best_effort',
1110
- date_time_output_format: 'iso',
1111
- use_client_time_zone: 1,
1112
- output_format_json_quote_64bit_integers: 0,
1113
- },
1114
- });
1115
-
1116
- const verifyRows = await verifyResult.json();
1117
- if (verifyRows.data.length > 0) {
1118
- const updatedMessage = transformRows<MastraMessageV2>(verifyRows.data)[0];
1119
-
1120
- if (updatedMessage) {
1121
- // Check if the update was applied correctly
1122
- let needsRetry = false;
1123
- for (const [key, value] of Object.entries(fieldsToUpdate)) {
1124
- if (key === 'content') {
1125
- // For content updates, check if the content was updated
1126
- const expectedContent = typeof value === 'string' ? value : JSON.stringify(value);
1127
- const actualContent =
1128
- typeof updatedMessage.content === 'string'
1129
- ? updatedMessage.content
1130
- : JSON.stringify(updatedMessage.content);
1131
- if (actualContent !== expectedContent) {
1132
- needsRetry = true;
1133
- break;
1134
- }
1135
- } else if (updatedMessage[key as keyof MastraMessageV2] !== value) {
1136
- needsRetry = true;
1137
- break;
1138
- }
1139
- }
1140
-
1141
- if (needsRetry) {
1142
- console.log('Update not applied correctly, retrying with DELETE + INSERT for message:', id);
1143
-
1144
- // Use DELETE + INSERT as fallback
1145
- await this.client.command({
1146
- query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
1147
- query_params: { messageId: id },
1148
- clickhouse_settings: {
1149
- date_time_input_format: 'best_effort',
1150
- use_client_time_zone: 1,
1151
- output_format_json_quote_64bit_integers: 0,
1152
- },
1153
- });
1154
-
1155
- // Reconstruct the updated content if needed
1156
- let updatedContent = existingMessage.content || {};
1157
- if (fieldsToUpdate.content) {
1158
- const existingContent = existingMessage.content || {};
1159
- const existingMetadata = existingContent.metadata || {};
1160
- const updateMetadata = fieldsToUpdate.content.metadata || {};
1161
-
1162
- updatedContent = {
1163
- ...existingContent,
1164
- ...fieldsToUpdate.content,
1165
- metadata: {
1166
- ...existingMetadata,
1167
- ...updateMetadata,
1168
- },
1169
- };
1170
- }
1171
-
1172
- const updatedMessageData = {
1173
- ...existingMessage,
1174
- ...fieldsToUpdate,
1175
- content: updatedContent,
1176
- };
1177
-
1178
- await this.client.insert({
1179
- table: TABLE_MESSAGES,
1180
- format: 'JSONEachRow',
1181
- values: [
1182
- {
1183
- id: updatedMessageData.id,
1184
- thread_id: updatedMessageData.threadId,
1185
- resourceId: updatedMessageData.resourceId,
1186
- content:
1187
- typeof updatedMessageData.content === 'string'
1188
- ? updatedMessageData.content
1189
- : JSON.stringify(updatedMessageData.content),
1190
- createdAt: updatedMessageData.createdAt.toISOString(),
1191
- role: updatedMessageData.role,
1192
- type: updatedMessageData.type || 'v2',
1193
- },
1194
- ],
1195
- clickhouse_settings: {
1196
- date_time_input_format: 'best_effort',
1197
- use_client_time_zone: 1,
1198
- output_format_json_quote_64bit_integers: 0,
1199
- },
1200
- });
1201
- }
1202
- }
1203
- }
1204
- }
1205
-
1206
- // Update thread timestamps with a small delay to ensure timestamp difference
1207
- if (threadIdsToUpdate.size > 0) {
1208
- // Add a small delay to ensure timestamp difference
1209
- await new Promise(resolve => setTimeout(resolve, 10));
1210
-
1211
- const now = new Date().toISOString().replace('Z', '');
1212
-
1213
- // Get existing threads to preserve their data
1214
- const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async threadId => {
1215
- // Get existing thread data
1216
- const threadResult = await this.client.query({
1217
- query: `SELECT id, resourceId, title, metadata, createdAt FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
1218
- query_params: { threadId },
1219
- clickhouse_settings: {
1220
- date_time_input_format: 'best_effort',
1221
- date_time_output_format: 'iso',
1222
- use_client_time_zone: 1,
1223
- output_format_json_quote_64bit_integers: 0,
1224
- },
1225
- });
1226
-
1227
- const threadRows = await threadResult.json();
1228
- if (threadRows.data.length > 0) {
1229
- const existingThread = threadRows.data[0] as any;
1230
-
1231
- // Delete existing thread
1232
- await this.client.command({
1233
- query: `DELETE FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
1234
- query_params: { threadId },
1235
- clickhouse_settings: {
1236
- date_time_input_format: 'best_effort',
1237
- use_client_time_zone: 1,
1238
- output_format_json_quote_64bit_integers: 0,
1239
- },
1240
- });
1241
-
1242
- // Insert updated thread with new timestamp
1243
- await this.client.insert({
1244
- table: TABLE_THREADS,
1245
- format: 'JSONEachRow',
1246
- values: [
1247
- {
1248
- id: existingThread.id,
1249
- resourceId: existingThread.resourceId,
1250
- title: existingThread.title,
1251
- metadata: existingThread.metadata,
1252
- createdAt: existingThread.createdAt,
1253
- updatedAt: now,
1254
- },
1255
- ],
1256
- clickhouse_settings: {
1257
- date_time_input_format: 'best_effort',
1258
- use_client_time_zone: 1,
1259
- output_format_json_quote_64bit_integers: 0,
1260
- },
1261
- });
1262
- }
1263
- });
1264
-
1265
- await Promise.all(threadUpdatePromises);
1266
- }
1267
-
1268
- // Re-fetch to return the fully updated messages
1269
- const updatedMessages: MastraMessageV2[] = [];
1270
- for (const messageId of messageIds) {
1271
- const updatedResult = await this.client.query({
1272
- query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
1273
- query_params: { messageId },
1274
- clickhouse_settings: {
1275
- date_time_input_format: 'best_effort',
1276
- date_time_output_format: 'iso',
1277
- use_client_time_zone: 1,
1278
- output_format_json_quote_64bit_integers: 0,
1279
- },
1280
- });
1281
- const updatedRows = await updatedResult.json();
1282
- if (updatedRows.data.length > 0) {
1283
- const message = transformRows<MastraMessageV2>(updatedRows.data)[0];
1284
- if (message) {
1285
- updatedMessages.push(message);
1286
- }
1287
- }
1288
- }
1289
-
1290
- // Parse content back to objects
1291
- return updatedMessages.map(message => {
1292
- if (typeof message.content === 'string') {
1293
- try {
1294
- message.content = JSON.parse(message.content);
1295
- } catch {
1296
- // ignore if not valid json
1297
- }
1298
- }
1299
- return message;
1300
- });
1301
- } catch (error) {
1302
- throw new MastraError(
1303
- {
1304
- id: 'CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED',
1305
- domain: ErrorDomain.STORAGE,
1306
- category: ErrorCategory.THIRD_PARTY,
1307
- details: { messageIds: messages.map(m => m.id).join(',') },
1308
- },
1309
- error,
1310
- );
1311
- }
1312
- }
1313
-
1314
- async getResourceById({ resourceId }: { resourceId: string }): Promise<StorageResourceType | null> {
1315
- try {
1316
- const result = await this.client.query({
1317
- query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${TABLE_RESOURCES} WHERE id = {resourceId:String}`,
1318
- query_params: { resourceId },
1319
- clickhouse_settings: {
1320
- date_time_input_format: 'best_effort',
1321
- date_time_output_format: 'iso',
1322
- use_client_time_zone: 1,
1323
- output_format_json_quote_64bit_integers: 0,
1324
- },
1325
- });
1326
-
1327
- const rows = await result.json();
1328
- if (rows.data.length === 0) {
1329
- return null;
1330
- }
1331
-
1332
- const resource = rows.data[0] as any;
1333
- return {
1334
- id: resource.id,
1335
- workingMemory:
1336
- resource.workingMemory && typeof resource.workingMemory === 'object'
1337
- ? JSON.stringify(resource.workingMemory)
1338
- : resource.workingMemory,
1339
- metadata:
1340
- resource.metadata && typeof resource.metadata === 'string'
1341
- ? JSON.parse(resource.metadata)
1342
- : resource.metadata,
1343
- createdAt: new Date(resource.createdAt),
1344
- updatedAt: new Date(resource.updatedAt),
1345
- };
1346
- } catch (error) {
1347
- throw new MastraError(
1348
- {
1349
- id: 'CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED',
1350
- domain: ErrorDomain.STORAGE,
1351
- category: ErrorCategory.THIRD_PARTY,
1352
- details: { resourceId },
1353
- },
1354
- error,
1355
- );
1356
- }
1357
- }
1358
-
1359
- async saveResource({ resource }: { resource: StorageResourceType }): Promise<StorageResourceType> {
1360
- try {
1361
- await this.client.insert({
1362
- table: TABLE_RESOURCES,
1363
- format: 'JSONEachRow',
1364
- values: [
1365
- {
1366
- id: resource.id,
1367
- workingMemory: resource.workingMemory,
1368
- metadata: JSON.stringify(resource.metadata),
1369
- createdAt: resource.createdAt.toISOString(),
1370
- updatedAt: resource.updatedAt.toISOString(),
1371
- },
1372
- ],
1373
- clickhouse_settings: {
1374
- date_time_input_format: 'best_effort',
1375
- use_client_time_zone: 1,
1376
- output_format_json_quote_64bit_integers: 0,
1377
- },
1378
- });
1379
-
1380
- return resource;
1381
- } catch (error) {
1382
- throw new MastraError(
1383
- {
1384
- id: 'CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED',
1385
- domain: ErrorDomain.STORAGE,
1386
- category: ErrorCategory.THIRD_PARTY,
1387
- details: { resourceId: resource.id },
1388
- },
1389
- error,
1390
- );
1391
- }
1392
- }
1393
-
1394
- async updateResource({
1395
- resourceId,
1396
- workingMemory,
1397
- metadata,
1398
- }: {
1399
- resourceId: string;
1400
- workingMemory?: string;
1401
- metadata?: Record<string, unknown>;
1402
- }): Promise<StorageResourceType> {
1403
- try {
1404
- const existingResource = await this.getResourceById({ resourceId });
1405
-
1406
- if (!existingResource) {
1407
- // Create new resource if it doesn't exist
1408
- const newResource: StorageResourceType = {
1409
- id: resourceId,
1410
- workingMemory,
1411
- metadata: metadata || {},
1412
- createdAt: new Date(),
1413
- updatedAt: new Date(),
1414
- };
1415
- return this.saveResource({ resource: newResource });
1416
- }
1417
-
1418
- const updatedResource = {
1419
- ...existingResource,
1420
- workingMemory: workingMemory !== undefined ? workingMemory : existingResource.workingMemory,
1421
- metadata: {
1422
- ...existingResource.metadata,
1423
- ...metadata,
1424
- },
1425
- updatedAt: new Date(),
1426
- };
1427
-
1428
- // Use ALTER TABLE UPDATE for ClickHouse
1429
- const updateQuery = `
1430
- ALTER TABLE ${TABLE_RESOURCES}
1431
- UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
1432
- WHERE id = {resourceId:String}
1433
- `;
1434
-
1435
- await this.client.command({
1436
- query: updateQuery,
1437
- query_params: {
1438
- workingMemory: updatedResource.workingMemory,
1439
- metadata: JSON.stringify(updatedResource.metadata),
1440
- updatedAt: updatedResource.updatedAt.toISOString().replace('Z', ''),
1441
- resourceId,
1442
- },
1443
- clickhouse_settings: {
1444
- date_time_input_format: 'best_effort',
1445
- use_client_time_zone: 1,
1446
- output_format_json_quote_64bit_integers: 0,
1447
- },
1448
- });
1449
-
1450
- // Optimize table to apply changes
1451
- await this.client.command({
1452
- query: `OPTIMIZE TABLE ${TABLE_RESOURCES} FINAL`,
1453
- clickhouse_settings: {
1454
- date_time_input_format: 'best_effort',
1455
- use_client_time_zone: 1,
1456
- output_format_json_quote_64bit_integers: 0,
1457
- },
1458
- });
1459
-
1460
- return updatedResource;
1461
- } catch (error) {
1462
- throw new MastraError(
1463
- {
1464
- id: 'CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED',
1465
- domain: ErrorDomain.STORAGE,
1466
- category: ErrorCategory.THIRD_PARTY,
1467
- details: { resourceId },
1468
- },
1469
- error,
1470
- );
1471
- }
1472
- }
1473
- }