@mastra/clickhouse 0.12.0 → 0.12.1-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,17 +1,6 @@
1
- import { randomUUID } from 'crypto';
2
- import {
3
- createSampleMessageV1,
4
- createSampleThread,
5
- createSampleWorkflowSnapshot,
6
- checkWorkflowSnapshot,
7
- createSampleMessageV2,
8
- } from '@internal/storage-test-utils';
9
- import type { MastraMessageV1, StorageColumn, WorkflowRunState } from '@mastra/core';
10
- import type { TABLE_NAMES } from '@mastra/core/storage';
11
- import { TABLE_THREADS, TABLE_MESSAGES, TABLE_WORKFLOW_SNAPSHOT } from '@mastra/core/storage';
12
- import { describe, it, expect, beforeAll, beforeEach, afterAll, vi, afterEach } from 'vitest';
13
-
14
- import { ClickhouseStore, TABLE_ENGINES } from '.';
1
+ import { createTestSuite } from '@internal/storage-test-utils';
2
+ import { vi } from 'vitest';
3
+ import { ClickhouseStore } from '.';
15
4
  import type { ClickhouseConfig } from '.';
16
5
 
17
6
  vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 });
@@ -20,1083 +9,18 @@ const TEST_CONFIG: ClickhouseConfig = {
20
9
  url: process.env.CLICKHOUSE_URL || 'http://localhost:8123',
21
10
  username: process.env.CLICKHOUSE_USERNAME || 'default',
22
11
  password: process.env.CLICKHOUSE_PASSWORD || 'password',
23
- ttl: {
24
- mastra_traces: {
25
- row: { interval: 10, unit: 'SECOND' },
26
- },
27
- mastra_evals: {
28
- columns: {
29
- result: { interval: 10, unit: 'SECOND' },
30
- },
31
- },
32
- },
12
+ // ttl: {
13
+ // mastra_traces: {
14
+ // row: { interval: 600, unit: 'SECOND' },
15
+ // },
16
+ // mastra_evals: {
17
+ // columns: {
18
+ // result: { interval: 10, unit: 'SECOND' },
19
+ // },
20
+ // },
21
+ // },
33
22
  };
34
23
 
35
- const createSampleTrace = () => ({
36
- id: `trace-${randomUUID()}`,
37
- name: 'Test Trace',
38
- createdAt: new Date(),
39
- updatedAt: new Date(),
40
- metadata: { key: 'value' },
41
- });
42
-
43
- const createSampleEval = () => ({
44
- agent_name: 'test-agent',
45
- run_id: 'test-run-1',
46
- result: '{ "score": 1 }',
47
- createdAt: new Date(),
48
- });
49
-
50
- describe('ClickhouseStore', () => {
51
- let store: ClickhouseStore;
52
-
53
- beforeAll(async () => {
54
- store = new ClickhouseStore(TEST_CONFIG);
55
- await store.init();
56
- });
57
-
58
- beforeEach(async () => {
59
- // Clear tables before each test
60
- await store.clearTable({ tableName: TABLE_THREADS });
61
- await store.clearTable({ tableName: TABLE_MESSAGES });
62
- await store.clearTable({ tableName: TABLE_WORKFLOW_SNAPSHOT });
63
- });
64
-
65
- describe('Thread Operations', () => {
66
- it('should create and retrieve a thread', async () => {
67
- const thread = createSampleThread();
68
-
69
- // Save thread
70
- const savedThread = await store.saveThread({ thread });
71
- expect(savedThread).toEqual(thread);
72
-
73
- // Retrieve thread
74
- const retrievedThread = await store.getThreadById({ threadId: thread.id });
75
- expect(retrievedThread?.title).toEqual(thread.title);
76
- }, 10e3);
77
-
78
- it('should return null for non-existent thread', async () => {
79
- const result = await store.getThreadById({ threadId: 'non-existent' });
80
- expect(result).toBeNull();
81
- }, 10e3);
82
-
83
- it('should get threads by resource ID', async () => {
84
- const thread1 = createSampleThread();
85
- const thread2 = { ...createSampleThread(), resourceId: thread1.resourceId };
86
-
87
- await store.saveThread({ thread: thread1 });
88
- await store.saveThread({ thread: thread2 });
89
-
90
- const threads = await store.getThreadsByResourceId({ resourceId: thread1.resourceId });
91
- expect(threads).toHaveLength(2);
92
- expect(threads.map(t => t.id)).toEqual(expect.arrayContaining([thread1.id, thread2.id]));
93
- }, 10e3);
94
-
95
- it('should update thread title and metadata', async () => {
96
- const thread = createSampleThread();
97
- await store.saveThread({ thread });
98
-
99
- const newMetadata = { newKey: 'newValue' };
100
- const updatedThread = await store.updateThread({
101
- id: thread.id,
102
- title: 'Updated Title',
103
- metadata: newMetadata,
104
- });
105
-
106
- expect(updatedThread.title).toBe('Updated Title');
107
- expect(updatedThread.metadata).toEqual({
108
- ...thread.metadata,
109
- ...newMetadata,
110
- });
111
-
112
- // Verify persistence
113
- const retrievedThread = await store.getThreadById({ threadId: thread.id });
114
- expect(retrievedThread).toEqual(updatedThread);
115
- }, 10e3);
116
-
117
- it('should delete thread and its messages', async () => {
118
- const thread = createSampleThread();
119
- await store.saveThread({ thread });
120
-
121
- // Add some messages
122
- const messages = [
123
- createSampleMessageV1({ threadId: thread.id, resourceId: 'clickhouse-test' }),
124
- createSampleMessageV1({ threadId: thread.id, resourceId: 'clickhouse-test' }),
125
- ];
126
- await store.saveMessages({ messages });
127
-
128
- await store.deleteThread({ threadId: thread.id });
129
-
130
- const retrievedThread = await store.getThreadById({ threadId: thread.id });
131
- expect(retrievedThread).toBeNull();
132
-
133
- // Verify messages were also deleted
134
- const retrievedMessages = await store.getMessages({ threadId: thread.id });
135
- expect(retrievedMessages).toHaveLength(0);
136
- }, 10e3);
137
-
138
- it('should update thread updatedAt when a message is saved to it', async () => {
139
- const thread = createSampleThread();
140
- await store.saveThread({ thread });
141
-
142
- // Get the initial thread to capture the original updatedAt
143
- const initialThread = await store.getThreadById({ threadId: thread.id });
144
- expect(initialThread).toBeDefined();
145
- const originalUpdatedAt = initialThread!.updatedAt;
146
-
147
- // Wait a small amount to ensure different timestamp
148
- await new Promise(resolve => setTimeout(resolve, 10));
149
-
150
- // Create and save a message to the thread
151
- const message = createSampleMessageV1({ threadId: thread.id });
152
- await store.saveMessages({ messages: [message] });
153
-
154
- // Retrieve the thread again and check that updatedAt was updated
155
- const updatedThread = await store.getThreadById({ threadId: thread.id });
156
- expect(updatedThread).toBeDefined();
157
- expect(updatedThread!.updatedAt.getTime()).toBeGreaterThan(originalUpdatedAt.getTime());
158
- }, 10e3);
159
- });
160
-
161
- describe('Message Operations', () => {
162
- it('should save and retrieve messages', async () => {
163
- const thread = createSampleThread();
164
- await store.saveThread({ thread });
165
-
166
- const messages = [
167
- createSampleMessageV1({
168
- threadId: thread.id,
169
- createdAt: new Date(Date.now() - 1000 * 60 * 60 * 24),
170
- resourceId: 'clickhouse-test',
171
- }),
172
- createSampleMessageV1({ threadId: thread.id, resourceId: 'clickhouse-test' }),
173
- ];
174
-
175
- // Save messages
176
- const savedMessages = await store.saveMessages({ messages });
177
- expect(savedMessages).toEqual(messages);
178
-
179
- // Retrieve messages
180
- const retrievedMessages = await store.getMessages({ threadId: thread.id });
181
- expect(retrievedMessages).toHaveLength(2);
182
- const checkMessages = messages.map(m => {
183
- const { resourceId, ...rest } = m;
184
- return rest;
185
- });
186
- expect(retrievedMessages).toEqual(expect.arrayContaining(checkMessages));
187
- }, 10e3);
188
-
189
- it('should handle empty message array', async () => {
190
- const result = await store.saveMessages({ messages: [] });
191
- expect(result).toEqual([]);
192
- }, 10e3);
193
-
194
- it('should maintain message order', async () => {
195
- const thread = createSampleThread();
196
- await store.saveThread({ thread });
197
-
198
- const messages = [
199
- {
200
- ...createSampleMessageV1({
201
- threadId: thread.id,
202
- createdAt: new Date(Date.now() - 1000 * 3),
203
- content: 'First',
204
- resourceId: 'clickhouse-test',
205
- }),
206
- role: 'user',
207
- },
208
- {
209
- ...createSampleMessageV1({
210
- threadId: thread.id,
211
- createdAt: new Date(Date.now() - 1000 * 2),
212
- content: 'Second',
213
- resourceId: 'clickhouse-test',
214
- }),
215
- role: 'assistant',
216
- },
217
- {
218
- ...createSampleMessageV1({
219
- threadId: thread.id,
220
- createdAt: new Date(Date.now() - 1000 * 1),
221
- content: 'Third',
222
- resourceId: 'clickhouse-test',
223
- }),
224
- role: 'user',
225
- },
226
- ] as MastraMessageV1[];
227
-
228
- await store.saveMessages({ messages });
229
-
230
- const retrievedMessages = await store.getMessages({ threadId: thread.id });
231
- expect(retrievedMessages).toHaveLength(3);
232
-
233
- // Verify order is maintained
234
- retrievedMessages.forEach((msg, idx) => {
235
- // @ts-expect-error
236
- expect(msg.content[0].text).toBe(messages[idx].content[0].text);
237
- });
238
- }, 10e3);
239
-
240
- it('should upsert messages: duplicate id+threadId results in update, not duplicate row', async () => {
241
- const thread = await createSampleThread({ resourceId: 'clickhouse-test' });
242
- await store.saveThread({ thread });
243
- const baseMessage = createSampleMessageV2({
244
- threadId: thread.id,
245
- createdAt: new Date(),
246
- content: { content: 'Original' },
247
- resourceId: 'clickhouse-test',
248
- });
249
-
250
- // Insert the message for the first time
251
- await store.saveMessages({ messages: [baseMessage], format: 'v2' });
252
-
253
- // Insert again with the same id and threadId but different content
254
- const updatedMessage = {
255
- ...createSampleMessageV2({
256
- threadId: thread.id,
257
- createdAt: new Date(),
258
- content: { content: 'Updated' },
259
- resourceId: 'clickhouse-test',
260
- }),
261
- id: baseMessage.id,
262
- };
263
- await store.saveMessages({ messages: [updatedMessage], format: 'v2' });
264
- await new Promise(resolve => setTimeout(resolve, 500));
265
-
266
- // Retrieve messages for the thread
267
- const retrievedMessages = await store.getMessages({ threadId: thread.id, format: 'v2' });
268
-
269
- // Only one message should exist for that id+threadId
270
- expect(retrievedMessages.filter(m => m.id === baseMessage.id)).toHaveLength(1);
271
-
272
- // The content should be the updated one
273
- expect(retrievedMessages.find(m => m.id === baseMessage.id)?.content.content).toBe('Updated');
274
- }, 10e3);
275
-
276
- it('should upsert messages: duplicate id and different threadid', async () => {
277
- const thread1 = await createSampleThread();
278
- const thread2 = await createSampleThread();
279
- await store.saveThread({ thread: thread1 });
280
- await store.saveThread({ thread: thread2 });
281
-
282
- const message = createSampleMessageV2({
283
- threadId: thread1.id,
284
- createdAt: new Date(),
285
- content: { content: 'Thread1 Content' },
286
- resourceId: thread1.resourceId,
287
- });
288
-
289
- // Insert message into thread1
290
- await store.saveMessages({ messages: [message], format: 'v2' });
291
-
292
- // Attempt to insert a message with the same id but different threadId
293
- const conflictingMessage = {
294
- ...createSampleMessageV2({
295
- threadId: thread2.id,
296
- createdAt: new Date(),
297
- content: { content: 'Thread2 Content' },
298
- resourceId: thread2.resourceId,
299
- }),
300
- id: message.id,
301
- };
302
-
303
- // Save should also save the message to the new thread
304
- await store.saveMessages({ messages: [conflictingMessage], format: 'v2' });
305
-
306
- // Retrieve messages for both threads
307
- const thread1Messages = await store.getMessages({ threadId: thread1.id, format: 'v2' });
308
- const thread2Messages = await store.getMessages({ threadId: thread2.id, format: 'v2' });
309
-
310
- // Thread 1 should have the message with that id
311
- expect(thread1Messages.find(m => m.id === message.id)?.content.content).toBe('Thread1 Content');
312
-
313
- // Thread 2 should have the message with that id
314
- expect(thread2Messages.find(m => m.id === message.id)?.content.content).toBe('Thread2 Content');
315
- }, 10e3);
316
-
317
- // it('should retrieve messages w/ next/prev messages by message id + resource id', async () => {
318
- // const messages: MastraMessageV2[] = [
319
- // createSampleMessageV2({ threadId: 'thread-one', content: 'First', resourceId: 'cross-thread-resource' }),
320
- // createSampleMessageV2({ threadId: 'thread-one', content: 'Second', resourceId: 'cross-thread-resource' }),
321
- // createSampleMessageV2({ threadId: 'thread-one', content: 'Third', resourceId: 'cross-thread-resource' }),
322
-
323
- // createSampleMessageV2({ threadId: 'thread-two', content: 'Fourth', resourceId: 'cross-thread-resource' }),
324
- // createSampleMessageV2({ threadId: 'thread-two', content: 'Fifth', resourceId: 'cross-thread-resource' }),
325
- // createSampleMessageV2({ threadId: 'thread-two', content: 'Sixth', resourceId: 'cross-thread-resource' }),
326
-
327
- // createSampleMessageV2({ threadId: 'thread-three', content: 'Seventh', resourceId: 'other-resource' }),
328
- // createSampleMessageV2({ threadId: 'thread-three', content: 'Eighth', resourceId: 'other-resource' }),
329
- // ];
330
-
331
- // await store.saveMessages({ messages: messages, format: 'v2' });
332
-
333
- // const retrievedMessages = await store.getMessages({ threadId: 'thread-one', format: 'v2' });
334
- // expect(retrievedMessages).toHaveLength(3);
335
- // expect(retrievedMessages.map((m: any) => m.content.parts[0].text)).toEqual(['First', 'Second', 'Third']);
336
-
337
- // const retrievedMessages2 = await store.getMessages({ threadId: 'thread-two', format: 'v2' });
338
- // expect(retrievedMessages2).toHaveLength(3);
339
- // expect(retrievedMessages2.map((m: any) => m.content.parts[0].text)).toEqual(['Fourth', 'Fifth', 'Sixth']);
340
-
341
- // const retrievedMessages3 = await store.getMessages({ threadId: 'thread-three', format: 'v2' });
342
- // expect(retrievedMessages3).toHaveLength(2);
343
- // expect(retrievedMessages3.map((m: any) => m.content.parts[0].text)).toEqual(['Seventh', 'Eighth']);
344
-
345
- // const crossThreadMessages = await store.getMessages({
346
- // threadId: 'thread-doesnt-exist',
347
- // resourceId: 'cross-thread-resource',
348
- // format: 'v2',
349
- // selectBy: {
350
- // last: 0,
351
- // include: [
352
- // {
353
- // id: messages[1].id,
354
- // withNextMessages: 2,
355
- // withPreviousMessages: 2,
356
- // },
357
- // {
358
- // id: messages[4].id,
359
- // withPreviousMessages: 2,
360
- // withNextMessages: 2,
361
- // },
362
- // ],
363
- // },
364
- // });
365
-
366
- // expect(crossThreadMessages).toHaveLength(6);
367
- // expect(crossThreadMessages.filter(m => m.threadId === `thread-one`)).toHaveLength(3);
368
- // expect(crossThreadMessages.filter(m => m.threadId === `thread-two`)).toHaveLength(3);
369
-
370
- // const crossThreadMessages2 = await store.getMessages({
371
- // threadId: 'thread-one',
372
- // resourceId: 'cross-thread-resource',
373
- // format: 'v2',
374
- // selectBy: {
375
- // last: 0,
376
- // include: [
377
- // {
378
- // id: messages[4].id,
379
- // withPreviousMessages: 1,
380
- // withNextMessages: 30,
381
- // },
382
- // ],
383
- // },
384
- // });
385
-
386
- // expect(crossThreadMessages2).toHaveLength(3);
387
- // expect(crossThreadMessages2.filter(m => m.threadId === `thread-one`)).toHaveLength(0);
388
- // expect(crossThreadMessages2.filter(m => m.threadId === `thread-two`)).toHaveLength(3);
389
-
390
- // const crossThreadMessages3 = await store.getMessages({
391
- // threadId: 'thread-two',
392
- // resourceId: 'cross-thread-resource',
393
- // format: 'v2',
394
- // selectBy: {
395
- // last: 0,
396
- // include: [
397
- // {
398
- // id: messages[1].id,
399
- // withNextMessages: 1,
400
- // withPreviousMessages: 1,
401
- // },
402
- // ],
403
- // },
404
- // });
405
-
406
- // expect(crossThreadMessages3).toHaveLength(3);
407
- // expect(crossThreadMessages3.filter(m => m.threadId === `thread-one`)).toHaveLength(3);
408
- // expect(crossThreadMessages3.filter(m => m.threadId === `thread-two`)).toHaveLength(0);
409
- // });
410
-
411
- // it('should rollback on error during message save', async () => {
412
- // const thread = createSampleThread();
413
- // await store.saveThread({ thread });
414
-
415
- // const messages = [
416
- // createSampleMessageV1({ threadId: thread.id }),
417
- // { ...createSampleMessageV1({ threadId: thread.id }), id: null }, // This will cause an error
418
- // ];
419
-
420
- // await expect(store.saveMessages({ messages })).rejects.toThrow();
421
-
422
- // // Verify no messages were saved
423
- // const savedMessages = await store.getMessages({ threadId: thread.id });
424
- // expect(savedMessages).toHaveLength(0);
425
- // });
426
- });
427
-
428
- describe('Traces and TTL', () => {
429
- it('should create and retrieve a trace, but not when row level ttl expires', async () => {
430
- const trace = createSampleTrace();
431
- await store.batchInsert({
432
- tableName: 'mastra_traces',
433
- records: [trace],
434
- });
435
- let traces = await store.getTraces({
436
- page: 0,
437
- perPage: 10,
438
- });
439
-
440
- expect(traces).toHaveLength(1);
441
- expect(traces[0]!.id).toBe(trace.id);
442
-
443
- await new Promise(resolve => setTimeout(resolve, 10e3));
444
- await store.optimizeTable({ tableName: 'mastra_traces' });
445
-
446
- traces = await store.getTraces({
447
- page: 0,
448
- perPage: 10,
449
- });
450
-
451
- expect(traces).toHaveLength(0);
452
- }, 60e3);
453
-
454
- // NOTE: unable to clear column level TTLs for the test case nicely, but it does seem to get applied correctly
455
- it.skip('should create and retrieve a trace, but not expired columns when column level ttl expires', async () => {
456
- await store.clearTable({ tableName: 'mastra_evals' });
457
- const ev = createSampleEval();
458
- await store.batchInsert({
459
- tableName: 'mastra_evals',
460
- records: [ev],
461
- });
462
- let evals = await store.getEvalsByAgentName('test-agent');
463
- console.log(evals);
464
-
465
- expect(evals).toHaveLength(1);
466
- expect(evals[0]!.agentName).toBe('test-agent');
467
- expect(evals[0]!.runId).toBe('test-run-1');
468
-
469
- await new Promise(resolve => setTimeout(resolve, 12e3));
470
- await store.materializeTtl({ tableName: 'mastra_evals' });
471
- await store.optimizeTable({ tableName: 'mastra_evals' });
472
-
473
- evals = await store.getEvalsByAgentName('test-agent');
474
-
475
- expect(evals).toHaveLength(1);
476
- expect(evals[0]!.agentName).toBe('test-agent');
477
- expect(evals[0]!.runId).toBeNull();
478
- }, 60e3);
479
- });
480
-
481
- describe('Edge Cases and Error Handling', () => {
482
- it('should handle large metadata objects', async () => {
483
- const thread = createSampleThread();
484
- const largeMetadata = {
485
- ...thread.metadata,
486
- largeArray: Array.from({ length: 1000 }, (_, i) => ({ index: i, data: 'test'.repeat(100) })),
487
- };
488
-
489
- const threadWithLargeMetadata = {
490
- ...thread,
491
- metadata: largeMetadata,
492
- };
493
-
494
- await store.saveThread({ thread: threadWithLargeMetadata });
495
- const retrieved = await store.getThreadById({ threadId: thread.id });
496
-
497
- expect(retrieved?.metadata).toEqual(largeMetadata);
498
- }, 10e3);
499
-
500
- it('should handle special characters in thread titles', async () => {
501
- const thread = {
502
- ...createSampleThread(),
503
- title: 'Special \'quotes\' and "double quotes" and emoji 🎉',
504
- };
505
-
506
- await store.saveThread({ thread });
507
- const retrieved = await store.getThreadById({ threadId: thread.id });
508
-
509
- expect(retrieved?.title).toBe(thread.title);
510
- }, 10e3);
511
-
512
- it('should handle concurrent thread updates', async () => {
513
- const thread = createSampleThread();
514
- await store.saveThread({ thread });
515
-
516
- // Perform multiple updates concurrently
517
- const updates = Array.from({ length: 5 }, (_, i) =>
518
- store.updateThread({
519
- id: thread.id,
520
- title: `Update ${i}`,
521
- metadata: { update: i },
522
- }),
523
- );
524
-
525
- await expect(Promise.all(updates)).resolves.toBeDefined();
526
-
527
- // Verify final state
528
- const finalThread = await store.getThreadById({ threadId: thread.id });
529
- expect(finalThread).toBeDefined();
530
- }, 10e3);
531
- });
532
-
533
- describe('Workflow Snapshots', () => {
534
- it('should persist and load workflow snapshots', async () => {
535
- const workflowName = 'test-workflow';
536
- const runId = `run-${randomUUID()}`;
537
- const snapshot = {
538
- status: 'running',
539
- context: {
540
- input: { type: 'manual' },
541
- },
542
- value: {},
543
- activePaths: [],
544
- suspendedPaths: {},
545
- runId,
546
- timestamp: new Date().getTime(),
547
- } as unknown as WorkflowRunState;
548
-
549
- await store.persistWorkflowSnapshot({
550
- workflowName,
551
- runId,
552
- snapshot,
553
- });
554
-
555
- const loadedSnapshot = await store.loadWorkflowSnapshot({
556
- workflowName,
557
- runId,
558
- });
559
-
560
- expect(loadedSnapshot).toEqual(snapshot);
561
- }, 10e3);
562
-
563
- it('should return null for non-existent workflow snapshot', async () => {
564
- const result = await store.loadWorkflowSnapshot({
565
- workflowName: 'non-existent',
566
- runId: 'non-existent',
567
- });
568
-
569
- expect(result).toBeNull();
570
- }, 10e3);
571
-
572
- it('should update existing workflow snapshot', async () => {
573
- const workflowName = 'test-workflow';
574
- const runId = `run-${randomUUID()}`;
575
- const initialSnapshot = {
576
- status: 'running',
577
- context: {
578
- input: { type: 'manual' },
579
- },
580
- value: {},
581
- activePaths: [],
582
- suspendedPaths: {},
583
- runId,
584
- timestamp: new Date().getTime(),
585
- } as unknown as WorkflowRunState;
586
-
587
- await store.persistWorkflowSnapshot({
588
- workflowName,
589
- runId,
590
- snapshot: initialSnapshot,
591
- });
592
-
593
- const updatedSnapshot = {
594
- status: 'completed',
595
- context: {
596
- input: { type: 'manual' },
597
- 'step-1': { status: 'success', result: { data: 'test' } },
598
- },
599
- value: {},
600
- activePaths: [],
601
- suspendedPaths: {},
602
- runId,
603
- timestamp: new Date().getTime(),
604
- } as unknown as WorkflowRunState;
605
-
606
- await store.persistWorkflowSnapshot({
607
- workflowName,
608
- runId,
609
- snapshot: updatedSnapshot,
610
- });
611
-
612
- const loadedSnapshot = await store.loadWorkflowSnapshot({
613
- workflowName,
614
- runId,
615
- });
616
-
617
- expect(loadedSnapshot).toEqual(updatedSnapshot);
618
- }, 10e3);
619
-
620
- it('should handle complex workflow state', async () => {
621
- const workflowName = 'complex-workflow';
622
- const runId = `run-${randomUUID()}`;
623
- const complexSnapshot = {
624
- value: { currentState: 'running' },
625
- context: {
626
- 'step-1': {
627
- status: 'success',
628
- output: {
629
- nestedData: {
630
- array: [1, 2, 3],
631
- object: { key: 'value' },
632
- date: new Date().toISOString(),
633
- },
634
- },
635
- },
636
- 'step-2': {
637
- status: 'waiting',
638
- dependencies: ['step-3', 'step-4'],
639
- },
640
- input: {
641
- type: 'scheduled',
642
- metadata: {
643
- schedule: '0 0 * * *',
644
- timezone: 'UTC',
645
- },
646
- },
647
- },
648
- activePaths: [
649
- {
650
- stepPath: ['step-1'],
651
- stepId: 'step-1',
652
- status: 'success',
653
- },
654
- {
655
- stepPath: ['step-2'],
656
- stepId: 'step-2',
657
- status: 'waiting',
658
- },
659
- ],
660
- suspendedPaths: {},
661
- runId: runId,
662
- timestamp: Date.now(),
663
- } as unknown as WorkflowRunState;
664
-
665
- await store.persistWorkflowSnapshot({
666
- workflowName,
667
- runId,
668
- snapshot: complexSnapshot,
669
- });
670
-
671
- const loadedSnapshot = await store.loadWorkflowSnapshot({
672
- workflowName,
673
- runId,
674
- });
675
-
676
- expect(loadedSnapshot).toEqual(complexSnapshot);
677
- }, 10e3);
678
- });
679
-
680
- describe('getWorkflowRuns', () => {
681
- beforeEach(async () => {
682
- await store.clearTable({ tableName: TABLE_WORKFLOW_SNAPSHOT });
683
- });
684
- it('returns empty array when no workflows exist', async () => {
685
- const { runs, total } = await store.getWorkflowRuns();
686
- expect(runs).toEqual([]);
687
- expect(total).toBe(0);
688
- });
689
-
690
- it('returns all workflows by default', async () => {
691
- const workflowName1 = 'default_test_1';
692
- const workflowName2 = 'default_test_2';
693
-
694
- const { snapshot: workflow1, runId: runId1, stepId: stepId1 } = createSampleWorkflowSnapshot('success');
695
- const { snapshot: workflow2, runId: runId2, stepId: stepId2 } = createSampleWorkflowSnapshot('suspended');
696
-
697
- await store.persistWorkflowSnapshot({
698
- workflowName: workflowName1,
699
- runId: runId1,
700
- snapshot: workflow1,
701
- });
702
- await new Promise(resolve => setTimeout(resolve, 10)); // Small delay to ensure different timestamps
703
- await store.persistWorkflowSnapshot({
704
- workflowName: workflowName2,
705
- runId: runId2,
706
- snapshot: workflow2,
707
- });
708
-
709
- const { runs, total } = await store.getWorkflowRuns();
710
- expect(runs).toHaveLength(2);
711
- expect(total).toBe(2);
712
- expect(runs[0]!.workflowName).toBe(workflowName2); // Most recent first
713
- expect(runs[1]!.workflowName).toBe(workflowName1);
714
- const firstSnapshot = runs[0]!.snapshot;
715
- const secondSnapshot = runs[1]!.snapshot;
716
- checkWorkflowSnapshot(firstSnapshot, stepId2, 'suspended');
717
- checkWorkflowSnapshot(secondSnapshot, stepId1, 'success');
718
- });
719
-
720
- it('filters by workflow name', async () => {
721
- const workflowName1 = 'filter_test_1';
722
- const workflowName2 = 'filter_test_2';
723
-
724
- const { snapshot: workflow1, runId: runId1, stepId: stepId1 } = createSampleWorkflowSnapshot('success');
725
- const { snapshot: workflow2, runId: runId2 } = createSampleWorkflowSnapshot('failed');
726
-
727
- await store.persistWorkflowSnapshot({
728
- workflowName: workflowName1,
729
- runId: runId1,
730
- snapshot: workflow1,
731
- });
732
- await new Promise(resolve => setTimeout(resolve, 10)); // Small delay to ensure different timestamps
733
- await store.persistWorkflowSnapshot({
734
- workflowName: workflowName2,
735
- runId: runId2,
736
- snapshot: workflow2,
737
- });
738
-
739
- const { runs, total } = await store.getWorkflowRuns({
740
- workflowName: workflowName1,
741
- });
742
- expect(runs).toHaveLength(1);
743
- expect(total).toBe(1);
744
- expect(runs[0]!.workflowName).toBe(workflowName1);
745
- const snapshot = runs[0]!.snapshot;
746
- checkWorkflowSnapshot(snapshot, stepId1, 'success');
747
- });
748
-
749
- it('filters by date range', async () => {
750
- const now = new Date();
751
- const yesterday = new Date(now.getTime() - 24 * 60 * 60 * 1000);
752
- const twoDaysAgo = new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000);
753
- const workflowName1 = 'date_test_1';
754
- const workflowName2 = 'date_test_2';
755
- const workflowName3 = 'date_test_3';
756
-
757
- const { snapshot: workflow1, runId: runId1 } = createSampleWorkflowSnapshot('success');
758
- const { snapshot: workflow2, runId: runId2, stepId: stepId2 } = createSampleWorkflowSnapshot('suspended');
759
- const { snapshot: workflow3, runId: runId3, stepId: stepId3 } = createSampleWorkflowSnapshot('failed');
760
-
761
- await store.insert({
762
- tableName: TABLE_WORKFLOW_SNAPSHOT,
763
- record: {
764
- workflow_name: workflowName1,
765
- run_id: runId1,
766
- snapshot: workflow1,
767
- createdAt: twoDaysAgo,
768
- updatedAt: twoDaysAgo,
769
- },
770
- });
771
- await store.insert({
772
- tableName: TABLE_WORKFLOW_SNAPSHOT,
773
- record: {
774
- workflow_name: workflowName2,
775
- run_id: runId2,
776
- snapshot: workflow2,
777
- createdAt: yesterday,
778
- updatedAt: yesterday,
779
- },
780
- });
781
- await store.insert({
782
- tableName: TABLE_WORKFLOW_SNAPSHOT,
783
- record: {
784
- workflow_name: workflowName3,
785
- run_id: runId3,
786
- snapshot: workflow3,
787
- createdAt: now,
788
- updatedAt: now,
789
- },
790
- });
791
-
792
- const { runs } = await store.getWorkflowRuns({
793
- fromDate: yesterday,
794
- toDate: now,
795
- });
796
-
797
- expect(runs).toHaveLength(2);
798
- expect(runs[0]!.workflowName).toBe(workflowName3);
799
- expect(runs[1]!.workflowName).toBe(workflowName2);
800
- const firstSnapshot = runs[0]!.snapshot;
801
- const secondSnapshot = runs[1]!.snapshot;
802
- checkWorkflowSnapshot(firstSnapshot, stepId3, 'failed');
803
- checkWorkflowSnapshot(secondSnapshot, stepId2, 'suspended');
804
- });
805
-
806
- it('handles pagination', async () => {
807
- const workflowName1 = 'page_test_1';
808
- const workflowName2 = 'page_test_2';
809
- const workflowName3 = 'page_test_3';
810
-
811
- const { snapshot: workflow1, runId: runId1, stepId: stepId1 } = createSampleWorkflowSnapshot('success');
812
- const { snapshot: workflow2, runId: runId2, stepId: stepId2 } = createSampleWorkflowSnapshot('suspended');
813
- const { snapshot: workflow3, runId: runId3, stepId: stepId3 } = createSampleWorkflowSnapshot('failed');
814
-
815
- await store.persistWorkflowSnapshot({
816
- workflowName: workflowName1,
817
- runId: runId1,
818
- snapshot: workflow1,
819
- });
820
- await new Promise(resolve => setTimeout(resolve, 10)); // Small delay to ensure different timestamps
821
- await store.persistWorkflowSnapshot({
822
- workflowName: workflowName2,
823
- runId: runId2,
824
- snapshot: workflow2,
825
- });
826
- await new Promise(resolve => setTimeout(resolve, 10)); // Small delay to ensure different timestamps
827
- await store.persistWorkflowSnapshot({
828
- workflowName: workflowName3,
829
- runId: runId3,
830
- snapshot: workflow3,
831
- });
832
-
833
- // Get first page
834
- const page1 = await store.getWorkflowRuns({
835
- limit: 2,
836
- offset: 0,
837
- });
838
- expect(page1.runs).toHaveLength(2);
839
- expect(page1.total).toBe(3); // Total count of all records
840
- expect(page1.runs[0]!.workflowName).toBe(workflowName3);
841
- expect(page1.runs[1]!.workflowName).toBe(workflowName2);
842
- const firstSnapshot = page1.runs[0]!.snapshot;
843
- const secondSnapshot = page1.runs[1]!.snapshot;
844
- checkWorkflowSnapshot(firstSnapshot, stepId3, 'failed');
845
- checkWorkflowSnapshot(secondSnapshot, stepId2, 'suspended');
846
-
847
- // Get second page
848
- const page2 = await store.getWorkflowRuns({
849
- limit: 2,
850
- offset: 2,
851
- });
852
- expect(page2.runs).toHaveLength(1);
853
- expect(page2.total).toBe(3);
854
- expect(page2.runs[0]!.workflowName).toBe(workflowName1);
855
- const snapshot = page2.runs[0]!.snapshot!;
856
- checkWorkflowSnapshot(snapshot, stepId1, 'success');
857
- }, 10e3);
858
- });
859
- describe('getWorkflowRunById', () => {
860
- const workflowName = 'workflow-id-test';
861
- let runId: string;
862
- let stepId: string;
863
-
864
- beforeEach(async () => {
865
- // Insert a workflow run for positive test
866
- const sample = createSampleWorkflowSnapshot('success');
867
- runId = sample.runId;
868
- stepId = sample.stepId;
869
- await store.insert({
870
- tableName: TABLE_WORKFLOW_SNAPSHOT,
871
- record: {
872
- workflow_name: workflowName,
873
- run_id: runId,
874
- resourceId: 'resource-abc',
875
- snapshot: sample.snapshot,
876
- createdAt: new Date(),
877
- updatedAt: new Date(),
878
- },
879
- });
880
- });
881
-
882
- it('should retrieve a workflow run by ID', async () => {
883
- const found = await store.getWorkflowRunById({
884
- runId,
885
- workflowName,
886
- });
887
- expect(found).not.toBeNull();
888
- expect(found?.runId).toBe(runId);
889
- checkWorkflowSnapshot(found?.snapshot!, stepId, 'success');
890
- });
891
-
892
- it('should return null for non-existent workflow run ID', async () => {
893
- const notFound = await store.getWorkflowRunById({
894
- runId: 'non-existent-id',
895
- workflowName,
896
- });
897
- expect(notFound).toBeNull();
898
- });
899
- });
900
- describe('getWorkflowRuns with resourceId', () => {
901
- const workflowName = 'workflow-id-test';
902
- let resourceId: string;
903
- let runIds: string[] = [];
904
-
905
- beforeEach(async () => {
906
- // Insert multiple workflow runs for the same resourceId
907
- resourceId = 'resource-shared';
908
- for (const status of ['completed', 'running']) {
909
- const sample = createSampleWorkflowSnapshot(status as WorkflowRunState['context']['steps']['status']);
910
- runIds.push(sample.runId);
911
- await store.insert({
912
- tableName: TABLE_WORKFLOW_SNAPSHOT,
913
- record: {
914
- workflow_name: workflowName,
915
- run_id: sample.runId,
916
- resourceId,
917
- snapshot: sample.snapshot,
918
- createdAt: new Date(),
919
- updatedAt: new Date(),
920
- },
921
- });
922
- }
923
- // Insert a run with a different resourceId
924
- const other = createSampleWorkflowSnapshot('suspended');
925
- await store.insert({
926
- tableName: TABLE_WORKFLOW_SNAPSHOT,
927
- record: {
928
- workflow_name: workflowName,
929
- run_id: other.runId,
930
- resourceId: 'resource-other',
931
- snapshot: other.snapshot,
932
- createdAt: new Date(),
933
- updatedAt: new Date(),
934
- },
935
- });
936
- });
937
-
938
- it('should retrieve all workflow runs by resourceId', async () => {
939
- const { runs } = await store.getWorkflowRuns({
940
- resourceId,
941
- workflowName,
942
- });
943
- expect(Array.isArray(runs)).toBe(true);
944
- expect(runs.length).toBeGreaterThanOrEqual(2);
945
- for (const run of runs) {
946
- expect(run.resourceId).toBe(resourceId);
947
- }
948
- });
949
-
950
- it('should return an empty array if no workflow runs match resourceId', async () => {
951
- const { runs } = await store.getWorkflowRuns({
952
- resourceId: 'non-existent-resource',
953
- workflowName,
954
- });
955
- expect(Array.isArray(runs)).toBe(true);
956
- expect(runs.length).toBe(0);
957
- });
958
- });
959
-
960
- describe('hasColumn', () => {
961
- const tempTable = 'temp_test_table';
962
-
963
- beforeEach(async () => {
964
- // Always try to drop the table before each test, ignore errors if it doesn't exist
965
- try {
966
- await store['db'].query({ query: `DROP TABLE IF EXISTS ${tempTable}` });
967
- } catch {
968
- /* ignore */
969
- }
970
- });
971
-
972
- it('returns true if the column exists', async () => {
973
- await store['db'].query({
974
- query: `CREATE TABLE temp_test_table (
975
- id UInt64,
976
- resourceId String
977
- ) ENGINE = MergeTree()
978
- ORDER BY id
979
- `,
980
- });
981
- expect(await store['hasColumn'](tempTable, 'resourceId')).toBe(true);
982
- });
983
-
984
- it('returns false if the column does not exist', async () => {
985
- await store['db'].query({
986
- query: `CREATE TABLE temp_test_table (
987
- id UInt64,
988
- ) ENGINE = MergeTree()
989
- ORDER BY id
990
- `,
991
- });
992
- expect(await store['hasColumn'](tempTable, 'resourceId')).toBe(false);
993
- });
994
-
995
- afterEach(async () => {
996
- // Clean up after each test
997
- try {
998
- await store['db'].query({ query: `DROP TABLE IF EXISTS ${tempTable}` });
999
- } catch {
1000
- /* ignore */
1001
- }
1002
- });
1003
- });
1004
-
1005
- describe('alterTable', () => {
1006
- const TEST_TABLE = 'test_alter_table';
1007
- const BASE_SCHEMA = {
1008
- id: { type: 'integer', primaryKey: true, nullable: false },
1009
- name: { type: 'text', nullable: true },
1010
- createdAt: { type: 'timestamp', nullable: false },
1011
- updatedAt: { type: 'timestamp', nullable: false },
1012
- } as Record<string, StorageColumn>;
1013
-
1014
- TABLE_ENGINES[TEST_TABLE] = 'MergeTree()';
1015
-
1016
- beforeEach(async () => {
1017
- await store.createTable({ tableName: TEST_TABLE as TABLE_NAMES, schema: BASE_SCHEMA });
1018
- });
1019
-
1020
- afterEach(async () => {
1021
- await store.clearTable({ tableName: TEST_TABLE as TABLE_NAMES });
1022
- });
1023
-
1024
- it('adds a new column to an existing table', async () => {
1025
- await store.alterTable({
1026
- tableName: TEST_TABLE as TABLE_NAMES,
1027
- schema: { ...BASE_SCHEMA, age: { type: 'integer', nullable: true } },
1028
- ifNotExists: ['age'],
1029
- });
1030
-
1031
- await store.insert({
1032
- tableName: TEST_TABLE as TABLE_NAMES,
1033
- record: { id: 1, name: 'Alice', age: 42, createdAt: new Date(), updatedAt: new Date() },
1034
- });
1035
-
1036
- const row = await store.load<{ id: string; name: string; age?: number }>({
1037
- tableName: TEST_TABLE as TABLE_NAMES,
1038
- keys: { id: '1' },
1039
- });
1040
- expect(row?.age).toBe(42);
1041
- });
1042
-
1043
- it('is idempotent when adding an existing column', async () => {
1044
- await store.alterTable({
1045
- tableName: TEST_TABLE as TABLE_NAMES,
1046
- schema: { ...BASE_SCHEMA, foo: { type: 'text', nullable: true } },
1047
- ifNotExists: ['foo'],
1048
- });
1049
- // Add the column again (should not throw)
1050
- await expect(
1051
- store.alterTable({
1052
- tableName: TEST_TABLE as TABLE_NAMES,
1053
- schema: { ...BASE_SCHEMA, foo: { type: 'text', nullable: true } },
1054
- ifNotExists: ['foo'],
1055
- }),
1056
- ).resolves.not.toThrow();
1057
- });
1058
-
1059
- it('should add a default value to a column when using not null', async () => {
1060
- await store.insert({
1061
- tableName: TEST_TABLE as TABLE_NAMES,
1062
- record: { id: 1, name: 'Bob', createdAt: new Date(), updatedAt: new Date() },
1063
- });
1064
-
1065
- await expect(
1066
- store.alterTable({
1067
- tableName: TEST_TABLE as TABLE_NAMES,
1068
- schema: { ...BASE_SCHEMA, text_column: { type: 'text', nullable: false } },
1069
- ifNotExists: ['text_column'],
1070
- }),
1071
- ).resolves.not.toThrow();
1072
-
1073
- await expect(
1074
- store.alterTable({
1075
- tableName: TEST_TABLE as TABLE_NAMES,
1076
- schema: { ...BASE_SCHEMA, timestamp_column: { type: 'timestamp', nullable: false } },
1077
- ifNotExists: ['timestamp_column'],
1078
- }),
1079
- ).resolves.not.toThrow();
1080
-
1081
- await expect(
1082
- store.alterTable({
1083
- tableName: TEST_TABLE as TABLE_NAMES,
1084
- schema: { ...BASE_SCHEMA, bigint_column: { type: 'bigint', nullable: false } },
1085
- ifNotExists: ['bigint_column'],
1086
- }),
1087
- ).resolves.not.toThrow();
1088
-
1089
- await expect(
1090
- store.alterTable({
1091
- tableName: TEST_TABLE as TABLE_NAMES,
1092
- schema: { ...BASE_SCHEMA, jsonb_column: { type: 'jsonb', nullable: false } },
1093
- ifNotExists: ['jsonb_column'],
1094
- }),
1095
- ).resolves.not.toThrow();
1096
- });
1097
- });
24
+ const storage = new ClickhouseStore(TEST_CONFIG);
1098
25
 
1099
- afterAll(async () => {
1100
- await store.close();
1101
- });
1102
- });
26
+ createTestSuite(storage);