@mastra/clickhouse 0.0.0-tsconfig-compile-20250703214351 → 0.0.0-update-scorers-api-20250801170445
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +105 -5
- package/LICENSE.md +12 -4
- package/dist/index.cjs +2055 -609
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +2 -4
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2054 -606
- package/dist/index.js.map +1 -0
- package/dist/storage/domains/legacy-evals/index.d.ts +21 -0
- package/dist/storage/domains/legacy-evals/index.d.ts.map +1 -0
- package/dist/storage/domains/memory/index.d.ts +79 -0
- package/dist/storage/domains/memory/index.d.ts.map +1 -0
- package/dist/storage/domains/operations/index.d.ts +42 -0
- package/dist/storage/domains/operations/index.d.ts.map +1 -0
- package/dist/storage/domains/scores/index.d.ts +43 -0
- package/dist/storage/domains/scores/index.d.ts.map +1 -0
- package/dist/storage/domains/traces/index.d.ts +21 -0
- package/dist/storage/domains/traces/index.d.ts.map +1 -0
- package/dist/storage/domains/utils.d.ts +28 -0
- package/dist/storage/domains/utils.d.ts.map +1 -0
- package/dist/storage/domains/workflows/index.d.ts +36 -0
- package/dist/storage/domains/workflows/index.d.ts.map +1 -0
- package/dist/{_tsup-dts-rollup.d.cts → storage/index.d.ts} +109 -94
- package/dist/storage/index.d.ts.map +1 -0
- package/package.json +8 -8
- package/src/storage/domains/legacy-evals/index.ts +246 -0
- package/src/storage/domains/memory/index.ts +1393 -0
- package/src/storage/domains/operations/index.ts +319 -0
- package/src/storage/domains/scores/index.ts +326 -0
- package/src/storage/domains/traces/index.ts +275 -0
- package/src/storage/domains/utils.ts +86 -0
- package/src/storage/domains/workflows/index.ts +285 -0
- package/src/storage/index.test.ts +15 -1143
- package/src/storage/index.ts +186 -1246
- package/tsconfig.build.json +9 -0
- package/tsconfig.json +1 -1
- package/tsup.config.ts +22 -0
- package/dist/_tsup-dts-rollup.d.ts +0 -191
- package/dist/index.d.cts +0 -4
package/src/storage/index.ts
CHANGED
|
@@ -1,20 +1,12 @@
|
|
|
1
1
|
import type { ClickHouseClient } from '@clickhouse/client';
|
|
2
2
|
import { createClient } from '@clickhouse/client';
|
|
3
|
-
import { MessageList } from '@mastra/core/agent';
|
|
4
3
|
import type { MastraMessageContentV2 } from '@mastra/core/agent';
|
|
5
4
|
import { MastraError, ErrorDomain, ErrorCategory } from '@mastra/core/error';
|
|
6
|
-
import type { MetricResult, TestInfo } from '@mastra/core/eval';
|
|
7
5
|
import type { MastraMessageV1, MastraMessageV2, StorageThreadType } from '@mastra/core/memory';
|
|
8
|
-
import {
|
|
9
|
-
|
|
10
|
-
TABLE_EVALS,
|
|
11
|
-
TABLE_MESSAGES,
|
|
12
|
-
TABLE_SCHEMAS,
|
|
13
|
-
TABLE_THREADS,
|
|
14
|
-
TABLE_TRACES,
|
|
15
|
-
TABLE_WORKFLOW_SNAPSHOT,
|
|
16
|
-
} from '@mastra/core/storage';
|
|
6
|
+
import type { ScoreRowData } from '@mastra/core/scores';
|
|
7
|
+
import { MastraStorage } from '@mastra/core/storage';
|
|
17
8
|
import type {
|
|
9
|
+
TABLE_SCHEMAS,
|
|
18
10
|
EvalRow,
|
|
19
11
|
PaginationInfo,
|
|
20
12
|
StorageColumn,
|
|
@@ -23,20 +15,20 @@ import type {
|
|
|
23
15
|
WorkflowRun,
|
|
24
16
|
WorkflowRuns,
|
|
25
17
|
StorageGetTracesArg,
|
|
26
|
-
|
|
18
|
+
StorageGetTracesPaginatedArg,
|
|
19
|
+
StoragePagination,
|
|
20
|
+
StorageDomains,
|
|
21
|
+
PaginationArgs,
|
|
22
|
+
StorageResourceType,
|
|
27
23
|
} from '@mastra/core/storage';
|
|
28
24
|
import type { Trace } from '@mastra/core/telemetry';
|
|
29
25
|
import type { WorkflowRunState } from '@mastra/core/workflows';
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
} catch {
|
|
37
|
-
return {};
|
|
38
|
-
}
|
|
39
|
-
}
|
|
26
|
+
import { LegacyEvalsStorageClickhouse } from './domains/legacy-evals';
|
|
27
|
+
import { MemoryStorageClickhouse } from './domains/memory';
|
|
28
|
+
import { StoreOperationsClickhouse } from './domains/operations';
|
|
29
|
+
import { ScoresStorageClickhouse } from './domains/scores';
|
|
30
|
+
import { TracesStorageClickhouse } from './domains/traces';
|
|
31
|
+
import { WorkflowsStorageClickhouse } from './domains/workflows';
|
|
40
32
|
|
|
41
33
|
type IntervalUnit =
|
|
42
34
|
| 'NANOSECOND'
|
|
@@ -69,47 +61,15 @@ export type ClickhouseConfig = {
|
|
|
69
61
|
};
|
|
70
62
|
};
|
|
71
63
|
|
|
72
|
-
export const TABLE_ENGINES: Record<SUPPORTED_TABLE_NAMES, string> = {
|
|
73
|
-
[TABLE_MESSAGES]: `MergeTree()`,
|
|
74
|
-
[TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
|
|
75
|
-
[TABLE_TRACES]: `MergeTree()`,
|
|
76
|
-
[TABLE_THREADS]: `ReplacingMergeTree()`,
|
|
77
|
-
[TABLE_EVALS]: `MergeTree()`,
|
|
78
|
-
};
|
|
79
|
-
|
|
80
|
-
export const COLUMN_TYPES: Record<StorageColumn['type'], string> = {
|
|
81
|
-
text: 'String',
|
|
82
|
-
timestamp: 'DateTime64(3)',
|
|
83
|
-
uuid: 'String',
|
|
84
|
-
jsonb: 'String',
|
|
85
|
-
integer: 'Int64',
|
|
86
|
-
bigint: 'Int64',
|
|
87
|
-
};
|
|
88
|
-
|
|
89
|
-
function transformRows<R>(rows: any[]): R[] {
|
|
90
|
-
return rows.map((row: any) => transformRow<R>(row));
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
function transformRow<R>(row: any): R {
|
|
94
|
-
if (!row) {
|
|
95
|
-
return row;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
if (row.createdAt) {
|
|
99
|
-
row.createdAt = new Date(row.createdAt);
|
|
100
|
-
}
|
|
101
|
-
if (row.updatedAt) {
|
|
102
|
-
row.updatedAt = new Date(row.updatedAt);
|
|
103
|
-
}
|
|
104
|
-
return row;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
64
|
export class ClickhouseStore extends MastraStorage {
|
|
108
65
|
protected db: ClickHouseClient;
|
|
109
66
|
protected ttl: ClickhouseConfig['ttl'] = {};
|
|
110
67
|
|
|
68
|
+
stores: StorageDomains;
|
|
69
|
+
|
|
111
70
|
constructor(config: ClickhouseConfig) {
|
|
112
71
|
super({ name: 'ClickhouseStore' });
|
|
72
|
+
|
|
113
73
|
this.db = createClient({
|
|
114
74
|
url: config.url,
|
|
115
75
|
username: config.username,
|
|
@@ -122,246 +82,53 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
122
82
|
},
|
|
123
83
|
});
|
|
124
84
|
this.ttl = config.ttl;
|
|
125
|
-
}
|
|
126
85
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
const
|
|
130
|
-
const
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
return {
|
|
142
|
-
input: row.input as string,
|
|
143
|
-
output: row.output as string,
|
|
144
|
-
result: resultValue as MetricResult,
|
|
145
|
-
agentName: row.agent_name as string,
|
|
146
|
-
metricName: row.metric_name as string,
|
|
147
|
-
instructions: row.instructions as string,
|
|
148
|
-
testInfo: testInfoValue as TestInfo,
|
|
149
|
-
globalRunId: row.global_run_id as string,
|
|
150
|
-
runId: row.run_id as string,
|
|
151
|
-
createdAt: row.created_at as string,
|
|
86
|
+
const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
|
|
87
|
+
const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
|
|
88
|
+
const scores = new ScoresStorageClickhouse({ client: this.db, operations });
|
|
89
|
+
const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
|
|
90
|
+
const traces = new TracesStorageClickhouse({ client: this.db, operations });
|
|
91
|
+
const memory = new MemoryStorageClickhouse({ client: this.db, operations });
|
|
92
|
+
|
|
93
|
+
this.stores = {
|
|
94
|
+
operations,
|
|
95
|
+
workflows,
|
|
96
|
+
scores,
|
|
97
|
+
legacyEvals,
|
|
98
|
+
traces,
|
|
99
|
+
memory,
|
|
152
100
|
};
|
|
153
101
|
}
|
|
154
102
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
103
|
+
get supports(): {
|
|
104
|
+
selectByIncludeResourceScope: boolean;
|
|
105
|
+
resourceWorkingMemory: boolean;
|
|
106
|
+
hasColumn: boolean;
|
|
107
|
+
createTable: boolean;
|
|
108
|
+
deleteMessages: boolean;
|
|
109
|
+
} {
|
|
110
|
+
return {
|
|
111
|
+
selectByIncludeResourceScope: true,
|
|
112
|
+
resourceWorkingMemory: true,
|
|
113
|
+
hasColumn: true,
|
|
114
|
+
createTable: true,
|
|
115
|
+
deleteMessages: false,
|
|
116
|
+
};
|
|
166
117
|
}
|
|
167
118
|
|
|
168
119
|
async getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]> {
|
|
169
|
-
|
|
170
|
-
const baseQuery = `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
|
|
171
|
-
const typeCondition =
|
|
172
|
-
type === 'test'
|
|
173
|
-
? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL"
|
|
174
|
-
: type === 'live'
|
|
175
|
-
? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)"
|
|
176
|
-
: '';
|
|
177
|
-
|
|
178
|
-
const result = await this.db.query({
|
|
179
|
-
query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
|
|
180
|
-
query_params: { var_agent_name: agentName },
|
|
181
|
-
clickhouse_settings: {
|
|
182
|
-
date_time_input_format: 'best_effort',
|
|
183
|
-
date_time_output_format: 'iso',
|
|
184
|
-
use_client_time_zone: 1,
|
|
185
|
-
output_format_json_quote_64bit_integers: 0,
|
|
186
|
-
},
|
|
187
|
-
});
|
|
188
|
-
|
|
189
|
-
if (!result) {
|
|
190
|
-
return [];
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
const rows = await result.json();
|
|
194
|
-
return rows.data.map((row: any) => this.transformEvalRow(row));
|
|
195
|
-
} catch (error: any) {
|
|
196
|
-
if (error?.message?.includes('no such table') || error?.message?.includes('does not exist')) {
|
|
197
|
-
return [];
|
|
198
|
-
}
|
|
199
|
-
throw new MastraError(
|
|
200
|
-
{
|
|
201
|
-
id: 'CLICKHOUSE_STORAGE_GET_EVALS_BY_AGENT_FAILED',
|
|
202
|
-
domain: ErrorDomain.STORAGE,
|
|
203
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
204
|
-
details: { agentName, type: type ?? null },
|
|
205
|
-
},
|
|
206
|
-
error,
|
|
207
|
-
);
|
|
208
|
-
}
|
|
120
|
+
return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
|
|
209
121
|
}
|
|
210
122
|
|
|
211
|
-
async
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
values: records.map(record => ({
|
|
216
|
-
...Object.fromEntries(
|
|
217
|
-
Object.entries(record).map(([key, value]) => [
|
|
218
|
-
key,
|
|
219
|
-
TABLE_SCHEMAS[tableName as TABLE_NAMES]?.[key]?.type === 'timestamp'
|
|
220
|
-
? new Date(value).toISOString()
|
|
221
|
-
: value,
|
|
222
|
-
]),
|
|
223
|
-
),
|
|
224
|
-
})),
|
|
225
|
-
format: 'JSONEachRow',
|
|
226
|
-
clickhouse_settings: {
|
|
227
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
228
|
-
date_time_input_format: 'best_effort',
|
|
229
|
-
use_client_time_zone: 1,
|
|
230
|
-
output_format_json_quote_64bit_integers: 0,
|
|
231
|
-
},
|
|
232
|
-
});
|
|
233
|
-
} catch (error: any) {
|
|
234
|
-
throw new MastraError(
|
|
235
|
-
{
|
|
236
|
-
id: 'CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED',
|
|
237
|
-
domain: ErrorDomain.STORAGE,
|
|
238
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
239
|
-
details: { tableName },
|
|
240
|
-
},
|
|
241
|
-
error,
|
|
242
|
-
);
|
|
243
|
-
}
|
|
123
|
+
async getEvals(
|
|
124
|
+
options: { agentName?: string; type?: 'test' | 'live' } & PaginationArgs,
|
|
125
|
+
): Promise<PaginationInfo & { evals: EvalRow[] }> {
|
|
126
|
+
return this.stores.legacyEvals.getEvals(options);
|
|
244
127
|
}
|
|
245
128
|
|
|
246
|
-
async
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
page,
|
|
250
|
-
perPage,
|
|
251
|
-
attributes,
|
|
252
|
-
filters,
|
|
253
|
-
fromDate,
|
|
254
|
-
toDate,
|
|
255
|
-
}: {
|
|
256
|
-
name?: string;
|
|
257
|
-
scope?: string;
|
|
258
|
-
page: number;
|
|
259
|
-
perPage: number;
|
|
260
|
-
attributes?: Record<string, string>;
|
|
261
|
-
filters?: Record<string, any>;
|
|
262
|
-
fromDate?: Date;
|
|
263
|
-
toDate?: Date;
|
|
264
|
-
}): Promise<any[]> {
|
|
265
|
-
const limit = perPage;
|
|
266
|
-
const offset = page * perPage;
|
|
267
|
-
|
|
268
|
-
const args: Record<string, any> = {};
|
|
269
|
-
|
|
270
|
-
const conditions: string[] = [];
|
|
271
|
-
if (name) {
|
|
272
|
-
conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
|
|
273
|
-
args.var_name = name;
|
|
274
|
-
}
|
|
275
|
-
if (scope) {
|
|
276
|
-
conditions.push(`scope = {var_scope:String}`);
|
|
277
|
-
args.var_scope = scope;
|
|
278
|
-
}
|
|
279
|
-
if (attributes) {
|
|
280
|
-
Object.entries(attributes).forEach(([key, value]) => {
|
|
281
|
-
conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
|
|
282
|
-
args[`var_attr_${key}`] = value;
|
|
283
|
-
});
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
if (filters) {
|
|
287
|
-
Object.entries(filters).forEach(([key, value]) => {
|
|
288
|
-
conditions.push(
|
|
289
|
-
`${key} = {var_col_${key}:${COLUMN_TYPES[TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? 'text']}}`,
|
|
290
|
-
);
|
|
291
|
-
args[`var_col_${key}`] = value;
|
|
292
|
-
});
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
if (fromDate) {
|
|
296
|
-
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
297
|
-
args.var_from_date = fromDate.getTime() / 1000; // Convert to Unix timestamp
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
if (toDate) {
|
|
301
|
-
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
302
|
-
args.var_to_date = toDate.getTime() / 1000; // Convert to Unix timestamp
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
306
|
-
|
|
307
|
-
try {
|
|
308
|
-
const result = await this.db.query({
|
|
309
|
-
query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
|
|
310
|
-
query_params: args,
|
|
311
|
-
clickhouse_settings: {
|
|
312
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
313
|
-
date_time_input_format: 'best_effort',
|
|
314
|
-
date_time_output_format: 'iso',
|
|
315
|
-
use_client_time_zone: 1,
|
|
316
|
-
output_format_json_quote_64bit_integers: 0,
|
|
317
|
-
},
|
|
318
|
-
});
|
|
319
|
-
|
|
320
|
-
if (!result) {
|
|
321
|
-
return [];
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
const resp = await result.json();
|
|
325
|
-
const rows: any[] = resp.data;
|
|
326
|
-
return rows.map(row => ({
|
|
327
|
-
id: row.id,
|
|
328
|
-
parentSpanId: row.parentSpanId,
|
|
329
|
-
traceId: row.traceId,
|
|
330
|
-
name: row.name,
|
|
331
|
-
scope: row.scope,
|
|
332
|
-
kind: row.kind,
|
|
333
|
-
status: safelyParseJSON(row.status as string),
|
|
334
|
-
events: safelyParseJSON(row.events as string),
|
|
335
|
-
links: safelyParseJSON(row.links as string),
|
|
336
|
-
attributes: safelyParseJSON(row.attributes as string),
|
|
337
|
-
startTime: row.startTime,
|
|
338
|
-
endTime: row.endTime,
|
|
339
|
-
other: safelyParseJSON(row.other as string),
|
|
340
|
-
createdAt: row.createdAt,
|
|
341
|
-
}));
|
|
342
|
-
} catch (error: any) {
|
|
343
|
-
if (error?.message?.includes('no such table') || error?.message?.includes('does not exist')) {
|
|
344
|
-
return [];
|
|
345
|
-
}
|
|
346
|
-
throw new MastraError(
|
|
347
|
-
{
|
|
348
|
-
id: 'CLICKHOUSE_STORAGE_GET_TRACES_FAILED',
|
|
349
|
-
domain: ErrorDomain.STORAGE,
|
|
350
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
351
|
-
details: {
|
|
352
|
-
name: name ?? null,
|
|
353
|
-
scope: scope ?? null,
|
|
354
|
-
page,
|
|
355
|
-
perPage,
|
|
356
|
-
attributes: attributes ? JSON.stringify(attributes) : null,
|
|
357
|
-
filters: filters ? JSON.stringify(filters) : null,
|
|
358
|
-
fromDate: fromDate?.toISOString() ?? null,
|
|
359
|
-
toDate: toDate?.toISOString() ?? null,
|
|
360
|
-
},
|
|
361
|
-
},
|
|
362
|
-
error,
|
|
363
|
-
);
|
|
364
|
-
}
|
|
129
|
+
async batchInsert({ tableName, records }: { tableName: TABLE_NAMES; records: Record<string, any>[] }): Promise<void> {
|
|
130
|
+
await this.stores.operations.batchInsert({ tableName, records });
|
|
131
|
+
// await this.optimizeTable({ tableName });
|
|
365
132
|
}
|
|
366
133
|
|
|
367
134
|
async optimizeTable({ tableName }: { tableName: TABLE_NAMES }): Promise<void> {
|
|
@@ -404,88 +171,16 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
404
171
|
tableName,
|
|
405
172
|
schema,
|
|
406
173
|
}: {
|
|
407
|
-
tableName:
|
|
174
|
+
tableName: TABLE_NAMES;
|
|
408
175
|
schema: Record<string, StorageColumn>;
|
|
409
176
|
}): Promise<void> {
|
|
410
|
-
|
|
411
|
-
const columns = Object.entries(schema)
|
|
412
|
-
.map(([name, def]) => {
|
|
413
|
-
const constraints = [];
|
|
414
|
-
if (!def.nullable) constraints.push('NOT NULL');
|
|
415
|
-
const columnTtl = this.ttl?.[tableName]?.columns?.[name];
|
|
416
|
-
return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(' ')} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? 'createdAt'}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ''}`;
|
|
417
|
-
})
|
|
418
|
-
.join(',\n');
|
|
419
|
-
|
|
420
|
-
const rowTtl = this.ttl?.[tableName]?.row;
|
|
421
|
-
const sql =
|
|
422
|
-
tableName === TABLE_WORKFLOW_SNAPSHOT
|
|
423
|
-
? `
|
|
424
|
-
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
425
|
-
${['id String'].concat(columns)}
|
|
426
|
-
)
|
|
427
|
-
ENGINE = ${TABLE_ENGINES[tableName]}
|
|
428
|
-
PRIMARY KEY (createdAt, run_id, workflow_name)
|
|
429
|
-
ORDER BY (createdAt, run_id, workflow_name)
|
|
430
|
-
${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? 'createdAt'}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ''}
|
|
431
|
-
SETTINGS index_granularity = 8192
|
|
432
|
-
`
|
|
433
|
-
: `
|
|
434
|
-
CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
435
|
-
${columns}
|
|
436
|
-
)
|
|
437
|
-
ENGINE = ${TABLE_ENGINES[tableName]}
|
|
438
|
-
PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? 'run_id' : 'id'})
|
|
439
|
-
ORDER BY (createdAt, ${tableName === TABLE_EVALS ? 'run_id' : 'id'})
|
|
440
|
-
${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ''}
|
|
441
|
-
SETTINGS index_granularity = 8192
|
|
442
|
-
`;
|
|
443
|
-
|
|
444
|
-
await this.db.query({
|
|
445
|
-
query: sql,
|
|
446
|
-
clickhouse_settings: {
|
|
447
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
448
|
-
date_time_input_format: 'best_effort',
|
|
449
|
-
date_time_output_format: 'iso',
|
|
450
|
-
use_client_time_zone: 1,
|
|
451
|
-
output_format_json_quote_64bit_integers: 0,
|
|
452
|
-
},
|
|
453
|
-
});
|
|
454
|
-
} catch (error: any) {
|
|
455
|
-
throw new MastraError(
|
|
456
|
-
{
|
|
457
|
-
id: 'CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED',
|
|
458
|
-
domain: ErrorDomain.STORAGE,
|
|
459
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
460
|
-
details: { tableName },
|
|
461
|
-
},
|
|
462
|
-
error,
|
|
463
|
-
);
|
|
464
|
-
}
|
|
177
|
+
return this.stores.operations.createTable({ tableName, schema });
|
|
465
178
|
}
|
|
466
179
|
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
case 'text':
|
|
470
|
-
return 'String';
|
|
471
|
-
case 'timestamp':
|
|
472
|
-
return 'DateTime64(3)';
|
|
473
|
-
case 'integer':
|
|
474
|
-
case 'bigint':
|
|
475
|
-
return 'Int64';
|
|
476
|
-
case 'jsonb':
|
|
477
|
-
return 'String';
|
|
478
|
-
default:
|
|
479
|
-
return super.getSqlType(type); // fallback to base implementation
|
|
480
|
-
}
|
|
180
|
+
async dropTable({ tableName }: { tableName: TABLE_NAMES }): Promise<void> {
|
|
181
|
+
return this.stores.operations.dropTable({ tableName });
|
|
481
182
|
}
|
|
482
183
|
|
|
483
|
-
/**
|
|
484
|
-
* Alters table schema to add columns if they don't exist
|
|
485
|
-
* @param tableName Name of the table
|
|
486
|
-
* @param schema Schema of the table
|
|
487
|
-
* @param ifNotExists Array of column names to add if they don't exist
|
|
488
|
-
*/
|
|
489
184
|
async alterTable({
|
|
490
185
|
tableName,
|
|
491
186
|
schema,
|
|
@@ -495,292 +190,93 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
495
190
|
schema: Record<string, StorageColumn>;
|
|
496
191
|
ifNotExists: string[];
|
|
497
192
|
}): Promise<void> {
|
|
498
|
-
|
|
499
|
-
// 1. Get existing columns
|
|
500
|
-
const describeSql = `DESCRIBE TABLE ${tableName}`;
|
|
501
|
-
const result = await this.db.query({
|
|
502
|
-
query: describeSql,
|
|
503
|
-
});
|
|
504
|
-
const rows = await result.json();
|
|
505
|
-
const existingColumnNames = new Set(rows.data.map((row: any) => row.name.toLowerCase()));
|
|
506
|
-
|
|
507
|
-
// 2. Add missing columns
|
|
508
|
-
for (const columnName of ifNotExists) {
|
|
509
|
-
if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
|
|
510
|
-
const columnDef = schema[columnName];
|
|
511
|
-
let sqlType = this.getSqlType(columnDef.type);
|
|
512
|
-
if (columnDef.nullable !== false) {
|
|
513
|
-
sqlType = `Nullable(${sqlType})`;
|
|
514
|
-
}
|
|
515
|
-
const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : '';
|
|
516
|
-
// Use backticks or double quotes as needed for identifiers
|
|
517
|
-
const alterSql =
|
|
518
|
-
`ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
|
|
519
|
-
|
|
520
|
-
await this.db.query({
|
|
521
|
-
query: alterSql,
|
|
522
|
-
});
|
|
523
|
-
this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
|
|
524
|
-
}
|
|
525
|
-
}
|
|
526
|
-
} catch (error: any) {
|
|
527
|
-
throw new MastraError(
|
|
528
|
-
{
|
|
529
|
-
id: 'CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED',
|
|
530
|
-
domain: ErrorDomain.STORAGE,
|
|
531
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
532
|
-
details: { tableName },
|
|
533
|
-
},
|
|
534
|
-
error,
|
|
535
|
-
);
|
|
536
|
-
}
|
|
193
|
+
return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
|
|
537
194
|
}
|
|
538
195
|
|
|
539
196
|
async clearTable({ tableName }: { tableName: TABLE_NAMES }): Promise<void> {
|
|
540
|
-
|
|
541
|
-
await this.db.query({
|
|
542
|
-
query: `TRUNCATE TABLE ${tableName}`,
|
|
543
|
-
clickhouse_settings: {
|
|
544
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
545
|
-
date_time_input_format: 'best_effort',
|
|
546
|
-
date_time_output_format: 'iso',
|
|
547
|
-
use_client_time_zone: 1,
|
|
548
|
-
output_format_json_quote_64bit_integers: 0,
|
|
549
|
-
},
|
|
550
|
-
});
|
|
551
|
-
} catch (error: any) {
|
|
552
|
-
throw new MastraError(
|
|
553
|
-
{
|
|
554
|
-
id: 'CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED',
|
|
555
|
-
domain: ErrorDomain.STORAGE,
|
|
556
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
557
|
-
details: { tableName },
|
|
558
|
-
},
|
|
559
|
-
error,
|
|
560
|
-
);
|
|
561
|
-
}
|
|
197
|
+
return this.stores.operations.clearTable({ tableName });
|
|
562
198
|
}
|
|
563
199
|
|
|
564
200
|
async insert({ tableName, record }: { tableName: TABLE_NAMES; record: Record<string, any> }): Promise<void> {
|
|
565
|
-
|
|
566
|
-
await this.db.insert({
|
|
567
|
-
table: tableName,
|
|
568
|
-
values: [
|
|
569
|
-
{
|
|
570
|
-
...record,
|
|
571
|
-
createdAt: record.createdAt.toISOString(),
|
|
572
|
-
updatedAt: record.updatedAt.toISOString(),
|
|
573
|
-
},
|
|
574
|
-
],
|
|
575
|
-
format: 'JSONEachRow',
|
|
576
|
-
clickhouse_settings: {
|
|
577
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
578
|
-
output_format_json_quote_64bit_integers: 0,
|
|
579
|
-
date_time_input_format: 'best_effort',
|
|
580
|
-
use_client_time_zone: 1,
|
|
581
|
-
},
|
|
582
|
-
});
|
|
583
|
-
} catch (error: any) {
|
|
584
|
-
throw new MastraError(
|
|
585
|
-
{
|
|
586
|
-
id: 'CLICKHOUSE_STORAGE_INSERT_FAILED',
|
|
587
|
-
domain: ErrorDomain.STORAGE,
|
|
588
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
589
|
-
details: { tableName },
|
|
590
|
-
},
|
|
591
|
-
error,
|
|
592
|
-
);
|
|
593
|
-
}
|
|
201
|
+
return this.stores.operations.insert({ tableName, record });
|
|
594
202
|
}
|
|
595
203
|
|
|
596
|
-
async load<R>({
|
|
597
|
-
tableName,
|
|
598
|
-
|
|
599
|
-
}: {
|
|
600
|
-
tableName: SUPPORTED_TABLE_NAMES;
|
|
601
|
-
keys: Record<string, string>;
|
|
602
|
-
}): Promise<R | null> {
|
|
603
|
-
try {
|
|
604
|
-
const keyEntries = Object.entries(keys);
|
|
605
|
-
const conditions = keyEntries
|
|
606
|
-
.map(
|
|
607
|
-
([key]) =>
|
|
608
|
-
`"${key}" = {var_${key}:${COLUMN_TYPES[TABLE_SCHEMAS[tableName as TABLE_NAMES]?.[key]?.type ?? 'text']}}`,
|
|
609
|
-
)
|
|
610
|
-
.join(' AND ');
|
|
611
|
-
const values = keyEntries.reduce((acc, [key, value]) => {
|
|
612
|
-
return { ...acc, [`var_${key}`]: value };
|
|
613
|
-
}, {});
|
|
204
|
+
async load<R>({ tableName, keys }: { tableName: TABLE_NAMES; keys: Record<string, string> }): Promise<R | null> {
|
|
205
|
+
return this.stores.operations.load({ tableName, keys });
|
|
206
|
+
}
|
|
614
207
|
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
208
|
+
async persistWorkflowSnapshot({
|
|
209
|
+
workflowName,
|
|
210
|
+
runId,
|
|
211
|
+
snapshot,
|
|
212
|
+
}: {
|
|
213
|
+
workflowName: string;
|
|
214
|
+
runId: string;
|
|
215
|
+
snapshot: WorkflowRunState;
|
|
216
|
+
}): Promise<void> {
|
|
217
|
+
return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, snapshot });
|
|
218
|
+
}
|
|
626
219
|
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
220
|
+
async loadWorkflowSnapshot({
|
|
221
|
+
workflowName,
|
|
222
|
+
runId,
|
|
223
|
+
}: {
|
|
224
|
+
workflowName: string;
|
|
225
|
+
runId: string;
|
|
226
|
+
}): Promise<WorkflowRunState | null> {
|
|
227
|
+
return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
|
|
228
|
+
}
|
|
630
229
|
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
230
|
+
async getWorkflowRuns({
|
|
231
|
+
workflowName,
|
|
232
|
+
fromDate,
|
|
233
|
+
toDate,
|
|
234
|
+
limit,
|
|
235
|
+
offset,
|
|
236
|
+
resourceId,
|
|
237
|
+
}: {
|
|
238
|
+
workflowName?: string;
|
|
239
|
+
fromDate?: Date;
|
|
240
|
+
toDate?: Date;
|
|
241
|
+
limit?: number;
|
|
242
|
+
offset?: number;
|
|
243
|
+
resourceId?: string;
|
|
244
|
+
} = {}): Promise<WorkflowRuns> {
|
|
245
|
+
return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
|
|
246
|
+
}
|
|
643
247
|
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
details: { tableName },
|
|
653
|
-
},
|
|
654
|
-
error,
|
|
655
|
-
);
|
|
656
|
-
}
|
|
248
|
+
async getWorkflowRunById({
|
|
249
|
+
runId,
|
|
250
|
+
workflowName,
|
|
251
|
+
}: {
|
|
252
|
+
runId: string;
|
|
253
|
+
workflowName?: string;
|
|
254
|
+
}): Promise<WorkflowRun | null> {
|
|
255
|
+
return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
|
|
657
256
|
}
|
|
658
257
|
|
|
659
|
-
async
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
query: `SELECT
|
|
663
|
-
id,
|
|
664
|
-
"resourceId",
|
|
665
|
-
title,
|
|
666
|
-
metadata,
|
|
667
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
668
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
669
|
-
FROM "${TABLE_THREADS}"
|
|
670
|
-
FINAL
|
|
671
|
-
WHERE id = {var_id:String}`,
|
|
672
|
-
query_params: { var_id: threadId },
|
|
673
|
-
clickhouse_settings: {
|
|
674
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
675
|
-
date_time_input_format: 'best_effort',
|
|
676
|
-
date_time_output_format: 'iso',
|
|
677
|
-
use_client_time_zone: 1,
|
|
678
|
-
output_format_json_quote_64bit_integers: 0,
|
|
679
|
-
},
|
|
680
|
-
});
|
|
258
|
+
async getTraces(args: StorageGetTracesArg): Promise<any[]> {
|
|
259
|
+
return this.stores.traces.getTraces(args);
|
|
260
|
+
}
|
|
681
261
|
|
|
682
|
-
|
|
683
|
-
|
|
262
|
+
async getTracesPaginated(args: StorageGetTracesPaginatedArg): Promise<PaginationInfo & { traces: Trace[] }> {
|
|
263
|
+
return this.stores.traces.getTracesPaginated(args);
|
|
264
|
+
}
|
|
684
265
|
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
266
|
+
async batchTraceInsert(args: { records: Trace[] }): Promise<void> {
|
|
267
|
+
return this.stores.traces.batchTraceInsert(args);
|
|
268
|
+
}
|
|
688
269
|
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
metadata: typeof thread.metadata === 'string' ? JSON.parse(thread.metadata) : thread.metadata,
|
|
692
|
-
createdAt: thread.createdAt,
|
|
693
|
-
updatedAt: thread.updatedAt,
|
|
694
|
-
};
|
|
695
|
-
} catch (error: any) {
|
|
696
|
-
throw new MastraError(
|
|
697
|
-
{
|
|
698
|
-
id: 'CLICKHOUSE_STORAGE_GET_THREAD_BY_ID_FAILED',
|
|
699
|
-
domain: ErrorDomain.STORAGE,
|
|
700
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
701
|
-
details: { threadId },
|
|
702
|
-
},
|
|
703
|
-
error,
|
|
704
|
-
);
|
|
705
|
-
}
|
|
270
|
+
async getThreadById({ threadId }: { threadId: string }): Promise<StorageThreadType | null> {
|
|
271
|
+
return this.stores.memory.getThreadById({ threadId });
|
|
706
272
|
}
|
|
707
273
|
|
|
708
274
|
async getThreadsByResourceId({ resourceId }: { resourceId: string }): Promise<StorageThreadType[]> {
|
|
709
|
-
|
|
710
|
-
const result = await this.db.query({
|
|
711
|
-
query: `SELECT
|
|
712
|
-
id,
|
|
713
|
-
"resourceId",
|
|
714
|
-
title,
|
|
715
|
-
metadata,
|
|
716
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
717
|
-
toDateTime64(updatedAt, 3) as updatedAt
|
|
718
|
-
FROM "${TABLE_THREADS}"
|
|
719
|
-
WHERE "resourceId" = {var_resourceId:String}`,
|
|
720
|
-
query_params: { var_resourceId: resourceId },
|
|
721
|
-
clickhouse_settings: {
|
|
722
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
723
|
-
date_time_input_format: 'best_effort',
|
|
724
|
-
date_time_output_format: 'iso',
|
|
725
|
-
use_client_time_zone: 1,
|
|
726
|
-
output_format_json_quote_64bit_integers: 0,
|
|
727
|
-
},
|
|
728
|
-
});
|
|
729
|
-
|
|
730
|
-
const rows = await result.json();
|
|
731
|
-
const threads = transformRows(rows.data) as StorageThreadType[];
|
|
732
|
-
|
|
733
|
-
return threads.map((thread: StorageThreadType) => ({
|
|
734
|
-
...thread,
|
|
735
|
-
metadata: typeof thread.metadata === 'string' ? JSON.parse(thread.metadata) : thread.metadata,
|
|
736
|
-
createdAt: thread.createdAt,
|
|
737
|
-
updatedAt: thread.updatedAt,
|
|
738
|
-
}));
|
|
739
|
-
} catch (error) {
|
|
740
|
-
throw new MastraError(
|
|
741
|
-
{
|
|
742
|
-
id: 'CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_FAILED',
|
|
743
|
-
domain: ErrorDomain.STORAGE,
|
|
744
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
745
|
-
details: { resourceId },
|
|
746
|
-
},
|
|
747
|
-
error,
|
|
748
|
-
);
|
|
749
|
-
}
|
|
275
|
+
return this.stores.memory.getThreadsByResourceId({ resourceId });
|
|
750
276
|
}
|
|
751
277
|
|
|
752
278
|
async saveThread({ thread }: { thread: StorageThreadType }): Promise<StorageThreadType> {
|
|
753
|
-
|
|
754
|
-
await this.db.insert({
|
|
755
|
-
table: TABLE_THREADS,
|
|
756
|
-
values: [
|
|
757
|
-
{
|
|
758
|
-
...thread,
|
|
759
|
-
createdAt: thread.createdAt.toISOString(),
|
|
760
|
-
updatedAt: thread.updatedAt.toISOString(),
|
|
761
|
-
},
|
|
762
|
-
],
|
|
763
|
-
format: 'JSONEachRow',
|
|
764
|
-
clickhouse_settings: {
|
|
765
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
766
|
-
date_time_input_format: 'best_effort',
|
|
767
|
-
use_client_time_zone: 1,
|
|
768
|
-
output_format_json_quote_64bit_integers: 0,
|
|
769
|
-
},
|
|
770
|
-
});
|
|
771
|
-
|
|
772
|
-
return thread;
|
|
773
|
-
} catch (error) {
|
|
774
|
-
throw new MastraError(
|
|
775
|
-
{
|
|
776
|
-
id: 'CLICKHOUSE_STORAGE_SAVE_THREAD_FAILED',
|
|
777
|
-
domain: ErrorDomain.STORAGE,
|
|
778
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
779
|
-
details: { threadId: thread.id },
|
|
780
|
-
},
|
|
781
|
-
error,
|
|
782
|
-
);
|
|
783
|
-
}
|
|
279
|
+
return this.stores.memory.saveThread({ thread });
|
|
784
280
|
}
|
|
785
281
|
|
|
786
282
|
async updateThread({
|
|
@@ -792,90 +288,19 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
792
288
|
title: string;
|
|
793
289
|
metadata: Record<string, unknown>;
|
|
794
290
|
}): Promise<StorageThreadType> {
|
|
795
|
-
|
|
796
|
-
// First get the existing thread to merge metadata
|
|
797
|
-
const existingThread = await this.getThreadById({ threadId: id });
|
|
798
|
-
if (!existingThread) {
|
|
799
|
-
throw new Error(`Thread ${id} not found`);
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
// Merge the existing metadata with the new metadata
|
|
803
|
-
const mergedMetadata = {
|
|
804
|
-
...existingThread.metadata,
|
|
805
|
-
...metadata,
|
|
806
|
-
};
|
|
807
|
-
|
|
808
|
-
const updatedThread = {
|
|
809
|
-
...existingThread,
|
|
810
|
-
title,
|
|
811
|
-
metadata: mergedMetadata,
|
|
812
|
-
updatedAt: new Date(),
|
|
813
|
-
};
|
|
814
|
-
|
|
815
|
-
await this.db.insert({
|
|
816
|
-
table: TABLE_THREADS,
|
|
817
|
-
format: 'JSONEachRow',
|
|
818
|
-
values: [
|
|
819
|
-
{
|
|
820
|
-
id: updatedThread.id,
|
|
821
|
-
resourceId: updatedThread.resourceId,
|
|
822
|
-
title: updatedThread.title,
|
|
823
|
-
metadata: updatedThread.metadata,
|
|
824
|
-
createdAt: updatedThread.createdAt,
|
|
825
|
-
updatedAt: updatedThread.updatedAt.toISOString(),
|
|
826
|
-
},
|
|
827
|
-
],
|
|
828
|
-
clickhouse_settings: {
|
|
829
|
-
date_time_input_format: 'best_effort',
|
|
830
|
-
use_client_time_zone: 1,
|
|
831
|
-
output_format_json_quote_64bit_integers: 0,
|
|
832
|
-
},
|
|
833
|
-
});
|
|
834
|
-
|
|
835
|
-
return updatedThread;
|
|
836
|
-
} catch (error) {
|
|
837
|
-
throw new MastraError(
|
|
838
|
-
{
|
|
839
|
-
id: 'CLICKHOUSE_STORAGE_UPDATE_THREAD_FAILED',
|
|
840
|
-
domain: ErrorDomain.STORAGE,
|
|
841
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
842
|
-
details: { threadId: id, title },
|
|
843
|
-
},
|
|
844
|
-
error,
|
|
845
|
-
);
|
|
846
|
-
}
|
|
291
|
+
return this.stores.memory.updateThread({ id, title, metadata });
|
|
847
292
|
}
|
|
848
293
|
|
|
849
294
|
async deleteThread({ threadId }: { threadId: string }): Promise<void> {
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
await this.db.command({
|
|
853
|
-
query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
|
|
854
|
-
query_params: { var_thread_id: threadId },
|
|
855
|
-
clickhouse_settings: {
|
|
856
|
-
output_format_json_quote_64bit_integers: 0,
|
|
857
|
-
},
|
|
858
|
-
});
|
|
295
|
+
return this.stores.memory.deleteThread({ threadId });
|
|
296
|
+
}
|
|
859
297
|
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
},
|
|
867
|
-
});
|
|
868
|
-
} catch (error) {
|
|
869
|
-
throw new MastraError(
|
|
870
|
-
{
|
|
871
|
-
id: 'CLICKHOUSE_STORAGE_DELETE_THREAD_FAILED',
|
|
872
|
-
domain: ErrorDomain.STORAGE,
|
|
873
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
874
|
-
details: { threadId },
|
|
875
|
-
},
|
|
876
|
-
error,
|
|
877
|
-
);
|
|
878
|
-
}
|
|
298
|
+
async getThreadsByResourceIdPaginated(args: {
|
|
299
|
+
resourceId: string;
|
|
300
|
+
page: number;
|
|
301
|
+
perPage: number;
|
|
302
|
+
}): Promise<PaginationInfo & { threads: StorageThreadType[] }> {
|
|
303
|
+
return this.stores.memory.getThreadsByResourceIdPaginated(args);
|
|
879
304
|
}
|
|
880
305
|
|
|
881
306
|
public async getMessages(args: StorageGetMessagesArg & { format?: 'v1' }): Promise<MastraMessageV1[]>;
|
|
@@ -886,126 +311,7 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
886
311
|
selectBy,
|
|
887
312
|
format,
|
|
888
313
|
}: StorageGetMessagesArg & { format?: 'v1' | 'v2' }): Promise<MastraMessageV1[] | MastraMessageV2[]> {
|
|
889
|
-
|
|
890
|
-
const messages: any[] = [];
|
|
891
|
-
const limit = this.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
|
|
892
|
-
const include = selectBy?.include || [];
|
|
893
|
-
|
|
894
|
-
if (include.length) {
|
|
895
|
-
const includeResult = await this.db.query({
|
|
896
|
-
query: `
|
|
897
|
-
WITH ordered_messages AS (
|
|
898
|
-
SELECT
|
|
899
|
-
*,
|
|
900
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
901
|
-
toDateTime64(updatedAt, 3) as updatedAt,
|
|
902
|
-
ROW_NUMBER() OVER (ORDER BY "createdAt" DESC) as row_num
|
|
903
|
-
FROM "${TABLE_MESSAGES}"
|
|
904
|
-
WHERE thread_id = {var_thread_id:String}
|
|
905
|
-
)
|
|
906
|
-
SELECT
|
|
907
|
-
m.id AS id,
|
|
908
|
-
m.content as content,
|
|
909
|
-
m.role as role,
|
|
910
|
-
m.type as type,
|
|
911
|
-
m.createdAt as createdAt,
|
|
912
|
-
m.updatedAt as updatedAt,
|
|
913
|
-
m.thread_id AS "threadId"
|
|
914
|
-
FROM ordered_messages m
|
|
915
|
-
WHERE m.id = ANY({var_include:Array(String)})
|
|
916
|
-
OR EXISTS (
|
|
917
|
-
SELECT 1 FROM ordered_messages target
|
|
918
|
-
WHERE target.id = ANY({var_include:Array(String)})
|
|
919
|
-
AND (
|
|
920
|
-
-- Get previous messages based on the max withPreviousMessages
|
|
921
|
-
(m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num)
|
|
922
|
-
OR
|
|
923
|
-
-- Get next messages based on the max withNextMessages
|
|
924
|
-
(m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num)
|
|
925
|
-
)
|
|
926
|
-
)
|
|
927
|
-
ORDER BY m."createdAt" DESC
|
|
928
|
-
`,
|
|
929
|
-
query_params: {
|
|
930
|
-
var_thread_id: threadId,
|
|
931
|
-
var_include: include.map(i => i.id),
|
|
932
|
-
var_withPreviousMessages: Math.max(...include.map(i => i.withPreviousMessages || 0)),
|
|
933
|
-
var_withNextMessages: Math.max(...include.map(i => i.withNextMessages || 0)),
|
|
934
|
-
},
|
|
935
|
-
clickhouse_settings: {
|
|
936
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
937
|
-
date_time_input_format: 'best_effort',
|
|
938
|
-
date_time_output_format: 'iso',
|
|
939
|
-
use_client_time_zone: 1,
|
|
940
|
-
output_format_json_quote_64bit_integers: 0,
|
|
941
|
-
},
|
|
942
|
-
});
|
|
943
|
-
|
|
944
|
-
const rows = await includeResult.json();
|
|
945
|
-
messages.push(...transformRows(rows.data));
|
|
946
|
-
}
|
|
947
|
-
|
|
948
|
-
// Then get the remaining messages, excluding the ids we just fetched
|
|
949
|
-
const result = await this.db.query({
|
|
950
|
-
query: `
|
|
951
|
-
SELECT
|
|
952
|
-
id,
|
|
953
|
-
content,
|
|
954
|
-
role,
|
|
955
|
-
type,
|
|
956
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
957
|
-
thread_id AS "threadId"
|
|
958
|
-
FROM "${TABLE_MESSAGES}"
|
|
959
|
-
WHERE thread_id = {threadId:String}
|
|
960
|
-
AND id NOT IN ({exclude:Array(String)})
|
|
961
|
-
ORDER BY "createdAt" DESC
|
|
962
|
-
LIMIT {limit:Int64}
|
|
963
|
-
`,
|
|
964
|
-
query_params: {
|
|
965
|
-
threadId,
|
|
966
|
-
exclude: messages.map(m => m.id),
|
|
967
|
-
limit,
|
|
968
|
-
},
|
|
969
|
-
clickhouse_settings: {
|
|
970
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
971
|
-
date_time_input_format: 'best_effort',
|
|
972
|
-
date_time_output_format: 'iso',
|
|
973
|
-
use_client_time_zone: 1,
|
|
974
|
-
output_format_json_quote_64bit_integers: 0,
|
|
975
|
-
},
|
|
976
|
-
});
|
|
977
|
-
|
|
978
|
-
const rows = await result.json();
|
|
979
|
-
messages.push(...transformRows(rows.data));
|
|
980
|
-
|
|
981
|
-
// Sort all messages by creation date
|
|
982
|
-
messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
|
|
983
|
-
|
|
984
|
-
// Parse message content
|
|
985
|
-
messages.forEach(message => {
|
|
986
|
-
if (typeof message.content === 'string') {
|
|
987
|
-
try {
|
|
988
|
-
message.content = JSON.parse(message.content);
|
|
989
|
-
} catch {
|
|
990
|
-
// If parsing fails, leave as string
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
});
|
|
994
|
-
|
|
995
|
-
const list = new MessageList({ threadId, resourceId }).add(messages, 'memory');
|
|
996
|
-
if (format === `v2`) return list.get.all.v2();
|
|
997
|
-
return list.get.all.v1();
|
|
998
|
-
} catch (error) {
|
|
999
|
-
throw new MastraError(
|
|
1000
|
-
{
|
|
1001
|
-
id: 'CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED',
|
|
1002
|
-
domain: ErrorDomain.STORAGE,
|
|
1003
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
1004
|
-
details: { threadId, resourceId: resourceId ?? '' },
|
|
1005
|
-
},
|
|
1006
|
-
error,
|
|
1007
|
-
);
|
|
1008
|
-
}
|
|
314
|
+
return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
|
|
1009
315
|
}
|
|
1010
316
|
|
|
1011
317
|
async saveMessages(args: { messages: MastraMessageV1[]; format?: undefined | 'v1' }): Promise<MastraMessageV1[]>;
|
|
@@ -1013,452 +319,86 @@ export class ClickhouseStore extends MastraStorage {
|
|
|
1013
319
|
async saveMessages(
|
|
1014
320
|
args: { messages: MastraMessageV1[]; format?: undefined | 'v1' } | { messages: MastraMessageV2[]; format: 'v2' },
|
|
1015
321
|
): Promise<MastraMessageV2[] | MastraMessageV1[]> {
|
|
1016
|
-
|
|
1017
|
-
if (messages.length === 0) return messages;
|
|
1018
|
-
|
|
1019
|
-
try {
|
|
1020
|
-
const threadId = messages[0]?.threadId;
|
|
1021
|
-
const resourceId = messages[0]?.resourceId;
|
|
1022
|
-
if (!threadId) {
|
|
1023
|
-
throw new Error('Thread ID is required');
|
|
1024
|
-
}
|
|
1025
|
-
|
|
1026
|
-
// Check if thread exists
|
|
1027
|
-
const thread = await this.getThreadById({ threadId });
|
|
1028
|
-
if (!thread) {
|
|
1029
|
-
throw new Error(`Thread ${threadId} not found`);
|
|
1030
|
-
}
|
|
1031
|
-
|
|
1032
|
-
// Clickhouse's MergeTree engine does not support native upserts or unique constraints on (id, thread_id).
|
|
1033
|
-
// Note: We cannot switch to ReplacingMergeTree without a schema migration,
|
|
1034
|
-
// as it would require altering the table engine.
|
|
1035
|
-
// To ensure correct upsert behavior, we first fetch existing (id, thread_id) pairs for the incoming messages.
|
|
1036
|
-
const existingResult = await this.db.query({
|
|
1037
|
-
query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
|
|
1038
|
-
query_params: {
|
|
1039
|
-
ids: messages.map(m => m.id),
|
|
1040
|
-
},
|
|
1041
|
-
clickhouse_settings: {
|
|
1042
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1043
|
-
date_time_input_format: 'best_effort',
|
|
1044
|
-
date_time_output_format: 'iso',
|
|
1045
|
-
use_client_time_zone: 1,
|
|
1046
|
-
output_format_json_quote_64bit_integers: 0,
|
|
1047
|
-
},
|
|
1048
|
-
format: 'JSONEachRow',
|
|
1049
|
-
});
|
|
1050
|
-
const existingRows: Array<{ id: string; thread_id: string }> = await existingResult.json();
|
|
1051
|
-
|
|
1052
|
-
const existingSet = new Set(existingRows.map(row => `${row.id}::${row.thread_id}`));
|
|
1053
|
-
// Partition the batch into new inserts and updates:
|
|
1054
|
-
// New messages are inserted in bulk.
|
|
1055
|
-
const toInsert = messages.filter(m => !existingSet.has(`${m.id}::${threadId}`));
|
|
1056
|
-
// Existing messages are updated via ALTER TABLE ... UPDATE.
|
|
1057
|
-
const toUpdate = messages.filter(m => existingSet.has(`${m.id}::${threadId}`));
|
|
1058
|
-
const updatePromises = toUpdate.map(message =>
|
|
1059
|
-
this.db.command({
|
|
1060
|
-
query: `
|
|
1061
|
-
ALTER TABLE ${TABLE_MESSAGES}
|
|
1062
|
-
UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}
|
|
1063
|
-
WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
|
|
1064
|
-
`,
|
|
1065
|
-
query_params: {
|
|
1066
|
-
var_content: typeof message.content === 'string' ? message.content : JSON.stringify(message.content),
|
|
1067
|
-
var_role: message.role,
|
|
1068
|
-
var_type: message.type || 'v2',
|
|
1069
|
-
var_id: message.id,
|
|
1070
|
-
var_thread_id: threadId,
|
|
1071
|
-
},
|
|
1072
|
-
clickhouse_settings: {
|
|
1073
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1074
|
-
date_time_input_format: 'best_effort',
|
|
1075
|
-
use_client_time_zone: 1,
|
|
1076
|
-
output_format_json_quote_64bit_integers: 0,
|
|
1077
|
-
},
|
|
1078
|
-
}),
|
|
1079
|
-
);
|
|
1080
|
-
|
|
1081
|
-
// Execute message inserts and thread update in parallel for better performance
|
|
1082
|
-
await Promise.all([
|
|
1083
|
-
// Insert messages
|
|
1084
|
-
this.db.insert({
|
|
1085
|
-
table: TABLE_MESSAGES,
|
|
1086
|
-
format: 'JSONEachRow',
|
|
1087
|
-
values: toInsert.map(message => ({
|
|
1088
|
-
id: message.id,
|
|
1089
|
-
thread_id: threadId,
|
|
1090
|
-
content: typeof message.content === 'string' ? message.content : JSON.stringify(message.content),
|
|
1091
|
-
createdAt: message.createdAt.toISOString(),
|
|
1092
|
-
role: message.role,
|
|
1093
|
-
type: message.type || 'v2',
|
|
1094
|
-
})),
|
|
1095
|
-
clickhouse_settings: {
|
|
1096
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1097
|
-
date_time_input_format: 'best_effort',
|
|
1098
|
-
use_client_time_zone: 1,
|
|
1099
|
-
output_format_json_quote_64bit_integers: 0,
|
|
1100
|
-
},
|
|
1101
|
-
}),
|
|
1102
|
-
...updatePromises,
|
|
1103
|
-
// Update thread's updatedAt timestamp
|
|
1104
|
-
this.db.insert({
|
|
1105
|
-
table: TABLE_THREADS,
|
|
1106
|
-
format: 'JSONEachRow',
|
|
1107
|
-
values: [
|
|
1108
|
-
{
|
|
1109
|
-
id: thread.id,
|
|
1110
|
-
resourceId: thread.resourceId,
|
|
1111
|
-
title: thread.title,
|
|
1112
|
-
metadata: thread.metadata,
|
|
1113
|
-
createdAt: thread.createdAt,
|
|
1114
|
-
updatedAt: new Date().toISOString(),
|
|
1115
|
-
},
|
|
1116
|
-
],
|
|
1117
|
-
clickhouse_settings: {
|
|
1118
|
-
date_time_input_format: 'best_effort',
|
|
1119
|
-
use_client_time_zone: 1,
|
|
1120
|
-
output_format_json_quote_64bit_integers: 0,
|
|
1121
|
-
},
|
|
1122
|
-
}),
|
|
1123
|
-
]);
|
|
1124
|
-
|
|
1125
|
-
const list = new MessageList({ threadId, resourceId }).add(messages, 'memory');
|
|
1126
|
-
if (format === `v2`) return list.get.all.v2();
|
|
1127
|
-
return list.get.all.v1();
|
|
1128
|
-
} catch (error: any) {
|
|
1129
|
-
throw new MastraError(
|
|
1130
|
-
{
|
|
1131
|
-
id: 'CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED',
|
|
1132
|
-
domain: ErrorDomain.STORAGE,
|
|
1133
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
1134
|
-
},
|
|
1135
|
-
error,
|
|
1136
|
-
);
|
|
1137
|
-
}
|
|
322
|
+
return this.stores.memory.saveMessages(args);
|
|
1138
323
|
}
|
|
1139
324
|
|
|
1140
|
-
async
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
}: {
|
|
1145
|
-
workflowName: string;
|
|
1146
|
-
runId: string;
|
|
1147
|
-
snapshot: WorkflowRunState;
|
|
1148
|
-
}): Promise<void> {
|
|
1149
|
-
try {
|
|
1150
|
-
const currentSnapshot = await this.load({
|
|
1151
|
-
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
1152
|
-
keys: { workflow_name: workflowName, run_id: runId },
|
|
1153
|
-
});
|
|
1154
|
-
|
|
1155
|
-
const now = new Date();
|
|
1156
|
-
const persisting = currentSnapshot
|
|
1157
|
-
? {
|
|
1158
|
-
...currentSnapshot,
|
|
1159
|
-
snapshot: JSON.stringify(snapshot),
|
|
1160
|
-
updatedAt: now.toISOString(),
|
|
1161
|
-
}
|
|
1162
|
-
: {
|
|
1163
|
-
workflow_name: workflowName,
|
|
1164
|
-
run_id: runId,
|
|
1165
|
-
snapshot: JSON.stringify(snapshot),
|
|
1166
|
-
createdAt: now.toISOString(),
|
|
1167
|
-
updatedAt: now.toISOString(),
|
|
1168
|
-
};
|
|
1169
|
-
|
|
1170
|
-
await this.db.insert({
|
|
1171
|
-
table: TABLE_WORKFLOW_SNAPSHOT,
|
|
1172
|
-
format: 'JSONEachRow',
|
|
1173
|
-
values: [persisting],
|
|
1174
|
-
clickhouse_settings: {
|
|
1175
|
-
// Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
|
|
1176
|
-
date_time_input_format: 'best_effort',
|
|
1177
|
-
use_client_time_zone: 1,
|
|
1178
|
-
output_format_json_quote_64bit_integers: 0,
|
|
1179
|
-
},
|
|
1180
|
-
});
|
|
1181
|
-
} catch (error: any) {
|
|
1182
|
-
throw new MastraError(
|
|
1183
|
-
{
|
|
1184
|
-
id: 'CLICKHOUSE_STORAGE_PERSIST_WORKFLOW_SNAPSHOT_FAILED',
|
|
1185
|
-
domain: ErrorDomain.STORAGE,
|
|
1186
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
1187
|
-
details: { workflowName, runId },
|
|
1188
|
-
},
|
|
1189
|
-
error,
|
|
1190
|
-
);
|
|
1191
|
-
}
|
|
325
|
+
async getMessagesPaginated(
|
|
326
|
+
args: StorageGetMessagesArg & { format?: 'v1' | 'v2' },
|
|
327
|
+
): Promise<PaginationInfo & { messages: MastraMessageV1[] | MastraMessageV2[] }> {
|
|
328
|
+
return this.stores.memory.getMessagesPaginated(args);
|
|
1192
329
|
}
|
|
1193
330
|
|
|
1194
|
-
async
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
}): Promise<
|
|
1201
|
-
|
|
1202
|
-
const result = await this.load({
|
|
1203
|
-
tableName: TABLE_WORKFLOW_SNAPSHOT,
|
|
1204
|
-
keys: {
|
|
1205
|
-
workflow_name: workflowName,
|
|
1206
|
-
run_id: runId,
|
|
1207
|
-
},
|
|
1208
|
-
});
|
|
1209
|
-
|
|
1210
|
-
if (!result) {
|
|
1211
|
-
return null;
|
|
1212
|
-
}
|
|
1213
|
-
|
|
1214
|
-
return (result as any).snapshot;
|
|
1215
|
-
} catch (error: any) {
|
|
1216
|
-
throw new MastraError(
|
|
1217
|
-
{
|
|
1218
|
-
id: 'CLICKHOUSE_STORAGE_LOAD_WORKFLOW_SNAPSHOT_FAILED',
|
|
1219
|
-
domain: ErrorDomain.STORAGE,
|
|
1220
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
1221
|
-
details: { workflowName, runId },
|
|
1222
|
-
},
|
|
1223
|
-
error,
|
|
1224
|
-
);
|
|
1225
|
-
}
|
|
331
|
+
async updateMessages(args: {
|
|
332
|
+
messages: (Partial<Omit<MastraMessageV2, 'createdAt'>> & {
|
|
333
|
+
id: string;
|
|
334
|
+
threadId?: string;
|
|
335
|
+
content?: { metadata?: MastraMessageContentV2['metadata']; content?: MastraMessageContentV2['content'] };
|
|
336
|
+
})[];
|
|
337
|
+
}): Promise<MastraMessageV2[]> {
|
|
338
|
+
return this.stores.memory.updateMessages(args);
|
|
1226
339
|
}
|
|
1227
340
|
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
try {
|
|
1232
|
-
parsedSnapshot = JSON.parse(row.snapshot as string) as WorkflowRunState;
|
|
1233
|
-
} catch (e) {
|
|
1234
|
-
// If parsing fails, return the raw snapshot string
|
|
1235
|
-
console.warn(`Failed to parse snapshot for workflow ${row.workflow_name}: ${e}`);
|
|
1236
|
-
}
|
|
1237
|
-
}
|
|
341
|
+
async getResourceById({ resourceId }: { resourceId: string }): Promise<StorageResourceType | null> {
|
|
342
|
+
return this.stores.memory.getResourceById({ resourceId });
|
|
343
|
+
}
|
|
1238
344
|
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
runId: row.run_id,
|
|
1242
|
-
snapshot: parsedSnapshot,
|
|
1243
|
-
createdAt: new Date(row.createdAt),
|
|
1244
|
-
updatedAt: new Date(row.updatedAt),
|
|
1245
|
-
resourceId: row.resourceId,
|
|
1246
|
-
};
|
|
345
|
+
async saveResource({ resource }: { resource: StorageResourceType }): Promise<StorageResourceType> {
|
|
346
|
+
return this.stores.memory.saveResource({ resource });
|
|
1247
347
|
}
|
|
1248
348
|
|
|
1249
|
-
async
|
|
1250
|
-
workflowName,
|
|
1251
|
-
fromDate,
|
|
1252
|
-
toDate,
|
|
1253
|
-
limit,
|
|
1254
|
-
offset,
|
|
349
|
+
async updateResource({
|
|
1255
350
|
resourceId,
|
|
351
|
+
workingMemory,
|
|
352
|
+
metadata,
|
|
1256
353
|
}: {
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
} = {}): Promise<WorkflowRuns> {
|
|
1264
|
-
try {
|
|
1265
|
-
const conditions: string[] = [];
|
|
1266
|
-
const values: Record<string, any> = {};
|
|
1267
|
-
|
|
1268
|
-
if (workflowName) {
|
|
1269
|
-
conditions.push(`workflow_name = {var_workflow_name:String}`);
|
|
1270
|
-
values.var_workflow_name = workflowName;
|
|
1271
|
-
}
|
|
1272
|
-
|
|
1273
|
-
if (resourceId) {
|
|
1274
|
-
const hasResourceId = await this.hasColumn(TABLE_WORKFLOW_SNAPSHOT, 'resourceId');
|
|
1275
|
-
if (hasResourceId) {
|
|
1276
|
-
conditions.push(`resourceId = {var_resourceId:String}`);
|
|
1277
|
-
values.var_resourceId = resourceId;
|
|
1278
|
-
} else {
|
|
1279
|
-
console.warn(`[${TABLE_WORKFLOW_SNAPSHOT}] resourceId column not found. Skipping resourceId filter.`);
|
|
1280
|
-
}
|
|
1281
|
-
}
|
|
1282
|
-
|
|
1283
|
-
if (fromDate) {
|
|
1284
|
-
conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
|
|
1285
|
-
values.var_from_date = fromDate.getTime() / 1000; // Convert to Unix timestamp
|
|
1286
|
-
}
|
|
1287
|
-
|
|
1288
|
-
if (toDate) {
|
|
1289
|
-
conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
|
|
1290
|
-
values.var_to_date = toDate.getTime() / 1000; // Convert to Unix timestamp
|
|
1291
|
-
}
|
|
1292
|
-
|
|
1293
|
-
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
1294
|
-
const limitClause = limit !== undefined ? `LIMIT ${limit}` : '';
|
|
1295
|
-
const offsetClause = offset !== undefined ? `OFFSET ${offset}` : '';
|
|
1296
|
-
|
|
1297
|
-
let total = 0;
|
|
1298
|
-
// Only get total count when using pagination
|
|
1299
|
-
if (limit !== undefined && offset !== undefined) {
|
|
1300
|
-
const countResult = await this.db.query({
|
|
1301
|
-
query: `SELECT COUNT(*) as count FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith('ReplacingMergeTree') ? 'FINAL' : ''} ${whereClause}`,
|
|
1302
|
-
query_params: values,
|
|
1303
|
-
format: 'JSONEachRow',
|
|
1304
|
-
});
|
|
1305
|
-
const countRows = await countResult.json();
|
|
1306
|
-
total = Number((countRows as Array<{ count: string | number }>)[0]?.count ?? 0);
|
|
1307
|
-
}
|
|
1308
|
-
|
|
1309
|
-
// Get results
|
|
1310
|
-
const result = await this.db.query({
|
|
1311
|
-
query: `
|
|
1312
|
-
SELECT
|
|
1313
|
-
workflow_name,
|
|
1314
|
-
run_id,
|
|
1315
|
-
snapshot,
|
|
1316
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
1317
|
-
toDateTime64(updatedAt, 3) as updatedAt,
|
|
1318
|
-
resourceId
|
|
1319
|
-
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith('ReplacingMergeTree') ? 'FINAL' : ''}
|
|
1320
|
-
${whereClause}
|
|
1321
|
-
ORDER BY createdAt DESC
|
|
1322
|
-
${limitClause}
|
|
1323
|
-
${offsetClause}
|
|
1324
|
-
`,
|
|
1325
|
-
query_params: values,
|
|
1326
|
-
format: 'JSONEachRow',
|
|
1327
|
-
});
|
|
354
|
+
resourceId: string;
|
|
355
|
+
workingMemory?: string;
|
|
356
|
+
metadata?: Record<string, unknown>;
|
|
357
|
+
}): Promise<StorageResourceType> {
|
|
358
|
+
return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
|
|
359
|
+
}
|
|
1328
360
|
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
return this.parseWorkflowRun(row);
|
|
1333
|
-
});
|
|
361
|
+
async getScoreById({ id }: { id: string }): Promise<ScoreRowData | null> {
|
|
362
|
+
return this.stores.scores.getScoreById({ id });
|
|
363
|
+
}
|
|
1334
364
|
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
} catch (error: any) {
|
|
1338
|
-
throw new MastraError(
|
|
1339
|
-
{
|
|
1340
|
-
id: 'CLICKHOUSE_STORAGE_GET_WORKFLOW_RUNS_FAILED',
|
|
1341
|
-
domain: ErrorDomain.STORAGE,
|
|
1342
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
1343
|
-
details: { workflowName: workflowName ?? '', resourceId: resourceId ?? '' },
|
|
1344
|
-
},
|
|
1345
|
-
error,
|
|
1346
|
-
);
|
|
1347
|
-
}
|
|
365
|
+
async saveScore(_score: ScoreRowData): Promise<{ score: ScoreRowData }> {
|
|
366
|
+
return this.stores.scores.saveScore(_score);
|
|
1348
367
|
}
|
|
1349
368
|
|
|
1350
|
-
async
|
|
369
|
+
async getScoresByRunId({
|
|
1351
370
|
runId,
|
|
1352
|
-
|
|
371
|
+
pagination,
|
|
1353
372
|
}: {
|
|
1354
373
|
runId: string;
|
|
1355
|
-
|
|
1356
|
-
}): Promise<
|
|
1357
|
-
|
|
1358
|
-
const conditions: string[] = [];
|
|
1359
|
-
const values: Record<string, any> = {};
|
|
1360
|
-
|
|
1361
|
-
if (runId) {
|
|
1362
|
-
conditions.push(`run_id = {var_runId:String}`);
|
|
1363
|
-
values.var_runId = runId;
|
|
1364
|
-
}
|
|
1365
|
-
|
|
1366
|
-
if (workflowName) {
|
|
1367
|
-
conditions.push(`workflow_name = {var_workflow_name:String}`);
|
|
1368
|
-
values.var_workflow_name = workflowName;
|
|
1369
|
-
}
|
|
1370
|
-
|
|
1371
|
-
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
1372
|
-
|
|
1373
|
-
// Get results
|
|
1374
|
-
const result = await this.db.query({
|
|
1375
|
-
query: `
|
|
1376
|
-
SELECT
|
|
1377
|
-
workflow_name,
|
|
1378
|
-
run_id,
|
|
1379
|
-
snapshot,
|
|
1380
|
-
toDateTime64(createdAt, 3) as createdAt,
|
|
1381
|
-
toDateTime64(updatedAt, 3) as updatedAt,
|
|
1382
|
-
resourceId
|
|
1383
|
-
FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith('ReplacingMergeTree') ? 'FINAL' : ''}
|
|
1384
|
-
${whereClause}
|
|
1385
|
-
`,
|
|
1386
|
-
query_params: values,
|
|
1387
|
-
format: 'JSONEachRow',
|
|
1388
|
-
});
|
|
1389
|
-
|
|
1390
|
-
const resultJson = await result.json();
|
|
1391
|
-
if (!Array.isArray(resultJson) || resultJson.length === 0) {
|
|
1392
|
-
return null;
|
|
1393
|
-
}
|
|
1394
|
-
return this.parseWorkflowRun(resultJson[0]);
|
|
1395
|
-
} catch (error: any) {
|
|
1396
|
-
throw new MastraError(
|
|
1397
|
-
{
|
|
1398
|
-
id: 'CLICKHOUSE_STORAGE_GET_WORKFLOW_RUN_BY_ID_FAILED',
|
|
1399
|
-
domain: ErrorDomain.STORAGE,
|
|
1400
|
-
category: ErrorCategory.THIRD_PARTY,
|
|
1401
|
-
details: { runId: runId ?? '', workflowName: workflowName ?? '' },
|
|
1402
|
-
},
|
|
1403
|
-
error,
|
|
1404
|
-
);
|
|
1405
|
-
}
|
|
1406
|
-
}
|
|
1407
|
-
|
|
1408
|
-
private async hasColumn(table: string, column: string): Promise<boolean> {
|
|
1409
|
-
const result = await this.db.query({
|
|
1410
|
-
query: `DESCRIBE TABLE ${table}`,
|
|
1411
|
-
format: 'JSONEachRow',
|
|
1412
|
-
});
|
|
1413
|
-
const columns = (await result.json()) as { name: string }[];
|
|
1414
|
-
return columns.some(c => c.name === column);
|
|
374
|
+
pagination: StoragePagination;
|
|
375
|
+
}): Promise<{ pagination: PaginationInfo; scores: ScoreRowData[] }> {
|
|
376
|
+
return this.stores.scores.getScoresByRunId({ runId, pagination });
|
|
1415
377
|
}
|
|
1416
378
|
|
|
1417
|
-
async
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
resourceId: string;
|
|
1428
|
-
page?: number;
|
|
1429
|
-
perPage?: number;
|
|
1430
|
-
}): Promise<PaginationInfo & { threads: StorageThreadType[] }> {
|
|
1431
|
-
throw new MastraError({
|
|
1432
|
-
id: 'CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED',
|
|
1433
|
-
domain: ErrorDomain.STORAGE,
|
|
1434
|
-
category: ErrorCategory.USER,
|
|
1435
|
-
text: 'Method not implemented.',
|
|
1436
|
-
});
|
|
379
|
+
async getScoresByEntityId({
|
|
380
|
+
entityId,
|
|
381
|
+
entityType,
|
|
382
|
+
pagination,
|
|
383
|
+
}: {
|
|
384
|
+
pagination: StoragePagination;
|
|
385
|
+
entityId: string;
|
|
386
|
+
entityType: string;
|
|
387
|
+
}): Promise<{ pagination: PaginationInfo; scores: ScoreRowData[] }> {
|
|
388
|
+
return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
|
|
1437
389
|
}
|
|
1438
390
|
|
|
1439
|
-
async
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
});
|
|
391
|
+
async getScoresByScorerId({
|
|
392
|
+
scorerId,
|
|
393
|
+
pagination,
|
|
394
|
+
}: {
|
|
395
|
+
scorerId: string;
|
|
396
|
+
pagination: StoragePagination;
|
|
397
|
+
}): Promise<{ pagination: PaginationInfo; scores: ScoreRowData[] }> {
|
|
398
|
+
return this.stores.scores.getScoresByScorerId({ scorerId, pagination });
|
|
1448
399
|
}
|
|
1449
400
|
|
|
1450
401
|
async close(): Promise<void> {
|
|
1451
402
|
await this.db.close();
|
|
1452
403
|
}
|
|
1453
|
-
|
|
1454
|
-
async updateMessages(_args: {
|
|
1455
|
-
messages: Partial<Omit<MastraMessageV2, 'createdAt'>> &
|
|
1456
|
-
{
|
|
1457
|
-
id: string;
|
|
1458
|
-
content?: { metadata?: MastraMessageContentV2['metadata']; content?: MastraMessageContentV2['content'] };
|
|
1459
|
-
}[];
|
|
1460
|
-
}): Promise<MastraMessageV2[]> {
|
|
1461
|
-
this.logger.error('updateMessages is not yet implemented in ClickhouseStore');
|
|
1462
|
-
throw new Error('Method not implemented');
|
|
1463
|
-
}
|
|
1464
404
|
}
|