@mastra/clickhouse 0.0.0-tsconfig-compile-20250703214351 → 0.0.0-unified-sidebar-20251010130811

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/CHANGELOG.md +524 -4
  2. package/LICENSE.md +12 -4
  3. package/dist/index.cjs +2278 -598
  4. package/dist/index.cjs.map +1 -0
  5. package/dist/index.d.ts +3 -4
  6. package/dist/index.d.ts.map +1 -0
  7. package/dist/index.js +2278 -598
  8. package/dist/index.js.map +1 -0
  9. package/dist/storage/domains/legacy-evals/index.d.ts +21 -0
  10. package/dist/storage/domains/legacy-evals/index.d.ts.map +1 -0
  11. package/dist/storage/domains/memory/index.d.ts +87 -0
  12. package/dist/storage/domains/memory/index.d.ts.map +1 -0
  13. package/dist/storage/domains/operations/index.d.ts +42 -0
  14. package/dist/storage/domains/operations/index.d.ts.map +1 -0
  15. package/dist/storage/domains/scores/index.d.ts +54 -0
  16. package/dist/storage/domains/scores/index.d.ts.map +1 -0
  17. package/dist/storage/domains/traces/index.d.ts +21 -0
  18. package/dist/storage/domains/traces/index.d.ts.map +1 -0
  19. package/dist/storage/domains/utils.d.ts +28 -0
  20. package/dist/storage/domains/utils.d.ts.map +1 -0
  21. package/dist/storage/domains/workflows/index.d.ts +55 -0
  22. package/dist/storage/domains/workflows/index.d.ts.map +1 -0
  23. package/dist/storage/index.d.ts +245 -0
  24. package/dist/storage/index.d.ts.map +1 -0
  25. package/package.json +25 -11
  26. package/dist/_tsup-dts-rollup.d.cts +0 -191
  27. package/dist/_tsup-dts-rollup.d.ts +0 -191
  28. package/dist/index.d.cts +0 -4
  29. package/docker-compose.yaml +0 -15
  30. package/eslint.config.js +0 -6
  31. package/src/index.ts +0 -1
  32. package/src/storage/index.test.ts +0 -1154
  33. package/src/storage/index.ts +0 -1464
  34. package/tsconfig.json +0 -5
  35. package/vitest.config.ts +0 -12
package/dist/index.js CHANGED
@@ -1,22 +1,20 @@
1
1
  import { createClient } from '@clickhouse/client';
2
- import { MessageList } from '@mastra/core/agent';
3
2
  import { MastraError, ErrorCategory, ErrorDomain } from '@mastra/core/error';
4
- import { TABLE_EVALS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, MastraStorage, TABLE_SCHEMAS } from '@mastra/core/storage';
3
+ import { TABLE_AI_SPANS, TABLE_RESOURCES, TABLE_SCORERS, TABLE_EVALS, TABLE_THREADS, TABLE_TRACES, TABLE_WORKFLOW_SNAPSHOT, TABLE_MESSAGES, MastraStorage, StoreOperations, TABLE_SCHEMAS, WorkflowsStorage, ScoresStorage, safelyParseJSON, LegacyEvalsStorage, TracesStorage, MemoryStorage, resolveMessageLimit } from '@mastra/core/storage';
4
+ import { MessageList } from '@mastra/core/agent';
5
+ import { saveScorePayloadSchema } from '@mastra/core/scores';
5
6
 
6
7
  // src/storage/index.ts
7
- function safelyParseJSON(jsonString) {
8
- try {
9
- return JSON.parse(jsonString);
10
- } catch {
11
- return {};
12
- }
13
- }
14
8
  var TABLE_ENGINES = {
15
9
  [TABLE_MESSAGES]: `MergeTree()`,
16
10
  [TABLE_WORKFLOW_SNAPSHOT]: `ReplacingMergeTree()`,
17
11
  [TABLE_TRACES]: `MergeTree()`,
18
12
  [TABLE_THREADS]: `ReplacingMergeTree()`,
19
- [TABLE_EVALS]: `MergeTree()`
13
+ [TABLE_EVALS]: `MergeTree()`,
14
+ [TABLE_SCORERS]: `MergeTree()`,
15
+ [TABLE_RESOURCES]: `ReplacingMergeTree()`,
16
+ // TODO: verify this is the correct engine for ai spans when implementing clickhouse storage
17
+ [TABLE_AI_SPANS]: `ReplacingMergeTree()`
20
18
  };
21
19
  var COLUMN_TYPES = {
22
20
  text: "String",
@@ -24,11 +22,10 @@ var COLUMN_TYPES = {
24
22
  uuid: "String",
25
23
  jsonb: "String",
26
24
  integer: "Int64",
27
- bigint: "Int64"
25
+ float: "Float64",
26
+ bigint: "Int64",
27
+ boolean: "Bool"
28
28
  };
29
- function transformRows(rows) {
30
- return rows.map((row) => transformRow(row));
31
- }
32
29
  function transformRow(row) {
33
30
  if (!row) {
34
31
  return row;
@@ -39,31 +36,56 @@ function transformRow(row) {
39
36
  if (row.updatedAt) {
40
37
  row.updatedAt = new Date(row.updatedAt);
41
38
  }
39
+ if (row.content && typeof row.content === "string") {
40
+ row.content = safelyParseJSON(row.content);
41
+ }
42
42
  return row;
43
43
  }
44
- var ClickhouseStore = class extends MastraStorage {
45
- db;
46
- ttl = {};
47
- constructor(config) {
48
- super({ name: "ClickhouseStore" });
49
- this.db = createClient({
50
- url: config.url,
51
- username: config.username,
52
- password: config.password,
53
- clickhouse_settings: {
54
- date_time_input_format: "best_effort",
55
- date_time_output_format: "iso",
56
- // This is crucial
57
- use_client_time_zone: 1,
58
- output_format_json_quote_64bit_integers: 0
59
- }
60
- });
61
- this.ttl = config.ttl;
44
+ function transformRows(rows) {
45
+ return rows.map((row) => transformRow(row));
46
+ }
47
+
48
+ // src/storage/domains/legacy-evals/index.ts
49
+ var LegacyEvalsStorageClickhouse = class extends LegacyEvalsStorage {
50
+ client;
51
+ operations;
52
+ constructor({ client, operations }) {
53
+ super();
54
+ this.client = client;
55
+ this.operations = operations;
62
56
  }
63
57
  transformEvalRow(row) {
64
58
  row = transformRow(row);
65
- const resultValue = JSON.parse(row.result);
66
- const testInfoValue = row.test_info ? JSON.parse(row.test_info) : void 0;
59
+ let resultValue;
60
+ try {
61
+ if (row.result && typeof row.result === "string" && row.result.trim() !== "") {
62
+ resultValue = JSON.parse(row.result);
63
+ } else if (typeof row.result === "object" && row.result !== null) {
64
+ resultValue = row.result;
65
+ } else if (row.result === null || row.result === void 0 || row.result === "") {
66
+ resultValue = { score: 0 };
67
+ } else {
68
+ throw new Error(`Invalid or empty result field: ${JSON.stringify(row.result)}`);
69
+ }
70
+ } catch (error) {
71
+ console.error("Error parsing result field:", row.result, error);
72
+ throw new MastraError({
73
+ id: "CLICKHOUSE_STORAGE_INVALID_RESULT_FORMAT",
74
+ text: `Invalid result format: ${JSON.stringify(row.result)}`,
75
+ domain: ErrorDomain.STORAGE,
76
+ category: ErrorCategory.USER
77
+ });
78
+ }
79
+ let testInfoValue;
80
+ try {
81
+ if (row.test_info && typeof row.test_info === "string" && row.test_info.trim() !== "" && row.test_info !== "null") {
82
+ testInfoValue = JSON.parse(row.test_info);
83
+ } else if (typeof row.test_info === "object" && row.test_info !== null) {
84
+ testInfoValue = row.test_info;
85
+ }
86
+ } catch {
87
+ testInfoValue = void 0;
88
+ }
67
89
  if (!resultValue || typeof resultValue !== "object" || !("score" in resultValue)) {
68
90
  throw new MastraError({
69
91
  id: "CLICKHOUSE_STORAGE_INVALID_METRIC_FORMAT",
@@ -85,23 +107,11 @@ var ClickhouseStore = class extends MastraStorage {
85
107
  createdAt: row.created_at
86
108
  };
87
109
  }
88
- escape(value) {
89
- if (typeof value === "string") {
90
- return `'${value.replace(/'/g, "''")}'`;
91
- }
92
- if (value instanceof Date) {
93
- return `'${value.toISOString()}'`;
94
- }
95
- if (value === null || value === void 0) {
96
- return "NULL";
97
- }
98
- return value.toString();
99
- }
100
110
  async getEvalsByAgentName(agentName, type) {
101
111
  try {
102
- const baseQuery = `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
103
- const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND JSONExtractString(test_info, 'testPath') IS NOT NULL" : type === "live" ? " AND (test_info IS NULL OR JSONExtractString(test_info, 'testPath') IS NULL)" : "";
104
- const result = await this.db.query({
112
+ const baseQuery = `SELECT *, toDateTime64(created_at, 3) as createdAt FROM ${TABLE_EVALS} WHERE agent_name = {var_agent_name:String}`;
113
+ const typeCondition = type === "test" ? " AND test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != ''" : type === "live" ? " AND (test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')" : "";
114
+ const result = await this.client.query({
105
115
  query: `${baseQuery}${typeCondition} ORDER BY createdAt DESC`,
106
116
  query_params: { var_agent_name: agentName },
107
117
  clickhouse_settings: {
@@ -131,388 +141,438 @@ var ClickhouseStore = class extends MastraStorage {
131
141
  );
132
142
  }
133
143
  }
134
- async batchInsert({ tableName, records }) {
135
- try {
136
- await this.db.insert({
137
- table: tableName,
138
- values: records.map((record) => ({
139
- ...Object.fromEntries(
140
- Object.entries(record).map(([key, value]) => [
141
- key,
142
- TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
143
- ])
144
- )
145
- })),
146
- format: "JSONEachRow",
147
- clickhouse_settings: {
148
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
149
- date_time_input_format: "best_effort",
150
- use_client_time_zone: 1,
151
- output_format_json_quote_64bit_integers: 0
152
- }
153
- });
154
- } catch (error) {
155
- throw new MastraError(
156
- {
157
- id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
158
- domain: ErrorDomain.STORAGE,
159
- category: ErrorCategory.THIRD_PARTY,
160
- details: { tableName }
161
- },
162
- error
163
- );
164
- }
165
- }
166
- async getTraces({
167
- name,
168
- scope,
169
- page,
170
- perPage,
171
- attributes,
172
- filters,
173
- fromDate,
174
- toDate
175
- }) {
176
- const limit = perPage;
177
- const offset = page * perPage;
178
- const args = {};
144
+ async getEvals(options = {}) {
145
+ const { agentName, type, page = 0, perPage = 100, dateRange } = options;
146
+ const fromDate = dateRange?.start;
147
+ const toDate = dateRange?.end;
179
148
  const conditions = [];
180
- if (name) {
181
- conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
182
- args.var_name = name;
183
- }
184
- if (scope) {
185
- conditions.push(`scope = {var_scope:String}`);
186
- args.var_scope = scope;
149
+ if (agentName) {
150
+ conditions.push(`agent_name = {var_agent_name:String}`);
187
151
  }
188
- if (attributes) {
189
- Object.entries(attributes).forEach(([key, value]) => {
190
- conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
191
- args[`var_attr_${key}`] = value;
192
- });
193
- }
194
- if (filters) {
195
- Object.entries(filters).forEach(([key, value]) => {
196
- conditions.push(
197
- `${key} = {var_col_${key}:${COLUMN_TYPES[TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"]}}`
198
- );
199
- args[`var_col_${key}`] = value;
200
- });
152
+ if (type === "test") {
153
+ conditions.push(
154
+ `(test_info IS NOT NULL AND test_info != 'null' AND JSONExtractString(test_info, 'testPath') IS NOT NULL AND JSONExtractString(test_info, 'testPath') != '')`
155
+ );
156
+ } else if (type === "live") {
157
+ conditions.push(
158
+ `(test_info IS NULL OR test_info = 'null' OR JSONExtractString(test_info, 'testPath') IS NULL OR JSONExtractString(test_info, 'testPath') = '')`
159
+ );
201
160
  }
202
161
  if (fromDate) {
203
- conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
204
- args.var_from_date = fromDate.getTime() / 1e3;
162
+ conditions.push(`created_at >= parseDateTime64BestEffort({var_from_date:String})`);
163
+ fromDate.toISOString();
205
164
  }
206
165
  if (toDate) {
207
- conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
208
- args.var_to_date = toDate.getTime() / 1e3;
166
+ conditions.push(`created_at <= parseDateTime64BestEffort({var_to_date:String})`);
167
+ toDate.toISOString();
209
168
  }
210
169
  const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
211
170
  try {
212
- const result = await this.db.query({
213
- query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
214
- query_params: args,
171
+ const countResult = await this.client.query({
172
+ query: `SELECT COUNT(*) as count FROM ${TABLE_EVALS} ${whereClause}`,
173
+ query_params: {
174
+ ...agentName ? { var_agent_name: agentName } : {},
175
+ ...fromDate ? { var_from_date: fromDate.toISOString() } : {},
176
+ ...toDate ? { var_to_date: toDate.toISOString() } : {}
177
+ },
215
178
  clickhouse_settings: {
216
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
217
179
  date_time_input_format: "best_effort",
218
180
  date_time_output_format: "iso",
219
181
  use_client_time_zone: 1,
220
182
  output_format_json_quote_64bit_integers: 0
221
183
  }
222
184
  });
223
- if (!result) {
224
- return [];
185
+ const countData = await countResult.json();
186
+ const total = Number(countData.data?.[0]?.count ?? 0);
187
+ const currentOffset = page * perPage;
188
+ const hasMore = currentOffset + perPage < total;
189
+ if (total === 0) {
190
+ return {
191
+ evals: [],
192
+ total: 0,
193
+ page,
194
+ perPage,
195
+ hasMore: false
196
+ };
225
197
  }
226
- const resp = await result.json();
227
- const rows = resp.data;
228
- return rows.map((row) => ({
229
- id: row.id,
230
- parentSpanId: row.parentSpanId,
231
- traceId: row.traceId,
232
- name: row.name,
233
- scope: row.scope,
234
- kind: row.kind,
235
- status: safelyParseJSON(row.status),
236
- events: safelyParseJSON(row.events),
237
- links: safelyParseJSON(row.links),
238
- attributes: safelyParseJSON(row.attributes),
239
- startTime: row.startTime,
240
- endTime: row.endTime,
241
- other: safelyParseJSON(row.other),
242
- createdAt: row.createdAt
243
- }));
198
+ const dataResult = await this.client.query({
199
+ query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_EVALS} ${whereClause} ORDER BY created_at DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
200
+ query_params: {
201
+ ...agentName ? { var_agent_name: agentName } : {},
202
+ ...fromDate ? { var_from_date: fromDate.toISOString() } : {},
203
+ ...toDate ? { var_to_date: toDate.toISOString() } : {},
204
+ var_limit: perPage || 100,
205
+ var_offset: currentOffset || 0
206
+ },
207
+ clickhouse_settings: {
208
+ date_time_input_format: "best_effort",
209
+ date_time_output_format: "iso",
210
+ use_client_time_zone: 1,
211
+ output_format_json_quote_64bit_integers: 0
212
+ }
213
+ });
214
+ const rows = await dataResult.json();
215
+ return {
216
+ evals: rows.data.map((row) => this.transformEvalRow(row)),
217
+ total,
218
+ page,
219
+ perPage,
220
+ hasMore
221
+ };
244
222
  } catch (error) {
245
223
  if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
246
- return [];
224
+ return {
225
+ evals: [],
226
+ total: 0,
227
+ page,
228
+ perPage,
229
+ hasMore: false
230
+ };
247
231
  }
248
232
  throw new MastraError(
249
233
  {
250
- id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
234
+ id: "CLICKHOUSE_STORAGE_GET_EVALS_FAILED",
251
235
  domain: ErrorDomain.STORAGE,
252
236
  category: ErrorCategory.THIRD_PARTY,
253
- details: {
254
- name: name ?? null,
255
- scope: scope ?? null,
256
- page,
257
- perPage,
258
- attributes: attributes ? JSON.stringify(attributes) : null,
259
- filters: filters ? JSON.stringify(filters) : null,
260
- fromDate: fromDate?.toISOString() ?? null,
261
- toDate: toDate?.toISOString() ?? null
262
- }
237
+ details: { agentName: agentName ?? "all", type: type ?? "all" }
263
238
  },
264
239
  error
265
240
  );
266
241
  }
267
242
  }
268
- async optimizeTable({ tableName }) {
243
+ };
244
+ var MemoryStorageClickhouse = class extends MemoryStorage {
245
+ client;
246
+ operations;
247
+ constructor({ client, operations }) {
248
+ super();
249
+ this.client = client;
250
+ this.operations = operations;
251
+ }
252
+ async getMessages({
253
+ threadId,
254
+ resourceId,
255
+ selectBy,
256
+ format
257
+ }) {
269
258
  try {
270
- await this.db.command({
271
- query: `OPTIMIZE TABLE ${tableName} FINAL`
259
+ if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
260
+ const messages = [];
261
+ const limit = resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
262
+ const include = selectBy?.include || [];
263
+ if (include.length) {
264
+ const unionQueries = [];
265
+ const params = [];
266
+ let paramIdx = 1;
267
+ for (const inc of include) {
268
+ const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
269
+ const searchId = inc.threadId || threadId;
270
+ unionQueries.push(`
271
+ SELECT * FROM (
272
+ WITH numbered_messages AS (
273
+ SELECT
274
+ id, content, role, type, "createdAt", thread_id, "resourceId",
275
+ ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
276
+ FROM "${TABLE_MESSAGES}"
277
+ WHERE thread_id = {var_thread_id_${paramIdx}:String}
278
+ ),
279
+ target_positions AS (
280
+ SELECT row_num as target_pos
281
+ FROM numbered_messages
282
+ WHERE id = {var_include_id_${paramIdx}:String}
283
+ )
284
+ SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
285
+ FROM numbered_messages m
286
+ CROSS JOIN target_positions t
287
+ WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
288
+ ) AS query_${paramIdx}
289
+ `);
290
+ params.push(
291
+ { [`var_thread_id_${paramIdx}`]: searchId },
292
+ { [`var_include_id_${paramIdx}`]: id },
293
+ { [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
294
+ { [`var_withNextMessages_${paramIdx}`]: withNextMessages }
295
+ );
296
+ paramIdx++;
297
+ }
298
+ const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
299
+ const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
300
+ const includeResult = await this.client.query({
301
+ query: finalQuery,
302
+ query_params: mergedParams,
303
+ clickhouse_settings: {
304
+ date_time_input_format: "best_effort",
305
+ date_time_output_format: "iso",
306
+ use_client_time_zone: 1,
307
+ output_format_json_quote_64bit_integers: 0
308
+ }
309
+ });
310
+ const rows2 = await includeResult.json();
311
+ const includedMessages = transformRows(rows2.data);
312
+ const seen = /* @__PURE__ */ new Set();
313
+ const dedupedMessages = includedMessages.filter((message) => {
314
+ if (seen.has(message.id)) return false;
315
+ seen.add(message.id);
316
+ return true;
317
+ });
318
+ messages.push(...dedupedMessages);
319
+ }
320
+ const result = await this.client.query({
321
+ query: `
322
+ SELECT
323
+ id,
324
+ content,
325
+ role,
326
+ type,
327
+ toDateTime64(createdAt, 3) as createdAt,
328
+ thread_id AS "threadId"
329
+ FROM "${TABLE_MESSAGES}"
330
+ WHERE thread_id = {threadId:String}
331
+ AND id NOT IN ({exclude:Array(String)})
332
+ ORDER BY "createdAt" DESC
333
+ LIMIT {limit:Int64}
334
+ `,
335
+ query_params: {
336
+ threadId,
337
+ exclude: messages.map((m) => m.id),
338
+ limit
339
+ },
340
+ clickhouse_settings: {
341
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
342
+ date_time_input_format: "best_effort",
343
+ date_time_output_format: "iso",
344
+ use_client_time_zone: 1,
345
+ output_format_json_quote_64bit_integers: 0
346
+ }
347
+ });
348
+ const rows = await result.json();
349
+ messages.push(...transformRows(rows.data));
350
+ messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
351
+ messages.forEach((message) => {
352
+ if (typeof message.content === "string") {
353
+ try {
354
+ message.content = JSON.parse(message.content);
355
+ } catch {
356
+ }
357
+ }
272
358
  });
359
+ const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
360
+ if (format === `v2`) return list.get.all.v2();
361
+ return list.get.all.v1();
273
362
  } catch (error) {
274
363
  throw new MastraError(
275
364
  {
276
- id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
365
+ id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
277
366
  domain: ErrorDomain.STORAGE,
278
367
  category: ErrorCategory.THIRD_PARTY,
279
- details: { tableName }
368
+ details: { threadId, resourceId: resourceId ?? "" }
280
369
  },
281
370
  error
282
371
  );
283
372
  }
284
373
  }
285
- async materializeTtl({ tableName }) {
374
+ async getMessagesById({
375
+ messageIds,
376
+ format
377
+ }) {
378
+ if (messageIds.length === 0) return [];
286
379
  try {
287
- await this.db.command({
288
- query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
380
+ const result = await this.client.query({
381
+ query: `
382
+ SELECT
383
+ id,
384
+ content,
385
+ role,
386
+ type,
387
+ toDateTime64(createdAt, 3) as createdAt,
388
+ thread_id AS "threadId",
389
+ "resourceId"
390
+ FROM "${TABLE_MESSAGES}"
391
+ WHERE id IN {messageIds:Array(String)}
392
+ ORDER BY "createdAt" DESC
393
+ `,
394
+ query_params: {
395
+ messageIds
396
+ },
397
+ clickhouse_settings: {
398
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
399
+ date_time_input_format: "best_effort",
400
+ date_time_output_format: "iso",
401
+ use_client_time_zone: 1,
402
+ output_format_json_quote_64bit_integers: 0
403
+ }
404
+ });
405
+ const rows = await result.json();
406
+ const messages = transformRows(rows.data);
407
+ messages.forEach((message) => {
408
+ if (typeof message.content === "string") {
409
+ try {
410
+ message.content = JSON.parse(message.content);
411
+ } catch {
412
+ }
413
+ }
289
414
  });
415
+ const list = new MessageList().add(messages, "memory");
416
+ if (format === `v1`) return list.get.all.v1();
417
+ return list.get.all.v2();
290
418
  } catch (error) {
291
419
  throw new MastraError(
292
420
  {
293
- id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
421
+ id: "CLICKHOUSE_STORAGE_GET_MESSAGES_BY_ID_FAILED",
294
422
  domain: ErrorDomain.STORAGE,
295
423
  category: ErrorCategory.THIRD_PARTY,
296
- details: { tableName }
424
+ details: { messageIds: JSON.stringify(messageIds) }
297
425
  },
298
426
  error
299
427
  );
300
428
  }
301
429
  }
302
- async createTable({
303
- tableName,
304
- schema
305
- }) {
306
- try {
307
- const columns = Object.entries(schema).map(([name, def]) => {
308
- const constraints = [];
309
- if (!def.nullable) constraints.push("NOT NULL");
310
- const columnTtl = this.ttl?.[tableName]?.columns?.[name];
311
- return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
312
- }).join(",\n");
313
- const rowTtl = this.ttl?.[tableName]?.row;
314
- const sql = tableName === TABLE_WORKFLOW_SNAPSHOT ? `
315
- CREATE TABLE IF NOT EXISTS ${tableName} (
316
- ${["id String"].concat(columns)}
317
- )
318
- ENGINE = ${TABLE_ENGINES[tableName]}
319
- PRIMARY KEY (createdAt, run_id, workflow_name)
320
- ORDER BY (createdAt, run_id, workflow_name)
321
- ${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
322
- SETTINGS index_granularity = 8192
323
- ` : `
324
- CREATE TABLE IF NOT EXISTS ${tableName} (
325
- ${columns}
326
- )
327
- ENGINE = ${TABLE_ENGINES[tableName]}
328
- PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
329
- ORDER BY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
330
- ${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
331
- SETTINGS index_granularity = 8192
332
- `;
333
- await this.db.query({
334
- query: sql,
335
- clickhouse_settings: {
336
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
337
- date_time_input_format: "best_effort",
338
- date_time_output_format: "iso",
339
- use_client_time_zone: 1,
340
- output_format_json_quote_64bit_integers: 0
341
- }
342
- });
343
- } catch (error) {
344
- throw new MastraError(
345
- {
346
- id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
347
- domain: ErrorDomain.STORAGE,
348
- category: ErrorCategory.THIRD_PARTY,
349
- details: { tableName }
350
- },
351
- error
352
- );
353
- }
354
- }
355
- getSqlType(type) {
356
- switch (type) {
357
- case "text":
358
- return "String";
359
- case "timestamp":
360
- return "DateTime64(3)";
361
- case "integer":
362
- case "bigint":
363
- return "Int64";
364
- case "jsonb":
365
- return "String";
366
- default:
367
- return super.getSqlType(type);
368
- }
369
- }
370
- /**
371
- * Alters table schema to add columns if they don't exist
372
- * @param tableName Name of the table
373
- * @param schema Schema of the table
374
- * @param ifNotExists Array of column names to add if they don't exist
375
- */
376
- async alterTable({
377
- tableName,
378
- schema,
379
- ifNotExists
380
- }) {
381
- try {
382
- const describeSql = `DESCRIBE TABLE ${tableName}`;
383
- const result = await this.db.query({
384
- query: describeSql
385
- });
386
- const rows = await result.json();
387
- const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
388
- for (const columnName of ifNotExists) {
389
- if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
390
- const columnDef = schema[columnName];
391
- let sqlType = this.getSqlType(columnDef.type);
392
- if (columnDef.nullable !== false) {
393
- sqlType = `Nullable(${sqlType})`;
394
- }
395
- const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
396
- const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
397
- await this.db.query({
398
- query: alterSql
399
- });
400
- this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
401
- }
430
+ async saveMessages(args) {
431
+ const { messages, format = "v1" } = args;
432
+ if (messages.length === 0) return messages;
433
+ for (const message of messages) {
434
+ const resourceId = message.resourceId;
435
+ if (!resourceId) {
436
+ throw new Error("Resource ID is required");
437
+ }
438
+ if (!message.threadId) {
439
+ throw new Error("Thread ID is required");
440
+ }
441
+ const thread = await this.getThreadById({ threadId: message.threadId });
442
+ if (!thread) {
443
+ throw new Error(`Thread ${message.threadId} not found`);
402
444
  }
403
- } catch (error) {
404
- throw new MastraError(
405
- {
406
- id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
407
- domain: ErrorDomain.STORAGE,
408
- category: ErrorCategory.THIRD_PARTY,
409
- details: { tableName }
410
- },
411
- error
412
- );
413
445
  }
414
- }
415
- async clearTable({ tableName }) {
446
+ const threadIdSet = /* @__PURE__ */ new Map();
447
+ await Promise.all(
448
+ messages.map(async (m) => {
449
+ const resourceId = m.resourceId;
450
+ if (!resourceId) {
451
+ throw new Error("Resource ID is required");
452
+ }
453
+ if (!m.threadId) {
454
+ throw new Error("Thread ID is required");
455
+ }
456
+ const thread = await this.getThreadById({ threadId: m.threadId });
457
+ if (!thread) {
458
+ throw new Error(`Thread ${m.threadId} not found`);
459
+ }
460
+ threadIdSet.set(m.threadId, thread);
461
+ })
462
+ );
416
463
  try {
417
- await this.db.query({
418
- query: `TRUNCATE TABLE ${tableName}`,
464
+ const existingResult = await this.client.query({
465
+ query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
466
+ query_params: {
467
+ ids: messages.map((m) => m.id)
468
+ },
419
469
  clickhouse_settings: {
420
470
  // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
421
471
  date_time_input_format: "best_effort",
422
472
  date_time_output_format: "iso",
423
473
  use_client_time_zone: 1,
424
474
  output_format_json_quote_64bit_integers: 0
425
- }
426
- });
427
- } catch (error) {
428
- throw new MastraError(
429
- {
430
- id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
431
- domain: ErrorDomain.STORAGE,
432
- category: ErrorCategory.THIRD_PARTY,
433
- details: { tableName }
434
475
  },
435
- error
436
- );
437
- }
438
- }
439
- async insert({ tableName, record }) {
440
- try {
441
- await this.db.insert({
442
- table: tableName,
443
- values: [
444
- {
445
- ...record,
446
- createdAt: record.createdAt.toISOString(),
447
- updatedAt: record.updatedAt.toISOString()
476
+ format: "JSONEachRow"
477
+ });
478
+ const existingRows = await existingResult.json();
479
+ const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
480
+ const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${m.threadId}`));
481
+ const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${m.threadId}`));
482
+ const toMove = messages.filter((m) => {
483
+ const existingRow = existingRows.find((row) => row.id === m.id);
484
+ return existingRow && existingRow.thread_id !== m.threadId;
485
+ });
486
+ const deletePromises = toMove.map((message) => {
487
+ const existingRow = existingRows.find((row) => row.id === message.id);
488
+ if (!existingRow) return Promise.resolve();
489
+ return this.client.command({
490
+ query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {var_id:String} AND thread_id = {var_old_thread_id:String}`,
491
+ query_params: {
492
+ var_id: message.id,
493
+ var_old_thread_id: existingRow.thread_id
494
+ },
495
+ clickhouse_settings: {
496
+ date_time_input_format: "best_effort",
497
+ use_client_time_zone: 1,
498
+ output_format_json_quote_64bit_integers: 0
448
499
  }
449
- ],
450
- format: "JSONEachRow",
451
- clickhouse_settings: {
452
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
453
- output_format_json_quote_64bit_integers: 0,
454
- date_time_input_format: "best_effort",
455
- use_client_time_zone: 1
456
- }
500
+ });
457
501
  });
458
- } catch (error) {
459
- throw new MastraError(
460
- {
461
- id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
462
- domain: ErrorDomain.STORAGE,
463
- category: ErrorCategory.THIRD_PARTY,
464
- details: { tableName }
465
- },
466
- error
502
+ const updatePromises = toUpdate.map(
503
+ (message) => this.client.command({
504
+ query: `
505
+ ALTER TABLE ${TABLE_MESSAGES}
506
+ UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}, resourceId = {var_resourceId:String}
507
+ WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
508
+ `,
509
+ query_params: {
510
+ var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
511
+ var_role: message.role,
512
+ var_type: message.type || "v2",
513
+ var_resourceId: message.resourceId,
514
+ var_id: message.id,
515
+ var_thread_id: message.threadId
516
+ },
517
+ clickhouse_settings: {
518
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
519
+ date_time_input_format: "best_effort",
520
+ use_client_time_zone: 1,
521
+ output_format_json_quote_64bit_integers: 0
522
+ }
523
+ })
467
524
  );
468
- }
469
- }
470
- async load({
471
- tableName,
472
- keys
473
- }) {
474
- try {
475
- const keyEntries = Object.entries(keys);
476
- const conditions = keyEntries.map(
477
- ([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
478
- ).join(" AND ");
479
- const values = keyEntries.reduce((acc, [key, value]) => {
480
- return { ...acc, [`var_${key}`]: value };
481
- }, {});
482
- const result = await this.db.query({
483
- query: `SELECT *, toDateTime64(createdAt, 3) as createdAt, toDateTime64(updatedAt, 3) as updatedAt FROM ${tableName} ${TABLE_ENGINES[tableName].startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions}`,
484
- query_params: values,
485
- clickhouse_settings: {
486
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
487
- date_time_input_format: "best_effort",
488
- date_time_output_format: "iso",
489
- use_client_time_zone: 1,
490
- output_format_json_quote_64bit_integers: 0
491
- }
492
- });
493
- if (!result) {
494
- return null;
495
- }
496
- const rows = await result.json();
497
- if (tableName === TABLE_WORKFLOW_SNAPSHOT) {
498
- const snapshot = rows.data[0];
499
- if (!snapshot) {
500
- return null;
501
- }
502
- if (typeof snapshot.snapshot === "string") {
503
- snapshot.snapshot = JSON.parse(snapshot.snapshot);
504
- }
505
- return transformRow(snapshot);
506
- }
507
- const data = transformRow(rows.data[0]);
508
- return data;
525
+ await Promise.all([
526
+ // Insert new messages (including moved messages)
527
+ this.client.insert({
528
+ table: TABLE_MESSAGES,
529
+ format: "JSONEachRow",
530
+ values: toInsert.map((message) => ({
531
+ id: message.id,
532
+ thread_id: message.threadId,
533
+ resourceId: message.resourceId,
534
+ content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
535
+ createdAt: message.createdAt.toISOString(),
536
+ role: message.role,
537
+ type: message.type || "v2"
538
+ })),
539
+ clickhouse_settings: {
540
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
541
+ date_time_input_format: "best_effort",
542
+ use_client_time_zone: 1,
543
+ output_format_json_quote_64bit_integers: 0
544
+ }
545
+ }),
546
+ ...updatePromises,
547
+ ...deletePromises,
548
+ // Update thread's updatedAt timestamp
549
+ this.client.insert({
550
+ table: TABLE_THREADS,
551
+ format: "JSONEachRow",
552
+ values: Array.from(threadIdSet.values()).map((thread) => ({
553
+ id: thread.id,
554
+ resourceId: thread.resourceId,
555
+ title: thread.title,
556
+ metadata: thread.metadata,
557
+ createdAt: thread.createdAt,
558
+ updatedAt: (/* @__PURE__ */ new Date()).toISOString()
559
+ })),
560
+ clickhouse_settings: {
561
+ date_time_input_format: "best_effort",
562
+ use_client_time_zone: 1,
563
+ output_format_json_quote_64bit_integers: 0
564
+ }
565
+ })
566
+ ]);
567
+ const list = new MessageList().add(messages, "memory");
568
+ if (format === `v2`) return list.get.all.v2();
569
+ return list.get.all.v1();
509
570
  } catch (error) {
510
571
  throw new MastraError(
511
572
  {
512
- id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
573
+ id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
513
574
  domain: ErrorDomain.STORAGE,
514
- category: ErrorCategory.THIRD_PARTY,
515
- details: { tableName }
575
+ category: ErrorCategory.THIRD_PARTY
516
576
  },
517
577
  error
518
578
  );
@@ -520,7 +580,7 @@ var ClickhouseStore = class extends MastraStorage {
520
580
  }
521
581
  async getThreadById({ threadId }) {
522
582
  try {
523
- const result = await this.db.query({
583
+ const result = await this.client.query({
524
584
  query: `SELECT
525
585
  id,
526
586
  "resourceId",
@@ -565,7 +625,7 @@ var ClickhouseStore = class extends MastraStorage {
565
625
  }
566
626
  async getThreadsByResourceId({ resourceId }) {
567
627
  try {
568
- const result = await this.db.query({
628
+ const result = await this.client.query({
569
629
  query: `SELECT
570
630
  id,
571
631
  "resourceId",
@@ -606,7 +666,7 @@ var ClickhouseStore = class extends MastraStorage {
606
666
  }
607
667
  async saveThread({ thread }) {
608
668
  try {
609
- await this.db.insert({
669
+ await this.client.insert({
610
670
  table: TABLE_THREADS,
611
671
  values: [
612
672
  {
@@ -656,7 +716,7 @@ var ClickhouseStore = class extends MastraStorage {
656
716
  metadata: mergedMetadata,
657
717
  updatedAt: /* @__PURE__ */ new Date()
658
718
  };
659
- await this.db.insert({
719
+ await this.client.insert({
660
720
  table: TABLE_THREADS,
661
721
  format: "JSONEachRow",
662
722
  values: [
@@ -690,14 +750,14 @@ var ClickhouseStore = class extends MastraStorage {
690
750
  }
691
751
  async deleteThread({ threadId }) {
692
752
  try {
693
- await this.db.command({
753
+ await this.client.command({
694
754
  query: `DELETE FROM "${TABLE_MESSAGES}" WHERE thread_id = {var_thread_id:String};`,
695
755
  query_params: { var_thread_id: threadId },
696
756
  clickhouse_settings: {
697
757
  output_format_json_quote_64bit_integers: 0
698
758
  }
699
759
  });
700
- await this.db.command({
760
+ await this.client.command({
701
761
  query: `DELETE FROM "${TABLE_THREADS}" WHERE id = {var_id:String};`,
702
762
  query_params: { var_id: threadId },
703
763
  clickhouse_settings: {
@@ -716,254 +776,1644 @@ var ClickhouseStore = class extends MastraStorage {
716
776
  );
717
777
  }
718
778
  }
719
- async getMessages({
720
- threadId,
721
- resourceId,
722
- selectBy,
723
- format
724
- }) {
779
+ async getThreadsByResourceIdPaginated(args) {
780
+ const { resourceId, page = 0, perPage = 100 } = args;
725
781
  try {
726
- const messages = [];
727
- const limit = this.resolveMessageLimit({ last: selectBy?.last, defaultLimit: 40 });
728
- const include = selectBy?.include || [];
729
- if (include.length) {
730
- const includeResult = await this.db.query({
731
- query: `
732
- WITH ordered_messages AS (
733
- SELECT
734
- *,
735
- toDateTime64(createdAt, 3) as createdAt,
736
- toDateTime64(updatedAt, 3) as updatedAt,
737
- ROW_NUMBER() OVER (ORDER BY "createdAt" DESC) as row_num
738
- FROM "${TABLE_MESSAGES}"
739
- WHERE thread_id = {var_thread_id:String}
740
- )
741
- SELECT
742
- m.id AS id,
743
- m.content as content,
744
- m.role as role,
745
- m.type as type,
746
- m.createdAt as createdAt,
747
- m.updatedAt as updatedAt,
748
- m.thread_id AS "threadId"
749
- FROM ordered_messages m
750
- WHERE m.id = ANY({var_include:Array(String)})
751
- OR EXISTS (
752
- SELECT 1 FROM ordered_messages target
753
- WHERE target.id = ANY({var_include:Array(String)})
754
- AND (
755
- -- Get previous messages based on the max withPreviousMessages
756
- (m.row_num <= target.row_num + {var_withPreviousMessages:Int64} AND m.row_num > target.row_num)
757
- OR
758
- -- Get next messages based on the max withNextMessages
759
- (m.row_num >= target.row_num - {var_withNextMessages:Int64} AND m.row_num < target.row_num)
760
- )
761
- )
762
- ORDER BY m."createdAt" DESC
763
- `,
764
- query_params: {
765
- var_thread_id: threadId,
766
- var_include: include.map((i) => i.id),
767
- var_withPreviousMessages: Math.max(...include.map((i) => i.withPreviousMessages || 0)),
768
- var_withNextMessages: Math.max(...include.map((i) => i.withNextMessages || 0))
769
- },
770
- clickhouse_settings: {
771
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
772
- date_time_input_format: "best_effort",
782
+ const currentOffset = page * perPage;
783
+ const countResult = await this.client.query({
784
+ query: `SELECT count() as total FROM ${TABLE_THREADS} WHERE resourceId = {resourceId:String}`,
785
+ query_params: { resourceId },
786
+ clickhouse_settings: {
787
+ date_time_input_format: "best_effort",
788
+ date_time_output_format: "iso",
789
+ use_client_time_zone: 1,
790
+ output_format_json_quote_64bit_integers: 0
791
+ }
792
+ });
793
+ const countData = await countResult.json();
794
+ const total = countData.data[0].total;
795
+ if (total === 0) {
796
+ return {
797
+ threads: [],
798
+ total: 0,
799
+ page,
800
+ perPage,
801
+ hasMore: false
802
+ };
803
+ }
804
+ const dataResult = await this.client.query({
805
+ query: `
806
+ SELECT
807
+ id,
808
+ resourceId,
809
+ title,
810
+ metadata,
811
+ toDateTime64(createdAt, 3) as createdAt,
812
+ toDateTime64(updatedAt, 3) as updatedAt
813
+ FROM ${TABLE_THREADS}
814
+ WHERE resourceId = {resourceId:String}
815
+ ORDER BY createdAt DESC
816
+ LIMIT {limit:Int64} OFFSET {offset:Int64}
817
+ `,
818
+ query_params: {
819
+ resourceId,
820
+ limit: perPage,
821
+ offset: currentOffset
822
+ },
823
+ clickhouse_settings: {
824
+ date_time_input_format: "best_effort",
825
+ date_time_output_format: "iso",
826
+ use_client_time_zone: 1,
827
+ output_format_json_quote_64bit_integers: 0
828
+ }
829
+ });
830
+ const rows = await dataResult.json();
831
+ const threads = transformRows(rows.data);
832
+ return {
833
+ threads,
834
+ total,
835
+ page,
836
+ perPage,
837
+ hasMore: currentOffset + threads.length < total
838
+ };
839
+ } catch (error) {
840
+ throw new MastraError(
841
+ {
842
+ id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
843
+ domain: ErrorDomain.STORAGE,
844
+ category: ErrorCategory.THIRD_PARTY,
845
+ details: { resourceId, page }
846
+ },
847
+ error
848
+ );
849
+ }
850
+ }
851
+ async getMessagesPaginated(args) {
852
+ const { threadId, resourceId, selectBy, format = "v1" } = args;
853
+ const page = selectBy?.pagination?.page || 0;
854
+ const perPageInput = selectBy?.pagination?.perPage;
855
+ const perPage = perPageInput !== void 0 ? perPageInput : resolveMessageLimit({ last: selectBy?.last, defaultLimit: 20 });
856
+ try {
857
+ if (!threadId.trim()) throw new Error("threadId must be a non-empty string");
858
+ const offset = page * perPage;
859
+ const dateRange = selectBy?.pagination?.dateRange;
860
+ const fromDate = dateRange?.start;
861
+ const toDate = dateRange?.end;
862
+ const messages = [];
863
+ if (selectBy?.include?.length) {
864
+ const include = selectBy.include;
865
+ const unionQueries = [];
866
+ const params = [];
867
+ let paramIdx = 1;
868
+ for (const inc of include) {
869
+ const { id, withPreviousMessages = 0, withNextMessages = 0 } = inc;
870
+ const searchId = inc.threadId || threadId;
871
+ unionQueries.push(`
872
+ SELECT * FROM (
873
+ WITH numbered_messages AS (
874
+ SELECT
875
+ id, content, role, type, "createdAt", thread_id, "resourceId",
876
+ ROW_NUMBER() OVER (ORDER BY "createdAt" ASC) as row_num
877
+ FROM "${TABLE_MESSAGES}"
878
+ WHERE thread_id = {var_thread_id_${paramIdx}:String}
879
+ ),
880
+ target_positions AS (
881
+ SELECT row_num as target_pos
882
+ FROM numbered_messages
883
+ WHERE id = {var_include_id_${paramIdx}:String}
884
+ )
885
+ SELECT DISTINCT m.id, m.content, m.role, m.type, m."createdAt", m.thread_id AS "threadId"
886
+ FROM numbered_messages m
887
+ CROSS JOIN target_positions t
888
+ WHERE m.row_num BETWEEN (t.target_pos - {var_withPreviousMessages_${paramIdx}:Int64}) AND (t.target_pos + {var_withNextMessages_${paramIdx}:Int64})
889
+ ) AS query_${paramIdx}
890
+ `);
891
+ params.push(
892
+ { [`var_thread_id_${paramIdx}`]: searchId },
893
+ { [`var_include_id_${paramIdx}`]: id },
894
+ { [`var_withPreviousMessages_${paramIdx}`]: withPreviousMessages },
895
+ { [`var_withNextMessages_${paramIdx}`]: withNextMessages }
896
+ );
897
+ paramIdx++;
898
+ }
899
+ const finalQuery = unionQueries.join(" UNION ALL ") + ' ORDER BY "createdAt" DESC';
900
+ const mergedParams = params.reduce((acc, paramObj) => ({ ...acc, ...paramObj }), {});
901
+ const includeResult = await this.client.query({
902
+ query: finalQuery,
903
+ query_params: mergedParams,
904
+ clickhouse_settings: {
905
+ date_time_input_format: "best_effort",
773
906
  date_time_output_format: "iso",
774
907
  use_client_time_zone: 1,
775
908
  output_format_json_quote_64bit_integers: 0
776
909
  }
777
910
  });
778
911
  const rows2 = await includeResult.json();
779
- messages.push(...transformRows(rows2.data));
912
+ const includedMessages = transformRows(rows2.data);
913
+ const seen = /* @__PURE__ */ new Set();
914
+ const dedupedMessages = includedMessages.filter((message) => {
915
+ if (seen.has(message.id)) return false;
916
+ seen.add(message.id);
917
+ return true;
918
+ });
919
+ messages.push(...dedupedMessages);
780
920
  }
781
- const result = await this.db.query({
782
- query: `
783
- SELECT
784
- id,
785
- content,
786
- role,
787
- type,
788
- toDateTime64(createdAt, 3) as createdAt,
789
- thread_id AS "threadId"
790
- FROM "${TABLE_MESSAGES}"
791
- WHERE thread_id = {threadId:String}
792
- AND id NOT IN ({exclude:Array(String)})
793
- ORDER BY "createdAt" DESC
794
- LIMIT {limit:Int64}
795
- `,
796
- query_params: {
797
- threadId,
798
- exclude: messages.map((m) => m.id),
799
- limit
800
- },
921
+ let countQuery = `SELECT count() as total FROM ${TABLE_MESSAGES} WHERE thread_id = {threadId:String}`;
922
+ const countParams = { threadId };
923
+ if (fromDate) {
924
+ countQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
925
+ countParams.fromDate = fromDate.toISOString();
926
+ }
927
+ if (toDate) {
928
+ countQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
929
+ countParams.toDate = toDate.toISOString();
930
+ }
931
+ const countResult = await this.client.query({
932
+ query: countQuery,
933
+ query_params: countParams,
801
934
  clickhouse_settings: {
802
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
803
935
  date_time_input_format: "best_effort",
804
936
  date_time_output_format: "iso",
805
937
  use_client_time_zone: 1,
806
938
  output_format_json_quote_64bit_integers: 0
807
939
  }
808
940
  });
809
- const rows = await result.json();
810
- messages.push(...transformRows(rows.data));
811
- messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
812
- messages.forEach((message) => {
813
- if (typeof message.content === "string") {
814
- try {
815
- message.content = JSON.parse(message.content);
816
- } catch {
817
- }
941
+ const countData = await countResult.json();
942
+ const total = countData.data[0].total;
943
+ if (total === 0 && messages.length === 0) {
944
+ return {
945
+ messages: [],
946
+ total: 0,
947
+ page,
948
+ perPage,
949
+ hasMore: false
950
+ };
951
+ }
952
+ const excludeIds = messages.map((m) => m.id);
953
+ let dataQuery = `
954
+ SELECT
955
+ id,
956
+ content,
957
+ role,
958
+ type,
959
+ toDateTime64(createdAt, 3) as createdAt,
960
+ thread_id AS "threadId",
961
+ resourceId
962
+ FROM ${TABLE_MESSAGES}
963
+ WHERE thread_id = {threadId:String}
964
+ `;
965
+ const dataParams = { threadId };
966
+ if (fromDate) {
967
+ dataQuery += ` AND createdAt >= parseDateTime64BestEffort({fromDate:String}, 3)`;
968
+ dataParams.fromDate = fromDate.toISOString();
969
+ }
970
+ if (toDate) {
971
+ dataQuery += ` AND createdAt <= parseDateTime64BestEffort({toDate:String}, 3)`;
972
+ dataParams.toDate = toDate.toISOString();
973
+ }
974
+ if (excludeIds.length > 0) {
975
+ dataQuery += ` AND id NOT IN ({excludeIds:Array(String)})`;
976
+ dataParams.excludeIds = excludeIds;
977
+ }
978
+ if (selectBy?.last) {
979
+ dataQuery += `
980
+ ORDER BY createdAt DESC
981
+ LIMIT {limit:Int64}
982
+ `;
983
+ dataParams.limit = perPage;
984
+ } else {
985
+ dataQuery += `
986
+ ORDER BY createdAt ASC
987
+ LIMIT {limit:Int64} OFFSET {offset:Int64}
988
+ `;
989
+ dataParams.limit = perPage;
990
+ dataParams.offset = offset;
991
+ }
992
+ const result = await this.client.query({
993
+ query: dataQuery,
994
+ query_params: dataParams,
995
+ clickhouse_settings: {
996
+ date_time_input_format: "best_effort",
997
+ date_time_output_format: "iso",
998
+ use_client_time_zone: 1,
999
+ output_format_json_quote_64bit_integers: 0
818
1000
  }
819
1001
  });
820
- const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
821
- if (format === `v2`) return list.get.all.v2();
822
- return list.get.all.v1();
1002
+ const rows = await result.json();
1003
+ const paginatedMessages = transformRows(rows.data);
1004
+ messages.push(...paginatedMessages);
1005
+ if (selectBy?.last) {
1006
+ messages.sort((a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime());
1007
+ }
1008
+ return {
1009
+ messages: format === "v2" ? messages : messages,
1010
+ total,
1011
+ page,
1012
+ perPage,
1013
+ hasMore: offset + perPage < total
1014
+ };
823
1015
  } catch (error) {
824
- throw new MastraError(
1016
+ const mastraError = new MastraError(
825
1017
  {
826
- id: "CLICKHOUSE_STORAGE_GET_MESSAGES_FAILED",
1018
+ id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
827
1019
  domain: ErrorDomain.STORAGE,
828
1020
  category: ErrorCategory.THIRD_PARTY,
829
- details: { threadId, resourceId: resourceId ?? "" }
1021
+ details: {
1022
+ threadId,
1023
+ resourceId: resourceId ?? ""
1024
+ }
830
1025
  },
831
1026
  error
832
1027
  );
1028
+ this.logger?.trackException?.(mastraError);
1029
+ this.logger?.error?.(mastraError.toString());
1030
+ return { messages: [], total: 0, page, perPage: perPageInput || 40, hasMore: false };
833
1031
  }
834
1032
  }
835
- async saveMessages(args) {
836
- const { messages, format = "v1" } = args;
837
- if (messages.length === 0) return messages;
1033
+ async updateMessages(args) {
1034
+ const { messages } = args;
1035
+ if (messages.length === 0) {
1036
+ return [];
1037
+ }
838
1038
  try {
839
- const threadId = messages[0]?.threadId;
840
- const resourceId = messages[0]?.resourceId;
841
- if (!threadId) {
842
- throw new Error("Thread ID is required");
843
- }
844
- const thread = await this.getThreadById({ threadId });
845
- if (!thread) {
846
- throw new Error(`Thread ${threadId} not found`);
847
- }
848
- const existingResult = await this.db.query({
849
- query: `SELECT id, thread_id FROM ${TABLE_MESSAGES} WHERE id IN ({ids:Array(String)})`,
850
- query_params: {
851
- ids: messages.map((m) => m.id)
852
- },
1039
+ const messageIds = messages.map((m) => m.id);
1040
+ const existingResult = await this.client.query({
1041
+ query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id IN (${messageIds.map((_, i) => `{id_${i}:String}`).join(",")})`,
1042
+ query_params: messageIds.reduce((acc, m, i) => ({ ...acc, [`id_${i}`]: m }), {}),
853
1043
  clickhouse_settings: {
854
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
855
1044
  date_time_input_format: "best_effort",
856
1045
  date_time_output_format: "iso",
857
1046
  use_client_time_zone: 1,
858
1047
  output_format_json_quote_64bit_integers: 0
859
- },
860
- format: "JSONEachRow"
1048
+ }
861
1049
  });
862
1050
  const existingRows = await existingResult.json();
863
- const existingSet = new Set(existingRows.map((row) => `${row.id}::${row.thread_id}`));
864
- const toInsert = messages.filter((m) => !existingSet.has(`${m.id}::${threadId}`));
865
- const toUpdate = messages.filter((m) => existingSet.has(`${m.id}::${threadId}`));
866
- const updatePromises = toUpdate.map(
867
- (message) => this.db.command({
868
- query: `
869
- ALTER TABLE ${TABLE_MESSAGES}
870
- UPDATE content = {var_content:String}, role = {var_role:String}, type = {var_type:String}
871
- WHERE id = {var_id:String} AND thread_id = {var_thread_id:String}
872
- `,
873
- query_params: {
874
- var_content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
875
- var_role: message.role,
876
- var_type: message.type || "v2",
877
- var_id: message.id,
878
- var_thread_id: threadId
879
- },
880
- clickhouse_settings: {
881
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
882
- date_time_input_format: "best_effort",
883
- use_client_time_zone: 1,
884
- output_format_json_quote_64bit_integers: 0
1051
+ const existingMessages = transformRows(existingRows.data);
1052
+ if (existingMessages.length === 0) {
1053
+ return [];
1054
+ }
1055
+ const parsedExistingMessages = existingMessages.map((msg) => {
1056
+ if (typeof msg.content === "string") {
1057
+ try {
1058
+ msg.content = JSON.parse(msg.content);
1059
+ } catch {
885
1060
  }
886
- })
887
- );
888
- await Promise.all([
889
- // Insert messages
890
- this.db.insert({
891
- table: TABLE_MESSAGES,
892
- format: "JSONEachRow",
893
- values: toInsert.map((message) => ({
894
- id: message.id,
895
- thread_id: threadId,
896
- content: typeof message.content === "string" ? message.content : JSON.stringify(message.content),
897
- createdAt: message.createdAt.toISOString(),
898
- role: message.role,
899
- type: message.type || "v2"
900
- })),
1061
+ }
1062
+ return msg;
1063
+ });
1064
+ const threadIdsToUpdate = /* @__PURE__ */ new Set();
1065
+ const updatePromises = [];
1066
+ for (const existingMessage of parsedExistingMessages) {
1067
+ const updatePayload = messages.find((m) => m.id === existingMessage.id);
1068
+ if (!updatePayload) continue;
1069
+ const { id, ...fieldsToUpdate } = updatePayload;
1070
+ if (Object.keys(fieldsToUpdate).length === 0) continue;
1071
+ threadIdsToUpdate.add(existingMessage.threadId);
1072
+ if (updatePayload.threadId && updatePayload.threadId !== existingMessage.threadId) {
1073
+ threadIdsToUpdate.add(updatePayload.threadId);
1074
+ }
1075
+ const setClauses = [];
1076
+ const values = {};
1077
+ let paramIdx = 1;
1078
+ let newContent = null;
1079
+ const updatableFields = { ...fieldsToUpdate };
1080
+ if (updatableFields.content) {
1081
+ const existingContent = existingMessage.content || {};
1082
+ const existingMetadata = existingContent.metadata || {};
1083
+ const updateMetadata = updatableFields.content.metadata || {};
1084
+ newContent = {
1085
+ ...existingContent,
1086
+ ...updatableFields.content,
1087
+ // Deep merge metadata
1088
+ metadata: {
1089
+ ...existingMetadata,
1090
+ ...updateMetadata
1091
+ }
1092
+ };
1093
+ setClauses.push(`content = {var_content_${paramIdx}:String}`);
1094
+ values[`var_content_${paramIdx}`] = JSON.stringify(newContent);
1095
+ paramIdx++;
1096
+ delete updatableFields.content;
1097
+ }
1098
+ for (const key in updatableFields) {
1099
+ if (Object.prototype.hasOwnProperty.call(updatableFields, key)) {
1100
+ const dbColumn = key === "threadId" ? "thread_id" : key;
1101
+ setClauses.push(`"${dbColumn}" = {var_${key}_${paramIdx}:String}`);
1102
+ values[`var_${key}_${paramIdx}`] = updatableFields[key];
1103
+ paramIdx++;
1104
+ }
1105
+ }
1106
+ if (setClauses.length > 0) {
1107
+ values[`var_id_${paramIdx}`] = id;
1108
+ const updateQuery = `
1109
+ ALTER TABLE ${TABLE_MESSAGES}
1110
+ UPDATE ${setClauses.join(", ")}
1111
+ WHERE id = {var_id_${paramIdx}:String}
1112
+ `;
1113
+ console.info("Updating message:", id, "with query:", updateQuery, "values:", values);
1114
+ updatePromises.push(
1115
+ this.client.command({
1116
+ query: updateQuery,
1117
+ query_params: values,
1118
+ clickhouse_settings: {
1119
+ date_time_input_format: "best_effort",
1120
+ use_client_time_zone: 1,
1121
+ output_format_json_quote_64bit_integers: 0
1122
+ }
1123
+ })
1124
+ );
1125
+ }
1126
+ }
1127
+ if (updatePromises.length > 0) {
1128
+ await Promise.all(updatePromises);
1129
+ }
1130
+ await this.client.command({
1131
+ query: `OPTIMIZE TABLE ${TABLE_MESSAGES} FINAL`,
1132
+ clickhouse_settings: {
1133
+ date_time_input_format: "best_effort",
1134
+ use_client_time_zone: 1,
1135
+ output_format_json_quote_64bit_integers: 0
1136
+ }
1137
+ });
1138
+ for (const existingMessage of parsedExistingMessages) {
1139
+ const updatePayload = messages.find((m) => m.id === existingMessage.id);
1140
+ if (!updatePayload) continue;
1141
+ const { id, ...fieldsToUpdate } = updatePayload;
1142
+ if (Object.keys(fieldsToUpdate).length === 0) continue;
1143
+ const verifyResult = await this.client.query({
1144
+ query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
1145
+ query_params: { messageId: id },
901
1146
  clickhouse_settings: {
902
- // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
903
1147
  date_time_input_format: "best_effort",
1148
+ date_time_output_format: "iso",
904
1149
  use_client_time_zone: 1,
905
1150
  output_format_json_quote_64bit_integers: 0
906
1151
  }
907
- }),
908
- ...updatePromises,
909
- // Update thread's updatedAt timestamp
910
- this.db.insert({
911
- table: TABLE_THREADS,
912
- format: "JSONEachRow",
913
- values: [
914
- {
915
- id: thread.id,
916
- resourceId: thread.resourceId,
917
- title: thread.title,
918
- metadata: thread.metadata,
919
- createdAt: thread.createdAt,
920
- updatedAt: (/* @__PURE__ */ new Date()).toISOString()
1152
+ });
1153
+ const verifyRows = await verifyResult.json();
1154
+ if (verifyRows.data.length > 0) {
1155
+ const updatedMessage = transformRows(verifyRows.data)[0];
1156
+ if (updatedMessage) {
1157
+ let needsRetry = false;
1158
+ for (const [key, value] of Object.entries(fieldsToUpdate)) {
1159
+ if (key === "content") {
1160
+ const expectedContent = typeof value === "string" ? value : JSON.stringify(value);
1161
+ const actualContent = typeof updatedMessage.content === "string" ? updatedMessage.content : JSON.stringify(updatedMessage.content);
1162
+ if (actualContent !== expectedContent) {
1163
+ needsRetry = true;
1164
+ break;
1165
+ }
1166
+ } else if (updatedMessage[key] !== value) {
1167
+ needsRetry = true;
1168
+ break;
1169
+ }
1170
+ }
1171
+ if (needsRetry) {
1172
+ console.info("Update not applied correctly, retrying with DELETE + INSERT for message:", id);
1173
+ await this.client.command({
1174
+ query: `DELETE FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
1175
+ query_params: { messageId: id },
1176
+ clickhouse_settings: {
1177
+ date_time_input_format: "best_effort",
1178
+ use_client_time_zone: 1,
1179
+ output_format_json_quote_64bit_integers: 0
1180
+ }
1181
+ });
1182
+ let updatedContent = existingMessage.content || {};
1183
+ if (fieldsToUpdate.content) {
1184
+ const existingContent = existingMessage.content || {};
1185
+ const existingMetadata = existingContent.metadata || {};
1186
+ const updateMetadata = fieldsToUpdate.content.metadata || {};
1187
+ updatedContent = {
1188
+ ...existingContent,
1189
+ ...fieldsToUpdate.content,
1190
+ metadata: {
1191
+ ...existingMetadata,
1192
+ ...updateMetadata
1193
+ }
1194
+ };
1195
+ }
1196
+ const updatedMessageData = {
1197
+ ...existingMessage,
1198
+ ...fieldsToUpdate,
1199
+ content: updatedContent
1200
+ };
1201
+ await this.client.insert({
1202
+ table: TABLE_MESSAGES,
1203
+ format: "JSONEachRow",
1204
+ values: [
1205
+ {
1206
+ id: updatedMessageData.id,
1207
+ thread_id: updatedMessageData.threadId,
1208
+ resourceId: updatedMessageData.resourceId,
1209
+ content: typeof updatedMessageData.content === "string" ? updatedMessageData.content : JSON.stringify(updatedMessageData.content),
1210
+ createdAt: updatedMessageData.createdAt.toISOString(),
1211
+ role: updatedMessageData.role,
1212
+ type: updatedMessageData.type || "v2"
1213
+ }
1214
+ ],
1215
+ clickhouse_settings: {
1216
+ date_time_input_format: "best_effort",
1217
+ use_client_time_zone: 1,
1218
+ output_format_json_quote_64bit_integers: 0
1219
+ }
1220
+ });
1221
+ }
1222
+ }
1223
+ }
1224
+ }
1225
+ if (threadIdsToUpdate.size > 0) {
1226
+ await new Promise((resolve) => setTimeout(resolve, 10));
1227
+ const now = (/* @__PURE__ */ new Date()).toISOString().replace("Z", "");
1228
+ const threadUpdatePromises = Array.from(threadIdsToUpdate).map(async (threadId) => {
1229
+ const threadResult = await this.client.query({
1230
+ query: `SELECT id, resourceId, title, metadata, createdAt FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
1231
+ query_params: { threadId },
1232
+ clickhouse_settings: {
1233
+ date_time_input_format: "best_effort",
1234
+ date_time_output_format: "iso",
1235
+ use_client_time_zone: 1,
1236
+ output_format_json_quote_64bit_integers: 0
921
1237
  }
922
- ],
1238
+ });
1239
+ const threadRows = await threadResult.json();
1240
+ if (threadRows.data.length > 0) {
1241
+ const existingThread = threadRows.data[0];
1242
+ await this.client.command({
1243
+ query: `DELETE FROM ${TABLE_THREADS} WHERE id = {threadId:String}`,
1244
+ query_params: { threadId },
1245
+ clickhouse_settings: {
1246
+ date_time_input_format: "best_effort",
1247
+ use_client_time_zone: 1,
1248
+ output_format_json_quote_64bit_integers: 0
1249
+ }
1250
+ });
1251
+ await this.client.insert({
1252
+ table: TABLE_THREADS,
1253
+ format: "JSONEachRow",
1254
+ values: [
1255
+ {
1256
+ id: existingThread.id,
1257
+ resourceId: existingThread.resourceId,
1258
+ title: existingThread.title,
1259
+ metadata: existingThread.metadata,
1260
+ createdAt: existingThread.createdAt,
1261
+ updatedAt: now
1262
+ }
1263
+ ],
1264
+ clickhouse_settings: {
1265
+ date_time_input_format: "best_effort",
1266
+ use_client_time_zone: 1,
1267
+ output_format_json_quote_64bit_integers: 0
1268
+ }
1269
+ });
1270
+ }
1271
+ });
1272
+ await Promise.all(threadUpdatePromises);
1273
+ }
1274
+ const updatedMessages = [];
1275
+ for (const messageId of messageIds) {
1276
+ const updatedResult = await this.client.query({
1277
+ query: `SELECT id, content, role, type, "createdAt", thread_id AS "threadId", "resourceId" FROM ${TABLE_MESSAGES} WHERE id = {messageId:String}`,
1278
+ query_params: { messageId },
923
1279
  clickhouse_settings: {
924
1280
  date_time_input_format: "best_effort",
1281
+ date_time_output_format: "iso",
925
1282
  use_client_time_zone: 1,
926
1283
  output_format_json_quote_64bit_integers: 0
927
1284
  }
928
- })
929
- ]);
930
- const list = new MessageList({ threadId, resourceId }).add(messages, "memory");
931
- if (format === `v2`) return list.get.all.v2();
932
- return list.get.all.v1();
1285
+ });
1286
+ const updatedRows = await updatedResult.json();
1287
+ if (updatedRows.data.length > 0) {
1288
+ const message = transformRows(updatedRows.data)[0];
1289
+ if (message) {
1290
+ updatedMessages.push(message);
1291
+ }
1292
+ }
1293
+ }
1294
+ return updatedMessages.map((message) => {
1295
+ if (typeof message.content === "string") {
1296
+ try {
1297
+ message.content = JSON.parse(message.content);
1298
+ } catch {
1299
+ }
1300
+ }
1301
+ return message;
1302
+ });
1303
+ } catch (error) {
1304
+ throw new MastraError(
1305
+ {
1306
+ id: "CLICKHOUSE_STORAGE_UPDATE_MESSAGES_FAILED",
1307
+ domain: ErrorDomain.STORAGE,
1308
+ category: ErrorCategory.THIRD_PARTY,
1309
+ details: { messageIds: messages.map((m) => m.id).join(",") }
1310
+ },
1311
+ error
1312
+ );
1313
+ }
1314
+ }
1315
+ async getResourceById({ resourceId }) {
1316
+ try {
1317
+ const result = await this.client.query({
1318
+ query: `SELECT id, workingMemory, metadata, createdAt, updatedAt FROM ${TABLE_RESOURCES} WHERE id = {resourceId:String}`,
1319
+ query_params: { resourceId },
1320
+ clickhouse_settings: {
1321
+ date_time_input_format: "best_effort",
1322
+ date_time_output_format: "iso",
1323
+ use_client_time_zone: 1,
1324
+ output_format_json_quote_64bit_integers: 0
1325
+ }
1326
+ });
1327
+ const rows = await result.json();
1328
+ if (rows.data.length === 0) {
1329
+ return null;
1330
+ }
1331
+ const resource = rows.data[0];
1332
+ return {
1333
+ id: resource.id,
1334
+ workingMemory: resource.workingMemory && typeof resource.workingMemory === "object" ? JSON.stringify(resource.workingMemory) : resource.workingMemory,
1335
+ metadata: resource.metadata && typeof resource.metadata === "string" ? JSON.parse(resource.metadata) : resource.metadata,
1336
+ createdAt: new Date(resource.createdAt),
1337
+ updatedAt: new Date(resource.updatedAt)
1338
+ };
1339
+ } catch (error) {
1340
+ throw new MastraError(
1341
+ {
1342
+ id: "CLICKHOUSE_STORAGE_GET_RESOURCE_BY_ID_FAILED",
1343
+ domain: ErrorDomain.STORAGE,
1344
+ category: ErrorCategory.THIRD_PARTY,
1345
+ details: { resourceId }
1346
+ },
1347
+ error
1348
+ );
1349
+ }
1350
+ }
1351
+ async saveResource({ resource }) {
1352
+ try {
1353
+ await this.client.insert({
1354
+ table: TABLE_RESOURCES,
1355
+ format: "JSONEachRow",
1356
+ values: [
1357
+ {
1358
+ id: resource.id,
1359
+ workingMemory: resource.workingMemory,
1360
+ metadata: JSON.stringify(resource.metadata),
1361
+ createdAt: resource.createdAt.toISOString(),
1362
+ updatedAt: resource.updatedAt.toISOString()
1363
+ }
1364
+ ],
1365
+ clickhouse_settings: {
1366
+ date_time_input_format: "best_effort",
1367
+ use_client_time_zone: 1,
1368
+ output_format_json_quote_64bit_integers: 0
1369
+ }
1370
+ });
1371
+ return resource;
1372
+ } catch (error) {
1373
+ throw new MastraError(
1374
+ {
1375
+ id: "CLICKHOUSE_STORAGE_SAVE_RESOURCE_FAILED",
1376
+ domain: ErrorDomain.STORAGE,
1377
+ category: ErrorCategory.THIRD_PARTY,
1378
+ details: { resourceId: resource.id }
1379
+ },
1380
+ error
1381
+ );
1382
+ }
1383
+ }
1384
+ async updateResource({
1385
+ resourceId,
1386
+ workingMemory,
1387
+ metadata
1388
+ }) {
1389
+ try {
1390
+ const existingResource = await this.getResourceById({ resourceId });
1391
+ if (!existingResource) {
1392
+ const newResource = {
1393
+ id: resourceId,
1394
+ workingMemory,
1395
+ metadata: metadata || {},
1396
+ createdAt: /* @__PURE__ */ new Date(),
1397
+ updatedAt: /* @__PURE__ */ new Date()
1398
+ };
1399
+ return this.saveResource({ resource: newResource });
1400
+ }
1401
+ const updatedResource = {
1402
+ ...existingResource,
1403
+ workingMemory: workingMemory !== void 0 ? workingMemory : existingResource.workingMemory,
1404
+ metadata: {
1405
+ ...existingResource.metadata,
1406
+ ...metadata
1407
+ },
1408
+ updatedAt: /* @__PURE__ */ new Date()
1409
+ };
1410
+ const updateQuery = `
1411
+ ALTER TABLE ${TABLE_RESOURCES}
1412
+ UPDATE workingMemory = {workingMemory:String}, metadata = {metadata:String}, updatedAt = {updatedAt:String}
1413
+ WHERE id = {resourceId:String}
1414
+ `;
1415
+ await this.client.command({
1416
+ query: updateQuery,
1417
+ query_params: {
1418
+ workingMemory: updatedResource.workingMemory,
1419
+ metadata: JSON.stringify(updatedResource.metadata),
1420
+ updatedAt: updatedResource.updatedAt.toISOString().replace("Z", ""),
1421
+ resourceId
1422
+ },
1423
+ clickhouse_settings: {
1424
+ date_time_input_format: "best_effort",
1425
+ use_client_time_zone: 1,
1426
+ output_format_json_quote_64bit_integers: 0
1427
+ }
1428
+ });
1429
+ await this.client.command({
1430
+ query: `OPTIMIZE TABLE ${TABLE_RESOURCES} FINAL`,
1431
+ clickhouse_settings: {
1432
+ date_time_input_format: "best_effort",
1433
+ use_client_time_zone: 1,
1434
+ output_format_json_quote_64bit_integers: 0
1435
+ }
1436
+ });
1437
+ return updatedResource;
1438
+ } catch (error) {
1439
+ throw new MastraError(
1440
+ {
1441
+ id: "CLICKHOUSE_STORAGE_UPDATE_RESOURCE_FAILED",
1442
+ domain: ErrorDomain.STORAGE,
1443
+ category: ErrorCategory.THIRD_PARTY,
1444
+ details: { resourceId }
1445
+ },
1446
+ error
1447
+ );
1448
+ }
1449
+ }
1450
+ };
1451
+ var StoreOperationsClickhouse = class extends StoreOperations {
1452
+ ttl;
1453
+ client;
1454
+ constructor({ client, ttl }) {
1455
+ super();
1456
+ this.ttl = ttl;
1457
+ this.client = client;
1458
+ }
1459
+ async hasColumn(table, column) {
1460
+ const result = await this.client.query({
1461
+ query: `DESCRIBE TABLE ${table}`,
1462
+ format: "JSONEachRow"
1463
+ });
1464
+ const columns = await result.json();
1465
+ return columns.some((c) => c.name === column);
1466
+ }
1467
+ getSqlType(type) {
1468
+ switch (type) {
1469
+ case "text":
1470
+ return "String";
1471
+ case "timestamp":
1472
+ return "DateTime64(3)";
1473
+ case "integer":
1474
+ case "bigint":
1475
+ return "Int64";
1476
+ case "jsonb":
1477
+ return "String";
1478
+ default:
1479
+ return super.getSqlType(type);
1480
+ }
1481
+ }
1482
+ async createTable({
1483
+ tableName,
1484
+ schema
1485
+ }) {
1486
+ try {
1487
+ const columns = Object.entries(schema).map(([name, def]) => {
1488
+ const constraints = [];
1489
+ if (!def.nullable) constraints.push("NOT NULL");
1490
+ const columnTtl = this.ttl?.[tableName]?.columns?.[name];
1491
+ return `"${name}" ${COLUMN_TYPES[def.type]} ${constraints.join(" ")} ${columnTtl ? `TTL toDateTime(${columnTtl.ttlKey ?? "createdAt"}) + INTERVAL ${columnTtl.interval} ${columnTtl.unit}` : ""}`;
1492
+ }).join(",\n");
1493
+ const rowTtl = this.ttl?.[tableName]?.row;
1494
+ const sql = tableName === TABLE_WORKFLOW_SNAPSHOT ? `
1495
+ CREATE TABLE IF NOT EXISTS ${tableName} (
1496
+ ${["id String"].concat(columns)}
1497
+ )
1498
+ ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
1499
+ PRIMARY KEY (createdAt, run_id, workflow_name)
1500
+ ORDER BY (createdAt, run_id, workflow_name)
1501
+ ${rowTtl ? `TTL toDateTime(${rowTtl.ttlKey ?? "createdAt"}) + INTERVAL ${rowTtl.interval} ${rowTtl.unit}` : ""}
1502
+ SETTINGS index_granularity = 8192
1503
+ ` : `
1504
+ CREATE TABLE IF NOT EXISTS ${tableName} (
1505
+ ${columns}
1506
+ )
1507
+ ENGINE = ${TABLE_ENGINES[tableName] ?? "MergeTree()"}
1508
+ PRIMARY KEY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
1509
+ ORDER BY (createdAt, ${tableName === TABLE_EVALS ? "run_id" : "id"})
1510
+ ${this.ttl?.[tableName]?.row ? `TTL toDateTime(createdAt) + INTERVAL ${this.ttl[tableName].row.interval} ${this.ttl[tableName].row.unit}` : ""}
1511
+ SETTINGS index_granularity = 8192
1512
+ `;
1513
+ await this.client.query({
1514
+ query: sql,
1515
+ clickhouse_settings: {
1516
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1517
+ date_time_input_format: "best_effort",
1518
+ date_time_output_format: "iso",
1519
+ use_client_time_zone: 1,
1520
+ output_format_json_quote_64bit_integers: 0
1521
+ }
1522
+ });
1523
+ } catch (error) {
1524
+ throw new MastraError(
1525
+ {
1526
+ id: "CLICKHOUSE_STORAGE_CREATE_TABLE_FAILED",
1527
+ domain: ErrorDomain.STORAGE,
1528
+ category: ErrorCategory.THIRD_PARTY,
1529
+ details: { tableName }
1530
+ },
1531
+ error
1532
+ );
1533
+ }
1534
+ }
1535
+ async alterTable({
1536
+ tableName,
1537
+ schema,
1538
+ ifNotExists
1539
+ }) {
1540
+ try {
1541
+ const describeSql = `DESCRIBE TABLE ${tableName}`;
1542
+ const result = await this.client.query({
1543
+ query: describeSql
1544
+ });
1545
+ const rows = await result.json();
1546
+ const existingColumnNames = new Set(rows.data.map((row) => row.name.toLowerCase()));
1547
+ for (const columnName of ifNotExists) {
1548
+ if (!existingColumnNames.has(columnName.toLowerCase()) && schema[columnName]) {
1549
+ const columnDef = schema[columnName];
1550
+ let sqlType = this.getSqlType(columnDef.type);
1551
+ if (columnDef.nullable !== false) {
1552
+ sqlType = `Nullable(${sqlType})`;
1553
+ }
1554
+ const defaultValue = columnDef.nullable === false ? this.getDefaultValue(columnDef.type) : "";
1555
+ const alterSql = `ALTER TABLE ${tableName} ADD COLUMN IF NOT EXISTS "${columnName}" ${sqlType} ${defaultValue}`.trim();
1556
+ await this.client.query({
1557
+ query: alterSql
1558
+ });
1559
+ this.logger?.debug?.(`Added column ${columnName} to table ${tableName}`);
1560
+ }
1561
+ }
1562
+ } catch (error) {
1563
+ throw new MastraError(
1564
+ {
1565
+ id: "CLICKHOUSE_STORAGE_ALTER_TABLE_FAILED",
1566
+ domain: ErrorDomain.STORAGE,
1567
+ category: ErrorCategory.THIRD_PARTY,
1568
+ details: { tableName }
1569
+ },
1570
+ error
1571
+ );
1572
+ }
1573
+ }
1574
+ async clearTable({ tableName }) {
1575
+ try {
1576
+ await this.client.query({
1577
+ query: `TRUNCATE TABLE ${tableName}`,
1578
+ clickhouse_settings: {
1579
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1580
+ date_time_input_format: "best_effort",
1581
+ date_time_output_format: "iso",
1582
+ use_client_time_zone: 1,
1583
+ output_format_json_quote_64bit_integers: 0
1584
+ }
1585
+ });
1586
+ } catch (error) {
1587
+ throw new MastraError(
1588
+ {
1589
+ id: "CLICKHOUSE_STORAGE_CLEAR_TABLE_FAILED",
1590
+ domain: ErrorDomain.STORAGE,
1591
+ category: ErrorCategory.THIRD_PARTY,
1592
+ details: { tableName }
1593
+ },
1594
+ error
1595
+ );
1596
+ }
1597
+ }
1598
+ async dropTable({ tableName }) {
1599
+ await this.client.query({
1600
+ query: `DROP TABLE IF EXISTS ${tableName}`
1601
+ });
1602
+ }
1603
+ async insert({ tableName, record }) {
1604
+ const createdAt = (record.createdAt || record.created_at || /* @__PURE__ */ new Date()).toISOString();
1605
+ const updatedAt = (record.updatedAt || /* @__PURE__ */ new Date()).toISOString();
1606
+ try {
1607
+ const result = await this.client.insert({
1608
+ table: tableName,
1609
+ values: [
1610
+ {
1611
+ ...record,
1612
+ createdAt,
1613
+ updatedAt
1614
+ }
1615
+ ],
1616
+ format: "JSONEachRow",
1617
+ clickhouse_settings: {
1618
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1619
+ output_format_json_quote_64bit_integers: 0,
1620
+ date_time_input_format: "best_effort",
1621
+ use_client_time_zone: 1
1622
+ }
1623
+ });
1624
+ console.info("INSERT RESULT", result);
1625
+ } catch (error) {
1626
+ throw new MastraError(
1627
+ {
1628
+ id: "CLICKHOUSE_STORAGE_INSERT_FAILED",
1629
+ domain: ErrorDomain.STORAGE,
1630
+ category: ErrorCategory.THIRD_PARTY,
1631
+ details: { tableName }
1632
+ },
1633
+ error
1634
+ );
1635
+ }
1636
+ }
1637
+ async batchInsert({ tableName, records }) {
1638
+ const recordsToBeInserted = records.map((record) => ({
1639
+ ...Object.fromEntries(
1640
+ Object.entries(record).map(([key, value]) => [
1641
+ key,
1642
+ TABLE_SCHEMAS[tableName]?.[key]?.type === "timestamp" ? new Date(value).toISOString() : value
1643
+ ])
1644
+ )
1645
+ }));
1646
+ try {
1647
+ await this.client.insert({
1648
+ table: tableName,
1649
+ values: recordsToBeInserted,
1650
+ format: "JSONEachRow",
1651
+ clickhouse_settings: {
1652
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1653
+ date_time_input_format: "best_effort",
1654
+ use_client_time_zone: 1,
1655
+ output_format_json_quote_64bit_integers: 0
1656
+ }
1657
+ });
1658
+ } catch (error) {
1659
+ throw new MastraError(
1660
+ {
1661
+ id: "CLICKHOUSE_STORAGE_BATCH_INSERT_FAILED",
1662
+ domain: ErrorDomain.STORAGE,
1663
+ category: ErrorCategory.THIRD_PARTY,
1664
+ details: { tableName }
1665
+ },
1666
+ error
1667
+ );
1668
+ }
1669
+ }
1670
+ async load({ tableName, keys }) {
1671
+ try {
1672
+ const engine = TABLE_ENGINES[tableName] ?? "MergeTree()";
1673
+ const keyEntries = Object.entries(keys);
1674
+ const conditions = keyEntries.map(
1675
+ ([key]) => `"${key}" = {var_${key}:${COLUMN_TYPES[TABLE_SCHEMAS[tableName]?.[key]?.type ?? "text"]}}`
1676
+ ).join(" AND ");
1677
+ const values = keyEntries.reduce((acc, [key, value]) => {
1678
+ return { ...acc, [`var_${key}`]: value };
1679
+ }, {});
1680
+ const hasUpdatedAt = TABLE_SCHEMAS[tableName]?.updatedAt;
1681
+ const selectClause = `SELECT *, toDateTime64(createdAt, 3) as createdAt${hasUpdatedAt ? ", toDateTime64(updatedAt, 3) as updatedAt" : ""}`;
1682
+ const result = await this.client.query({
1683
+ query: `${selectClause} FROM ${tableName} ${engine.startsWith("ReplacingMergeTree") ? "FINAL" : ""} WHERE ${conditions} ORDER BY createdAt DESC LIMIT 1`,
1684
+ query_params: values,
1685
+ clickhouse_settings: {
1686
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1687
+ date_time_input_format: "best_effort",
1688
+ date_time_output_format: "iso",
1689
+ use_client_time_zone: 1,
1690
+ output_format_json_quote_64bit_integers: 0
1691
+ }
1692
+ });
1693
+ if (!result) {
1694
+ return null;
1695
+ }
1696
+ const rows = await result.json();
1697
+ if (tableName === TABLE_WORKFLOW_SNAPSHOT) {
1698
+ const snapshot = rows.data[0];
1699
+ if (!snapshot) {
1700
+ return null;
1701
+ }
1702
+ if (typeof snapshot.snapshot === "string") {
1703
+ snapshot.snapshot = JSON.parse(snapshot.snapshot);
1704
+ }
1705
+ return transformRow(snapshot);
1706
+ }
1707
+ const data = transformRow(rows.data[0]);
1708
+ return data;
1709
+ } catch (error) {
1710
+ throw new MastraError(
1711
+ {
1712
+ id: "CLICKHOUSE_STORAGE_LOAD_FAILED",
1713
+ domain: ErrorDomain.STORAGE,
1714
+ category: ErrorCategory.THIRD_PARTY,
1715
+ details: { tableName }
1716
+ },
1717
+ error
1718
+ );
1719
+ }
1720
+ }
1721
+ };
1722
+ var ScoresStorageClickhouse = class extends ScoresStorage {
1723
+ client;
1724
+ operations;
1725
+ constructor({ client, operations }) {
1726
+ super();
1727
+ this.client = client;
1728
+ this.operations = operations;
1729
+ }
1730
+ transformScoreRow(row) {
1731
+ const scorer = safelyParseJSON(row.scorer);
1732
+ const preprocessStepResult = safelyParseJSON(row.preprocessStepResult);
1733
+ const analyzeStepResult = safelyParseJSON(row.analyzeStepResult);
1734
+ const metadata = safelyParseJSON(row.metadata);
1735
+ const input = safelyParseJSON(row.input);
1736
+ const output = safelyParseJSON(row.output);
1737
+ const additionalContext = safelyParseJSON(row.additionalContext);
1738
+ const runtimeContext = safelyParseJSON(row.runtimeContext);
1739
+ const entity = safelyParseJSON(row.entity);
1740
+ return {
1741
+ ...row,
1742
+ scorer,
1743
+ preprocessStepResult,
1744
+ analyzeStepResult,
1745
+ metadata,
1746
+ input,
1747
+ output,
1748
+ additionalContext,
1749
+ runtimeContext,
1750
+ entity,
1751
+ createdAt: new Date(row.createdAt),
1752
+ updatedAt: new Date(row.updatedAt)
1753
+ };
1754
+ }
1755
+ async getScoreById({ id }) {
1756
+ try {
1757
+ const result = await this.client.query({
1758
+ query: `SELECT * FROM ${TABLE_SCORERS} WHERE id = {var_id:String}`,
1759
+ query_params: { var_id: id },
1760
+ format: "JSONEachRow",
1761
+ clickhouse_settings: {
1762
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
1763
+ date_time_input_format: "best_effort",
1764
+ date_time_output_format: "iso",
1765
+ use_client_time_zone: 1,
1766
+ output_format_json_quote_64bit_integers: 0
1767
+ }
1768
+ });
1769
+ const resultJson = await result.json();
1770
+ if (!Array.isArray(resultJson) || resultJson.length === 0) {
1771
+ return null;
1772
+ }
1773
+ return this.transformScoreRow(resultJson[0]);
1774
+ } catch (error) {
1775
+ throw new MastraError(
1776
+ {
1777
+ id: "CLICKHOUSE_STORAGE_GET_SCORE_BY_ID_FAILED",
1778
+ domain: ErrorDomain.STORAGE,
1779
+ category: ErrorCategory.THIRD_PARTY,
1780
+ details: { scoreId: id }
1781
+ },
1782
+ error
1783
+ );
1784
+ }
1785
+ }
1786
+ async saveScore(score) {
1787
+ let parsedScore;
1788
+ try {
1789
+ parsedScore = saveScorePayloadSchema.parse(score);
1790
+ } catch (error) {
1791
+ throw new MastraError(
1792
+ {
1793
+ id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED_INVALID_SCORE_PAYLOAD",
1794
+ domain: ErrorDomain.STORAGE,
1795
+ category: ErrorCategory.USER,
1796
+ details: { scoreId: score.id }
1797
+ },
1798
+ error
1799
+ );
1800
+ }
1801
+ try {
1802
+ const record = {
1803
+ ...parsedScore
1804
+ };
1805
+ await this.client.insert({
1806
+ table: TABLE_SCORERS,
1807
+ values: [record],
1808
+ format: "JSONEachRow",
1809
+ clickhouse_settings: {
1810
+ date_time_input_format: "best_effort",
1811
+ use_client_time_zone: 1,
1812
+ output_format_json_quote_64bit_integers: 0
1813
+ }
1814
+ });
1815
+ return { score };
1816
+ } catch (error) {
1817
+ throw new MastraError(
1818
+ {
1819
+ id: "CLICKHOUSE_STORAGE_SAVE_SCORE_FAILED",
1820
+ domain: ErrorDomain.STORAGE,
1821
+ category: ErrorCategory.THIRD_PARTY,
1822
+ details: { scoreId: score.id }
1823
+ },
1824
+ error
1825
+ );
1826
+ }
1827
+ }
1828
+ async getScoresByRunId({
1829
+ runId,
1830
+ pagination
1831
+ }) {
1832
+ try {
1833
+ const countResult = await this.client.query({
1834
+ query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String}`,
1835
+ query_params: { var_runId: runId },
1836
+ format: "JSONEachRow"
1837
+ });
1838
+ const countRows = await countResult.json();
1839
+ let total = 0;
1840
+ if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
1841
+ const countObj = countRows[0];
1842
+ total = Number(countObj.count);
1843
+ }
1844
+ if (!total) {
1845
+ return {
1846
+ pagination: {
1847
+ total: 0,
1848
+ page: pagination.page,
1849
+ perPage: pagination.perPage,
1850
+ hasMore: false
1851
+ },
1852
+ scores: []
1853
+ };
1854
+ }
1855
+ const offset = pagination.page * pagination.perPage;
1856
+ const result = await this.client.query({
1857
+ query: `SELECT * FROM ${TABLE_SCORERS} WHERE runId = {var_runId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
1858
+ query_params: {
1859
+ var_runId: runId,
1860
+ var_limit: pagination.perPage,
1861
+ var_offset: offset
1862
+ },
1863
+ format: "JSONEachRow",
1864
+ clickhouse_settings: {
1865
+ date_time_input_format: "best_effort",
1866
+ date_time_output_format: "iso",
1867
+ use_client_time_zone: 1,
1868
+ output_format_json_quote_64bit_integers: 0
1869
+ }
1870
+ });
1871
+ const rows = await result.json();
1872
+ const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
1873
+ return {
1874
+ pagination: {
1875
+ total,
1876
+ page: pagination.page,
1877
+ perPage: pagination.perPage,
1878
+ hasMore: total > (pagination.page + 1) * pagination.perPage
1879
+ },
1880
+ scores
1881
+ };
1882
+ } catch (error) {
1883
+ throw new MastraError(
1884
+ {
1885
+ id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_RUN_ID_FAILED",
1886
+ domain: ErrorDomain.STORAGE,
1887
+ category: ErrorCategory.THIRD_PARTY,
1888
+ details: { runId }
1889
+ },
1890
+ error
1891
+ );
1892
+ }
1893
+ }
1894
+ async getScoresByScorerId({
1895
+ scorerId,
1896
+ entityId,
1897
+ entityType,
1898
+ source,
1899
+ pagination
1900
+ }) {
1901
+ let whereClause = `scorerId = {var_scorerId:String}`;
1902
+ if (entityId) {
1903
+ whereClause += ` AND entityId = {var_entityId:String}`;
1904
+ }
1905
+ if (entityType) {
1906
+ whereClause += ` AND entityType = {var_entityType:String}`;
1907
+ }
1908
+ if (source) {
1909
+ whereClause += ` AND source = {var_source:String}`;
1910
+ }
1911
+ try {
1912
+ const countResult = await this.client.query({
1913
+ query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE ${whereClause}`,
1914
+ query_params: {
1915
+ var_scorerId: scorerId,
1916
+ var_entityId: entityId,
1917
+ var_entityType: entityType,
1918
+ var_source: source
1919
+ },
1920
+ format: "JSONEachRow"
1921
+ });
1922
+ const countRows = await countResult.json();
1923
+ let total = 0;
1924
+ if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
1925
+ const countObj = countRows[0];
1926
+ total = Number(countObj.count);
1927
+ }
1928
+ if (!total) {
1929
+ return {
1930
+ pagination: {
1931
+ total: 0,
1932
+ page: pagination.page,
1933
+ perPage: pagination.perPage,
1934
+ hasMore: false
1935
+ },
1936
+ scores: []
1937
+ };
1938
+ }
1939
+ const offset = pagination.page * pagination.perPage;
1940
+ const result = await this.client.query({
1941
+ query: `SELECT * FROM ${TABLE_SCORERS} WHERE ${whereClause} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
1942
+ query_params: {
1943
+ var_scorerId: scorerId,
1944
+ var_limit: pagination.perPage,
1945
+ var_offset: offset,
1946
+ var_entityId: entityId,
1947
+ var_entityType: entityType,
1948
+ var_source: source
1949
+ },
1950
+ format: "JSONEachRow",
1951
+ clickhouse_settings: {
1952
+ date_time_input_format: "best_effort",
1953
+ date_time_output_format: "iso",
1954
+ use_client_time_zone: 1,
1955
+ output_format_json_quote_64bit_integers: 0
1956
+ }
1957
+ });
1958
+ const rows = await result.json();
1959
+ const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
1960
+ return {
1961
+ pagination: {
1962
+ total,
1963
+ page: pagination.page,
1964
+ perPage: pagination.perPage,
1965
+ hasMore: total > (pagination.page + 1) * pagination.perPage
1966
+ },
1967
+ scores
1968
+ };
1969
+ } catch (error) {
1970
+ throw new MastraError(
1971
+ {
1972
+ id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SCORER_ID_FAILED",
1973
+ domain: ErrorDomain.STORAGE,
1974
+ category: ErrorCategory.THIRD_PARTY,
1975
+ details: { scorerId }
1976
+ },
1977
+ error
1978
+ );
1979
+ }
1980
+ }
1981
+ async getScoresByEntityId({
1982
+ entityId,
1983
+ entityType,
1984
+ pagination
1985
+ }) {
1986
+ try {
1987
+ const countResult = await this.client.query({
1988
+ query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String}`,
1989
+ query_params: { var_entityId: entityId, var_entityType: entityType },
1990
+ format: "JSONEachRow"
1991
+ });
1992
+ const countRows = await countResult.json();
1993
+ let total = 0;
1994
+ if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
1995
+ const countObj = countRows[0];
1996
+ total = Number(countObj.count);
1997
+ }
1998
+ if (!total) {
1999
+ return {
2000
+ pagination: {
2001
+ total: 0,
2002
+ page: pagination.page,
2003
+ perPage: pagination.perPage,
2004
+ hasMore: false
2005
+ },
2006
+ scores: []
2007
+ };
2008
+ }
2009
+ const offset = pagination.page * pagination.perPage;
2010
+ const result = await this.client.query({
2011
+ query: `SELECT * FROM ${TABLE_SCORERS} WHERE entityId = {var_entityId:String} AND entityType = {var_entityType:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
2012
+ query_params: {
2013
+ var_entityId: entityId,
2014
+ var_entityType: entityType,
2015
+ var_limit: pagination.perPage,
2016
+ var_offset: offset
2017
+ },
2018
+ format: "JSONEachRow",
2019
+ clickhouse_settings: {
2020
+ date_time_input_format: "best_effort",
2021
+ date_time_output_format: "iso",
2022
+ use_client_time_zone: 1,
2023
+ output_format_json_quote_64bit_integers: 0
2024
+ }
2025
+ });
2026
+ const rows = await result.json();
2027
+ const scores = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
2028
+ return {
2029
+ pagination: {
2030
+ total,
2031
+ page: pagination.page,
2032
+ perPage: pagination.perPage,
2033
+ hasMore: total > (pagination.page + 1) * pagination.perPage
2034
+ },
2035
+ scores
2036
+ };
2037
+ } catch (error) {
2038
+ throw new MastraError(
2039
+ {
2040
+ id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_ENTITY_ID_FAILED",
2041
+ domain: ErrorDomain.STORAGE,
2042
+ category: ErrorCategory.THIRD_PARTY,
2043
+ details: { entityId, entityType }
2044
+ },
2045
+ error
2046
+ );
2047
+ }
2048
+ }
2049
+ async getScoresBySpan({
2050
+ traceId,
2051
+ spanId,
2052
+ pagination
2053
+ }) {
2054
+ try {
2055
+ const countResult = await this.client.query({
2056
+ query: `SELECT COUNT(*) as count FROM ${TABLE_SCORERS} WHERE traceId = {var_traceId:String} AND spanId = {var_spanId:String}`,
2057
+ query_params: {
2058
+ var_traceId: traceId,
2059
+ var_spanId: spanId
2060
+ },
2061
+ format: "JSONEachRow"
2062
+ });
2063
+ const countRows = await countResult.json();
2064
+ let total = 0;
2065
+ if (Array.isArray(countRows) && countRows.length > 0 && countRows[0]) {
2066
+ const countObj = countRows[0];
2067
+ total = Number(countObj.count);
2068
+ }
2069
+ if (!total) {
2070
+ return {
2071
+ pagination: {
2072
+ total: 0,
2073
+ page: pagination.page,
2074
+ perPage: pagination.perPage,
2075
+ hasMore: false
2076
+ },
2077
+ scores: []
2078
+ };
2079
+ }
2080
+ const limit = pagination.perPage + 1;
2081
+ const offset = pagination.page * pagination.perPage;
2082
+ const result = await this.client.query({
2083
+ query: `SELECT * FROM ${TABLE_SCORERS} WHERE traceId = {var_traceId:String} AND spanId = {var_spanId:String} ORDER BY createdAt DESC LIMIT {var_limit:Int64} OFFSET {var_offset:Int64}`,
2084
+ query_params: {
2085
+ var_traceId: traceId,
2086
+ var_spanId: spanId,
2087
+ var_limit: limit,
2088
+ var_offset: offset
2089
+ },
2090
+ format: "JSONEachRow",
2091
+ clickhouse_settings: {
2092
+ date_time_input_format: "best_effort",
2093
+ date_time_output_format: "iso",
2094
+ use_client_time_zone: 1,
2095
+ output_format_json_quote_64bit_integers: 0
2096
+ }
2097
+ });
2098
+ const rows = await result.json();
2099
+ const transformedRows = Array.isArray(rows) ? rows.map((row) => this.transformScoreRow(row)) : [];
2100
+ const hasMore = transformedRows.length > pagination.perPage;
2101
+ const scores = hasMore ? transformedRows.slice(0, pagination.perPage) : transformedRows;
2102
+ return {
2103
+ pagination: {
2104
+ total,
2105
+ page: pagination.page,
2106
+ perPage: pagination.perPage,
2107
+ hasMore
2108
+ },
2109
+ scores
2110
+ };
2111
+ } catch (error) {
2112
+ throw new MastraError(
2113
+ {
2114
+ id: "CLICKHOUSE_STORAGE_GET_SCORES_BY_SPAN_FAILED",
2115
+ domain: ErrorDomain.STORAGE,
2116
+ category: ErrorCategory.THIRD_PARTY,
2117
+ details: { traceId, spanId }
2118
+ },
2119
+ error
2120
+ );
2121
+ }
2122
+ }
2123
+ };
2124
+ var TracesStorageClickhouse = class extends TracesStorage {
2125
+ client;
2126
+ operations;
2127
+ constructor({ client, operations }) {
2128
+ super();
2129
+ this.client = client;
2130
+ this.operations = operations;
2131
+ }
2132
+ async getTracesPaginated(args) {
2133
+ const { name, scope, page = 0, perPage = 100, attributes, filters, dateRange } = args;
2134
+ const fromDate = dateRange?.start;
2135
+ const toDate = dateRange?.end;
2136
+ const currentOffset = page * perPage;
2137
+ const queryArgs = {};
2138
+ const conditions = [];
2139
+ if (name) {
2140
+ conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
2141
+ queryArgs.var_name = name;
2142
+ }
2143
+ if (scope) {
2144
+ conditions.push(`scope = {var_scope:String}`);
2145
+ queryArgs.var_scope = scope;
2146
+ }
2147
+ if (attributes) {
2148
+ Object.entries(attributes).forEach(([key, value]) => {
2149
+ conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
2150
+ queryArgs[`var_attr_${key}`] = value;
2151
+ });
2152
+ }
2153
+ if (filters) {
2154
+ Object.entries(filters).forEach(([key, value]) => {
2155
+ conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
2156
+ queryArgs[`var_col_${key}`] = value;
2157
+ });
2158
+ }
2159
+ if (fromDate) {
2160
+ conditions.push(`createdAt >= parseDateTime64BestEffort({var_from_date:String})`);
2161
+ queryArgs.var_from_date = fromDate.toISOString();
2162
+ }
2163
+ if (toDate) {
2164
+ conditions.push(`createdAt <= parseDateTime64BestEffort({var_to_date:String})`);
2165
+ queryArgs.var_to_date = toDate.toISOString();
2166
+ }
2167
+ const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
2168
+ try {
2169
+ const countResult = await this.client.query({
2170
+ query: `SELECT COUNT(*) as count FROM ${TABLE_TRACES} ${whereClause}`,
2171
+ query_params: queryArgs,
2172
+ clickhouse_settings: {
2173
+ date_time_input_format: "best_effort",
2174
+ date_time_output_format: "iso",
2175
+ use_client_time_zone: 1,
2176
+ output_format_json_quote_64bit_integers: 0
2177
+ }
2178
+ });
2179
+ const countData = await countResult.json();
2180
+ const total = Number(countData.data?.[0]?.count ?? 0);
2181
+ if (total === 0) {
2182
+ return {
2183
+ traces: [],
2184
+ total: 0,
2185
+ page,
2186
+ perPage,
2187
+ hasMore: false
2188
+ };
2189
+ }
2190
+ const result = await this.client.query({
2191
+ query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT {var_limit:UInt32} OFFSET {var_offset:UInt32}`,
2192
+ query_params: { ...queryArgs, var_limit: perPage, var_offset: currentOffset },
2193
+ clickhouse_settings: {
2194
+ date_time_input_format: "best_effort",
2195
+ date_time_output_format: "iso",
2196
+ use_client_time_zone: 1,
2197
+ output_format_json_quote_64bit_integers: 0
2198
+ }
2199
+ });
2200
+ if (!result) {
2201
+ return {
2202
+ traces: [],
2203
+ total,
2204
+ page,
2205
+ perPage,
2206
+ hasMore: false
2207
+ };
2208
+ }
2209
+ const resp = await result.json();
2210
+ const rows = resp.data;
2211
+ const traces = rows.map((row) => ({
2212
+ id: row.id,
2213
+ parentSpanId: row.parentSpanId,
2214
+ traceId: row.traceId,
2215
+ name: row.name,
2216
+ scope: row.scope,
2217
+ kind: row.kind,
2218
+ status: safelyParseJSON(row.status),
2219
+ events: safelyParseJSON(row.events),
2220
+ links: safelyParseJSON(row.links),
2221
+ attributes: safelyParseJSON(row.attributes),
2222
+ startTime: row.startTime,
2223
+ endTime: row.endTime,
2224
+ other: safelyParseJSON(row.other),
2225
+ createdAt: row.createdAt
2226
+ }));
2227
+ return {
2228
+ traces,
2229
+ total,
2230
+ page,
2231
+ perPage,
2232
+ hasMore: currentOffset + traces.length < total
2233
+ };
2234
+ } catch (error) {
2235
+ if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
2236
+ return {
2237
+ traces: [],
2238
+ total: 0,
2239
+ page,
2240
+ perPage,
2241
+ hasMore: false
2242
+ };
2243
+ }
2244
+ throw new MastraError(
2245
+ {
2246
+ id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
2247
+ domain: ErrorDomain.STORAGE,
2248
+ category: ErrorCategory.THIRD_PARTY,
2249
+ details: {
2250
+ name: name ?? null,
2251
+ scope: scope ?? null,
2252
+ page,
2253
+ perPage,
2254
+ attributes: attributes ? JSON.stringify(attributes) : null,
2255
+ filters: filters ? JSON.stringify(filters) : null,
2256
+ dateRange: dateRange ? JSON.stringify(dateRange) : null
2257
+ }
2258
+ },
2259
+ error
2260
+ );
2261
+ }
2262
+ }
2263
+ async getTraces({
2264
+ name,
2265
+ scope,
2266
+ page,
2267
+ perPage,
2268
+ attributes,
2269
+ filters,
2270
+ fromDate,
2271
+ toDate
2272
+ }) {
2273
+ const limit = perPage;
2274
+ const offset = page * perPage;
2275
+ const args = {};
2276
+ const conditions = [];
2277
+ if (name) {
2278
+ conditions.push(`name LIKE CONCAT({var_name:String}, '%')`);
2279
+ args.var_name = name;
2280
+ }
2281
+ if (scope) {
2282
+ conditions.push(`scope = {var_scope:String}`);
2283
+ args.var_scope = scope;
2284
+ }
2285
+ if (attributes) {
2286
+ Object.entries(attributes).forEach(([key, value]) => {
2287
+ conditions.push(`JSONExtractString(attributes, '${key}') = {var_attr_${key}:String}`);
2288
+ args[`var_attr_${key}`] = value;
2289
+ });
2290
+ }
2291
+ if (filters) {
2292
+ Object.entries(filters).forEach(([key, value]) => {
2293
+ conditions.push(`${key} = {var_col_${key}:${TABLE_SCHEMAS.mastra_traces?.[key]?.type ?? "text"}}`);
2294
+ args[`var_col_${key}`] = value;
2295
+ });
2296
+ }
2297
+ if (fromDate) {
2298
+ conditions.push(`createdAt >= {var_from_date:DateTime64(3)}`);
2299
+ args.var_from_date = fromDate.getTime() / 1e3;
2300
+ }
2301
+ if (toDate) {
2302
+ conditions.push(`createdAt <= {var_to_date:DateTime64(3)}`);
2303
+ args.var_to_date = toDate.getTime() / 1e3;
2304
+ }
2305
+ const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
2306
+ try {
2307
+ const result = await this.client.query({
2308
+ query: `SELECT *, toDateTime64(createdAt, 3) as createdAt FROM ${TABLE_TRACES} ${whereClause} ORDER BY "createdAt" DESC LIMIT ${limit} OFFSET ${offset}`,
2309
+ query_params: args,
2310
+ clickhouse_settings: {
2311
+ // Allows to insert serialized JS Dates (such as '2023-12-06T10:54:48.000Z')
2312
+ date_time_input_format: "best_effort",
2313
+ date_time_output_format: "iso",
2314
+ use_client_time_zone: 1,
2315
+ output_format_json_quote_64bit_integers: 0
2316
+ }
2317
+ });
2318
+ if (!result) {
2319
+ return [];
2320
+ }
2321
+ const resp = await result.json();
2322
+ const rows = resp.data;
2323
+ return rows.map((row) => ({
2324
+ id: row.id,
2325
+ parentSpanId: row.parentSpanId,
2326
+ traceId: row.traceId,
2327
+ name: row.name,
2328
+ scope: row.scope,
2329
+ kind: row.kind,
2330
+ status: safelyParseJSON(row.status),
2331
+ events: safelyParseJSON(row.events),
2332
+ links: safelyParseJSON(row.links),
2333
+ attributes: safelyParseJSON(row.attributes),
2334
+ startTime: row.startTime,
2335
+ endTime: row.endTime,
2336
+ other: safelyParseJSON(row.other),
2337
+ createdAt: row.createdAt
2338
+ }));
933
2339
  } catch (error) {
2340
+ if (error?.message?.includes("no such table") || error?.message?.includes("does not exist")) {
2341
+ return [];
2342
+ }
934
2343
  throw new MastraError(
935
2344
  {
936
- id: "CLICKHOUSE_STORAGE_SAVE_MESSAGES_FAILED",
2345
+ id: "CLICKHOUSE_STORAGE_GET_TRACES_FAILED",
937
2346
  domain: ErrorDomain.STORAGE,
938
- category: ErrorCategory.THIRD_PARTY
2347
+ category: ErrorCategory.THIRD_PARTY,
2348
+ details: {
2349
+ name: name ?? null,
2350
+ scope: scope ?? null,
2351
+ page,
2352
+ perPage,
2353
+ attributes: attributes ? JSON.stringify(attributes) : null,
2354
+ filters: filters ? JSON.stringify(filters) : null,
2355
+ fromDate: fromDate?.toISOString() ?? null,
2356
+ toDate: toDate?.toISOString() ?? null
2357
+ }
939
2358
  },
940
2359
  error
941
2360
  );
942
2361
  }
943
2362
  }
2363
+ async batchTraceInsert(args) {
2364
+ await this.operations.batchInsert({ tableName: TABLE_TRACES, records: args.records });
2365
+ }
2366
+ };
2367
+ var WorkflowsStorageClickhouse = class extends WorkflowsStorage {
2368
+ client;
2369
+ operations;
2370
+ constructor({ client, operations }) {
2371
+ super();
2372
+ this.operations = operations;
2373
+ this.client = client;
2374
+ }
2375
+ updateWorkflowResults({
2376
+ // workflowName,
2377
+ // runId,
2378
+ // stepId,
2379
+ // result,
2380
+ // runtimeContext,
2381
+ }) {
2382
+ throw new Error("Method not implemented.");
2383
+ }
2384
+ updateWorkflowState({
2385
+ // workflowName,
2386
+ // runId,
2387
+ // opts,
2388
+ }) {
2389
+ throw new Error("Method not implemented.");
2390
+ }
944
2391
  async persistWorkflowSnapshot({
945
2392
  workflowName,
946
2393
  runId,
2394
+ resourceId,
947
2395
  snapshot
948
2396
  }) {
949
2397
  try {
950
- const currentSnapshot = await this.load({
2398
+ const currentSnapshot = await this.operations.load({
951
2399
  tableName: TABLE_WORKFLOW_SNAPSHOT,
952
2400
  keys: { workflow_name: workflowName, run_id: runId }
953
2401
  });
954
2402
  const now = /* @__PURE__ */ new Date();
955
2403
  const persisting = currentSnapshot ? {
956
2404
  ...currentSnapshot,
2405
+ resourceId,
957
2406
  snapshot: JSON.stringify(snapshot),
958
2407
  updatedAt: now.toISOString()
959
2408
  } : {
960
2409
  workflow_name: workflowName,
961
2410
  run_id: runId,
2411
+ resourceId,
962
2412
  snapshot: JSON.stringify(snapshot),
963
2413
  createdAt: now.toISOString(),
964
2414
  updatedAt: now.toISOString()
965
2415
  };
966
- await this.db.insert({
2416
+ await this.client.insert({
967
2417
  table: TABLE_WORKFLOW_SNAPSHOT,
968
2418
  format: "JSONEachRow",
969
2419
  values: [persisting],
@@ -991,7 +2441,7 @@ var ClickhouseStore = class extends MastraStorage {
991
2441
  runId
992
2442
  }) {
993
2443
  try {
994
- const result = await this.load({
2444
+ const result = await this.operations.load({
995
2445
  tableName: TABLE_WORKFLOW_SNAPSHOT,
996
2446
  keys: {
997
2447
  workflow_name: workflowName,
@@ -1048,7 +2498,7 @@ var ClickhouseStore = class extends MastraStorage {
1048
2498
  values.var_workflow_name = workflowName;
1049
2499
  }
1050
2500
  if (resourceId) {
1051
- const hasResourceId = await this.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
2501
+ const hasResourceId = await this.operations.hasColumn(TABLE_WORKFLOW_SNAPSHOT, "resourceId");
1052
2502
  if (hasResourceId) {
1053
2503
  conditions.push(`resourceId = {var_resourceId:String}`);
1054
2504
  values.var_resourceId = resourceId;
@@ -1069,7 +2519,7 @@ var ClickhouseStore = class extends MastraStorage {
1069
2519
  const offsetClause = offset !== void 0 ? `OFFSET ${offset}` : "";
1070
2520
  let total = 0;
1071
2521
  if (limit !== void 0 && offset !== void 0) {
1072
- const countResult = await this.db.query({
2522
+ const countResult = await this.client.query({
1073
2523
  query: `SELECT COUNT(*) as count FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""} ${whereClause}`,
1074
2524
  query_params: values,
1075
2525
  format: "JSONEachRow"
@@ -1077,21 +2527,21 @@ var ClickhouseStore = class extends MastraStorage {
1077
2527
  const countRows = await countResult.json();
1078
2528
  total = Number(countRows[0]?.count ?? 0);
1079
2529
  }
1080
- const result = await this.db.query({
2530
+ const result = await this.client.query({
1081
2531
  query: `
1082
- SELECT
1083
- workflow_name,
1084
- run_id,
1085
- snapshot,
1086
- toDateTime64(createdAt, 3) as createdAt,
1087
- toDateTime64(updatedAt, 3) as updatedAt,
1088
- resourceId
1089
- FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
1090
- ${whereClause}
1091
- ORDER BY createdAt DESC
1092
- ${limitClause}
1093
- ${offsetClause}
1094
- `,
2532
+ SELECT
2533
+ workflow_name,
2534
+ run_id,
2535
+ snapshot,
2536
+ toDateTime64(createdAt, 3) as createdAt,
2537
+ toDateTime64(updatedAt, 3) as updatedAt,
2538
+ resourceId
2539
+ FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
2540
+ ${whereClause}
2541
+ ORDER BY createdAt DESC
2542
+ ${limitClause}
2543
+ ${offsetClause}
2544
+ `,
1095
2545
  query_params: values,
1096
2546
  format: "JSONEachRow"
1097
2547
  });
@@ -1129,18 +2579,19 @@ var ClickhouseStore = class extends MastraStorage {
1129
2579
  values.var_workflow_name = workflowName;
1130
2580
  }
1131
2581
  const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
1132
- const result = await this.db.query({
2582
+ const result = await this.client.query({
1133
2583
  query: `
1134
- SELECT
1135
- workflow_name,
1136
- run_id,
1137
- snapshot,
1138
- toDateTime64(createdAt, 3) as createdAt,
1139
- toDateTime64(updatedAt, 3) as updatedAt,
1140
- resourceId
1141
- FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
1142
- ${whereClause}
1143
- `,
2584
+ SELECT
2585
+ workflow_name,
2586
+ run_id,
2587
+ snapshot,
2588
+ toDateTime64(createdAt, 3) as createdAt,
2589
+ toDateTime64(updatedAt, 3) as updatedAt,
2590
+ resourceId
2591
+ FROM ${TABLE_WORKFLOW_SNAPSHOT} ${TABLE_ENGINES[TABLE_WORKFLOW_SNAPSHOT].startsWith("ReplacingMergeTree") ? "FINAL" : ""}
2592
+ ${whereClause}
2593
+ ORDER BY createdAt DESC LIMIT 1
2594
+ `,
1144
2595
  query_params: values,
1145
2596
  format: "JSONEachRow"
1146
2597
  });
@@ -1161,45 +2612,274 @@ var ClickhouseStore = class extends MastraStorage {
1161
2612
  );
1162
2613
  }
1163
2614
  }
1164
- async hasColumn(table, column) {
1165
- const result = await this.db.query({
1166
- query: `DESCRIBE TABLE ${table}`,
1167
- format: "JSONEachRow"
2615
+ };
2616
+
2617
+ // src/storage/index.ts
2618
+ var ClickhouseStore = class extends MastraStorage {
2619
+ db;
2620
+ ttl = {};
2621
+ stores;
2622
+ constructor(config) {
2623
+ super({ name: "ClickhouseStore" });
2624
+ this.db = createClient({
2625
+ url: config.url,
2626
+ username: config.username,
2627
+ password: config.password,
2628
+ clickhouse_settings: {
2629
+ date_time_input_format: "best_effort",
2630
+ date_time_output_format: "iso",
2631
+ // This is crucial
2632
+ use_client_time_zone: 1,
2633
+ output_format_json_quote_64bit_integers: 0
2634
+ }
1168
2635
  });
1169
- const columns = await result.json();
1170
- return columns.some((c) => c.name === column);
2636
+ this.ttl = config.ttl;
2637
+ const operations = new StoreOperationsClickhouse({ client: this.db, ttl: this.ttl });
2638
+ const workflows = new WorkflowsStorageClickhouse({ client: this.db, operations });
2639
+ const scores = new ScoresStorageClickhouse({ client: this.db, operations });
2640
+ const legacyEvals = new LegacyEvalsStorageClickhouse({ client: this.db, operations });
2641
+ const traces = new TracesStorageClickhouse({ client: this.db, operations });
2642
+ const memory = new MemoryStorageClickhouse({ client: this.db, operations });
2643
+ this.stores = {
2644
+ operations,
2645
+ workflows,
2646
+ scores,
2647
+ legacyEvals,
2648
+ traces,
2649
+ memory
2650
+ };
1171
2651
  }
1172
- async getTracesPaginated(_args) {
1173
- throw new MastraError({
1174
- id: "CLICKHOUSE_STORAGE_GET_TRACES_PAGINATED_FAILED",
1175
- domain: ErrorDomain.STORAGE,
1176
- category: ErrorCategory.USER,
1177
- text: "Method not implemented."
1178
- });
2652
+ get supports() {
2653
+ return {
2654
+ selectByIncludeResourceScope: true,
2655
+ resourceWorkingMemory: true,
2656
+ hasColumn: true,
2657
+ createTable: true,
2658
+ deleteMessages: false,
2659
+ getScoresBySpan: true
2660
+ };
1179
2661
  }
1180
- async getThreadsByResourceIdPaginated(_args) {
1181
- throw new MastraError({
1182
- id: "CLICKHOUSE_STORAGE_GET_THREADS_BY_RESOURCE_ID_PAGINATED_FAILED",
1183
- domain: ErrorDomain.STORAGE,
1184
- category: ErrorCategory.USER,
1185
- text: "Method not implemented."
1186
- });
2662
+ async getEvalsByAgentName(agentName, type) {
2663
+ return this.stores.legacyEvals.getEvalsByAgentName(agentName, type);
1187
2664
  }
1188
- async getMessagesPaginated(_args) {
1189
- throw new MastraError({
1190
- id: "CLICKHOUSE_STORAGE_GET_MESSAGES_PAGINATED_FAILED",
1191
- domain: ErrorDomain.STORAGE,
1192
- category: ErrorCategory.USER,
1193
- text: "Method not implemented."
1194
- });
2665
+ async getEvals(options) {
2666
+ return this.stores.legacyEvals.getEvals(options);
2667
+ }
2668
+ async batchInsert({ tableName, records }) {
2669
+ await this.stores.operations.batchInsert({ tableName, records });
2670
+ }
2671
+ async optimizeTable({ tableName }) {
2672
+ try {
2673
+ await this.db.command({
2674
+ query: `OPTIMIZE TABLE ${tableName} FINAL`
2675
+ });
2676
+ } catch (error) {
2677
+ throw new MastraError(
2678
+ {
2679
+ id: "CLICKHOUSE_STORAGE_OPTIMIZE_TABLE_FAILED",
2680
+ domain: ErrorDomain.STORAGE,
2681
+ category: ErrorCategory.THIRD_PARTY,
2682
+ details: { tableName }
2683
+ },
2684
+ error
2685
+ );
2686
+ }
2687
+ }
2688
+ async materializeTtl({ tableName }) {
2689
+ try {
2690
+ await this.db.command({
2691
+ query: `ALTER TABLE ${tableName} MATERIALIZE TTL;`
2692
+ });
2693
+ } catch (error) {
2694
+ throw new MastraError(
2695
+ {
2696
+ id: "CLICKHOUSE_STORAGE_MATERIALIZE_TTL_FAILED",
2697
+ domain: ErrorDomain.STORAGE,
2698
+ category: ErrorCategory.THIRD_PARTY,
2699
+ details: { tableName }
2700
+ },
2701
+ error
2702
+ );
2703
+ }
2704
+ }
2705
+ async createTable({
2706
+ tableName,
2707
+ schema
2708
+ }) {
2709
+ return this.stores.operations.createTable({ tableName, schema });
2710
+ }
2711
+ async dropTable({ tableName }) {
2712
+ return this.stores.operations.dropTable({ tableName });
2713
+ }
2714
+ async alterTable({
2715
+ tableName,
2716
+ schema,
2717
+ ifNotExists
2718
+ }) {
2719
+ return this.stores.operations.alterTable({ tableName, schema, ifNotExists });
2720
+ }
2721
+ async clearTable({ tableName }) {
2722
+ return this.stores.operations.clearTable({ tableName });
2723
+ }
2724
+ async insert({ tableName, record }) {
2725
+ return this.stores.operations.insert({ tableName, record });
2726
+ }
2727
+ async load({ tableName, keys }) {
2728
+ return this.stores.operations.load({ tableName, keys });
2729
+ }
2730
+ async updateWorkflowResults({
2731
+ workflowName,
2732
+ runId,
2733
+ stepId,
2734
+ result,
2735
+ runtimeContext
2736
+ }) {
2737
+ return this.stores.workflows.updateWorkflowResults({ workflowName, runId, stepId, result, runtimeContext });
2738
+ }
2739
+ async updateWorkflowState({
2740
+ workflowName,
2741
+ runId,
2742
+ opts
2743
+ }) {
2744
+ return this.stores.workflows.updateWorkflowState({ workflowName, runId, opts });
2745
+ }
2746
+ async persistWorkflowSnapshot({
2747
+ workflowName,
2748
+ runId,
2749
+ resourceId,
2750
+ snapshot
2751
+ }) {
2752
+ return this.stores.workflows.persistWorkflowSnapshot({ workflowName, runId, resourceId, snapshot });
2753
+ }
2754
+ async loadWorkflowSnapshot({
2755
+ workflowName,
2756
+ runId
2757
+ }) {
2758
+ return this.stores.workflows.loadWorkflowSnapshot({ workflowName, runId });
2759
+ }
2760
+ async getWorkflowRuns({
2761
+ workflowName,
2762
+ fromDate,
2763
+ toDate,
2764
+ limit,
2765
+ offset,
2766
+ resourceId
2767
+ } = {}) {
2768
+ return this.stores.workflows.getWorkflowRuns({ workflowName, fromDate, toDate, limit, offset, resourceId });
2769
+ }
2770
+ async getWorkflowRunById({
2771
+ runId,
2772
+ workflowName
2773
+ }) {
2774
+ return this.stores.workflows.getWorkflowRunById({ runId, workflowName });
2775
+ }
2776
+ async getTraces(args) {
2777
+ return this.stores.traces.getTraces(args);
2778
+ }
2779
+ async getTracesPaginated(args) {
2780
+ return this.stores.traces.getTracesPaginated(args);
2781
+ }
2782
+ async batchTraceInsert(args) {
2783
+ return this.stores.traces.batchTraceInsert(args);
2784
+ }
2785
+ async getThreadById({ threadId }) {
2786
+ return this.stores.memory.getThreadById({ threadId });
2787
+ }
2788
+ async getThreadsByResourceId({ resourceId }) {
2789
+ return this.stores.memory.getThreadsByResourceId({ resourceId });
2790
+ }
2791
+ async saveThread({ thread }) {
2792
+ return this.stores.memory.saveThread({ thread });
2793
+ }
2794
+ async updateThread({
2795
+ id,
2796
+ title,
2797
+ metadata
2798
+ }) {
2799
+ return this.stores.memory.updateThread({ id, title, metadata });
2800
+ }
2801
+ async deleteThread({ threadId }) {
2802
+ return this.stores.memory.deleteThread({ threadId });
2803
+ }
2804
+ async getThreadsByResourceIdPaginated(args) {
2805
+ return this.stores.memory.getThreadsByResourceIdPaginated(args);
2806
+ }
2807
+ async getMessages({
2808
+ threadId,
2809
+ resourceId,
2810
+ selectBy,
2811
+ format
2812
+ }) {
2813
+ return this.stores.memory.getMessages({ threadId, resourceId, selectBy, format });
2814
+ }
2815
+ async getMessagesById({
2816
+ messageIds,
2817
+ format
2818
+ }) {
2819
+ return this.stores.memory.getMessagesById({ messageIds, format });
2820
+ }
2821
+ async saveMessages(args) {
2822
+ return this.stores.memory.saveMessages(args);
2823
+ }
2824
+ async getMessagesPaginated(args) {
2825
+ return this.stores.memory.getMessagesPaginated(args);
2826
+ }
2827
+ async updateMessages(args) {
2828
+ return this.stores.memory.updateMessages(args);
2829
+ }
2830
+ async getResourceById({ resourceId }) {
2831
+ return this.stores.memory.getResourceById({ resourceId });
2832
+ }
2833
+ async saveResource({ resource }) {
2834
+ return this.stores.memory.saveResource({ resource });
2835
+ }
2836
+ async updateResource({
2837
+ resourceId,
2838
+ workingMemory,
2839
+ metadata
2840
+ }) {
2841
+ return this.stores.memory.updateResource({ resourceId, workingMemory, metadata });
2842
+ }
2843
+ async getScoreById({ id }) {
2844
+ return this.stores.scores.getScoreById({ id });
2845
+ }
2846
+ async saveScore(_score) {
2847
+ return this.stores.scores.saveScore(_score);
2848
+ }
2849
+ async getScoresByRunId({
2850
+ runId,
2851
+ pagination
2852
+ }) {
2853
+ return this.stores.scores.getScoresByRunId({ runId, pagination });
2854
+ }
2855
+ async getScoresByEntityId({
2856
+ entityId,
2857
+ entityType,
2858
+ pagination
2859
+ }) {
2860
+ return this.stores.scores.getScoresByEntityId({ entityId, entityType, pagination });
2861
+ }
2862
+ async getScoresByScorerId({
2863
+ scorerId,
2864
+ pagination,
2865
+ entityId,
2866
+ entityType,
2867
+ source
2868
+ }) {
2869
+ return this.stores.scores.getScoresByScorerId({ scorerId, pagination, entityId, entityType, source });
2870
+ }
2871
+ async getScoresBySpan({
2872
+ traceId,
2873
+ spanId,
2874
+ pagination
2875
+ }) {
2876
+ return this.stores.scores.getScoresBySpan({ traceId, spanId, pagination });
1195
2877
  }
1196
2878
  async close() {
1197
2879
  await this.db.close();
1198
2880
  }
1199
- async updateMessages(_args) {
1200
- this.logger.error("updateMessages is not yet implemented in ClickhouseStore");
1201
- throw new Error("Method not implemented");
1202
- }
1203
2881
  };
1204
2882
 
1205
2883
  export { COLUMN_TYPES, ClickhouseStore, TABLE_ENGINES };
2884
+ //# sourceMappingURL=index.js.map
2885
+ //# sourceMappingURL=index.js.map